Implement string interpolation.

CHANGELOG.md

@@ -19,6 +19,7 @@ Breaking changes
 New features
 ------------
 
+* String interpolation support is added via the `` `... ${`` ... ``} ...` `` syntax.
 * `FileModuleResolver` resolves relative paths under the parent path (i.e. the path holding the script that does the loading). This allows seamless cross-loading of scripts from a directory hierarchy instead of having all relative paths load from the current working directory.
 
 

src/ast.rs

@@ -8,6 +8,7 @@ use crate::stdlib::{
     collections::BTreeMap,
     fmt,
     hash::Hash,
+    iter::empty,
     num::NonZeroUsize,
     ops::{Add, AddAssign},
     string::String,
@@ -15,6 +16,7 @@ use crate::stdlib::{
     vec::Vec,
 };
 use crate::token::Token;
+use crate::utils::calc_fn_hash;
 use crate::{
     Dynamic, FnNamespace, FnPtr, Identifier, ImmutableString, Module, Position, Shared, StaticVec,
     INT,
@@ -844,10 +846,11 @@ impl StmtBlock {
 impl fmt::Debug for StmtBlock {
     #[inline(always)]
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Debug::fmt(&self.statements, f)?;
         if !self.pos.is_none() {
-            write!(f, "{} @ ", self.pos)?;
+            write!(f, " @ {:?}", self.pos)?;
         }
-        fmt::Debug::fmt(&self.statements, f)
+        Ok(())
     }
 }
 
@@ -1293,6 +1296,18 @@ pub struct OpAssignment {
     pub op: &'static str,
 }
 
+impl OpAssignment {
+    pub fn new(op: &'static str) -> Self {
+        let op2 = &op[..op.len() - 1]; // extract operator without =
+
+        Self {
+            hash_op_assign: calc_fn_hash(empty(), op, 2),
+            hash_op: calc_fn_hash(empty(), op2, 2),
+            op,
+        }
+    }
+}
+
 /// _(INTERNALS)_ An set of function call hashes.
 /// Exported under the `internals` feature only.
 ///
@@ -1548,6 +1563,8 @@ pub enum Expr {
     StringConstant(ImmutableString, Position),
     /// [`FnPtr`] constant.
     FnPointer(ImmutableString, Position),
+    /// An interpolated [string][ImmutableString].
+    InterpolatedString(Box<StaticVec<Expr>>),
     /// [ expr, ... ]
     Array(Box<StaticVec<Expr>>, Position),
     /// #{ name:expr, ... }
@@ -1608,7 +1625,7 @@ impl Expr {
             Self::Array(x, _) if self.is_constant() => {
                 let mut arr = Array::with_capacity(x.len());
                 arr.extend(x.iter().map(|v| v.get_constant_value().unwrap()));
-                Dynamic(Union::Array(Box::new(arr), AccessMode::ReadOnly))
+                arr.into()
             }
 
             #[cfg(not(feature = "no_object"))]
@@ -1617,7 +1634,7 @@ impl Expr {
                 x.0.iter().for_each(|(k, v)| {
                     *map.get_mut(k.name.as_str()).unwrap() = v.get_constant_value().unwrap()
                 });
-                Dynamic(Union::Map(Box::new(map), AccessMode::ReadOnly))
+                map.into()
             }
 
             _ => return None,
@@ -1643,6 +1660,7 @@ impl Expr {
             Self::IntegerConstant(_, pos) => *pos,
             Self::CharConstant(_, pos) => *pos,
             Self::StringConstant(_, pos) => *pos,
+            Self::InterpolatedString(x) => x.first().unwrap().position(),
             Self::FnPointer(_, pos) => *pos,
             Self::Array(_, pos) => *pos,
             Self::Map(_, pos) => *pos,
@@ -1672,6 +1690,9 @@ impl Expr {
             Self::IntegerConstant(_, pos) => *pos = new_pos,
             Self::CharConstant(_, pos) => *pos = new_pos,
             Self::StringConstant(_, pos) => *pos = new_pos,
+            Self::InterpolatedString(x) => {
+                x.first_mut().unwrap().set_position(new_pos);
+            }
             Self::FnPointer(_, pos) => *pos = new_pos,
             Self::Array(_, pos) => *pos = new_pos,
             Self::Map(_, pos) => *pos = new_pos,
@@ -1693,7 +1714,7 @@ impl Expr {
     #[inline]
     pub fn is_pure(&self) -> bool {
         match self {
-            Self::Array(x, _) => x.iter().all(Self::is_pure),
+            Self::InterpolatedString(x) | Self::Array(x, _) => x.iter().all(Self::is_pure),
 
             Self::Map(x, _) => x.0.iter().map(|(_, v)| v).all(Self::is_pure),
 
@@ -1731,10 +1752,8 @@ impl Expr {
             | Self::FnPointer(_, _)
             | Self::Unit(_) => true,
 
-            // An array literal is constant if all items are constant
+            Self::InterpolatedString(x) | Self::Array(x, _) => x.iter().all(Self::is_constant),
-            Self::Array(x, _) => x.iter().all(Self::is_constant),
 
-            // An map literal is constant if all items are constant
             Self::Map(x, _) => x.0.iter().map(|(_, expr)| expr).all(Self::is_constant),
 
             _ => false,
@@ -1763,6 +1782,7 @@ impl Expr {
             | Self::Unit(_) => false,
 
             Self::StringConstant(_, _)
+            | Self::InterpolatedString(_)
             | Self::FnCall(_, _)
             | Self::Stmt(_)
             | Self::Dot(_, _)
@@ -1814,7 +1834,7 @@ impl Expr {
                     }
                 }
             }
-            Self::Array(x, _) => {
+            Self::InterpolatedString(x) | Self::Array(x, _) => {
                 for e in x.as_ref() {
                     if !e.walk(path, on_node) {
                         return false;

src/engine.rs

@@ -25,8 +25,8 @@ use crate::stdlib::{
 use crate::syntax::CustomSyntax;
 use crate::utils::get_hasher;
 use crate::{
-    Dynamic, EvalAltResult, FnPtr, Identifier, Module, Position, RhaiResult, Scope, Shared,
+    Dynamic, EvalAltResult, FnPtr, Identifier, ImmutableString, Module, Position, RhaiResult,
-    StaticVec,
+    Scope, Shared, StaticVec,
 };
 
 #[cfg(not(feature = "no_index"))]
@@ -200,6 +200,9 @@ pub const FN_ANONYMOUS: &str = "anon$";
 /// Standard equality comparison operator.
 pub const OP_EQUALS: &str = "==";
 
+/// Standard concatenation operator.
+pub const OP_CONCAT: &str = "+=";
+
 /// Standard method function for containment testing.
 ///
 /// The `in` operator is implemented as a call to this method.
@@ -410,7 +413,7 @@ impl<'a> Target<'a> {
             Self::Value(_) => panic!("cannot update a value"),
             #[cfg(not(feature = "no_index"))]
             Self::StringChar(s, index, _) => {
-                let s = &mut *s.write_lock::<crate::ImmutableString>().unwrap();
+                let s = &mut *s.write_lock::<ImmutableString>().unwrap();
 
                 // Replace the character at the specified index position
                 let new_ch = new_val.as_char().map_err(|err| {
@@ -591,7 +594,7 @@ pub struct Limits {
     /// Not available under `no_module`.
     #[cfg(not(feature = "no_module"))]
     pub max_modules: usize,
-    /// Maximum length of a [string][crate::ImmutableString].
+    /// Maximum length of a [string][ImmutableString].
     pub max_string_size: Option<NonZeroUsize>,
     /// Maximum length of an [array][Array].
     ///
@@ -714,6 +717,9 @@ pub struct Engine {
     /// A map mapping type names to pretty-print names.
     pub(crate) type_names: BTreeMap<Identifier, Identifier>,
 
+    /// An empty [`ImmutableString`] for cloning purposes.
+    pub(crate) empty_string: ImmutableString,
+
     /// A set of symbols to disable.
     pub(crate) disabled_symbols: BTreeSet<Identifier>,
     /// A map containing custom keywords and precedence to recognize.
@@ -815,6 +821,7 @@ impl Engine {
             module_resolver: Box::new(crate::module::resolvers::DummyModuleResolver::new()),
 
             type_names: Default::default(),
+            empty_string: Default::default(),
             disabled_symbols: Default::default(),
             custom_keywords: Default::default(),
             custom_syntax: Default::default(),
@@ -875,6 +882,7 @@ impl Engine {
             module_resolver: Box::new(crate::module::resolvers::DummyModuleResolver::new()),
 
             type_names: Default::default(),
+            empty_string: Default::default(),
             disabled_symbols: Default::default(),
             custom_keywords: Default::default(),
             custom_syntax: Default::default(),
@@ -1587,8 +1595,8 @@ impl Engine {
             #[cfg(not(feature = "no_object"))]
             Dynamic(Union::Map(map, _)) => {
                 // val_map[idx]
-                let index = &*idx.read_lock::<crate::ImmutableString>().ok_or_else(|| {
+                let index = &*idx.read_lock::<ImmutableString>().ok_or_else(|| {
-                    self.make_type_mismatch_err::<crate::ImmutableString>(idx.type_name(), idx_pos)
+                    self.make_type_mismatch_err::<ImmutableString>(idx.type_name(), idx_pos)
                 })?;
 
                 if _create && !map.contains_key(index.as_str()) {
@@ -1698,6 +1706,34 @@ impl Engine {
                 self.eval_dot_index_chain(scope, mods, state, lib, this_ptr, expr, level, None)
             }
 
+            // `... ${...} ...`
+            Expr::InterpolatedString(x) => {
+                let mut pos = expr.position();
+                let mut result: Dynamic = self.empty_string.clone().into();
+
+                for expr in x.iter() {
+                    let item = self.eval_expr(scope, mods, state, lib, this_ptr, expr, level)?;
+                    self.eval_op_assignment(
+                        mods,
+                        state,
+                        lib,
+                        Some(OpAssignment::new(OP_CONCAT)),
+                        pos,
+                        (&mut result).into(),
+                        item,
+                        expr.position(),
+                    )?;
+                    pos = expr.position();
+                }
+
+                assert!(
+                    result.is::<ImmutableString>(),
+                    "interpolated string must be a string"
+                );
+
+                Ok(result)
+            }
+
             #[cfg(not(feature = "no_index"))]
             Expr::Array(x, _) => {
                 let mut arr = Array::with_capacity(x.len());
@@ -1707,7 +1743,7 @@ impl Engine {
                             .flatten(),
                     );
                 }
-                Ok(Dynamic(Union::Array(Box::new(arr), AccessMode::ReadWrite)))
+                Ok(arr.into())
             }
 
             #[cfg(not(feature = "no_object"))]
@@ -1718,7 +1754,7 @@ impl Engine {
                         .eval_expr(scope, mods, state, lib, this_ptr, expr, level)?
                         .flatten();
                 }
-                Ok(Dynamic(Union::Map(Box::new(map), AccessMode::ReadWrite)))
+                Ok(map.into())
             }
 
             // Normal function call
@@ -2445,7 +2481,7 @@ impl Engine {
 
                 if let Some(path) = self
                     .eval_expr(scope, mods, state, lib, this_ptr, &expr, level)?
-                    .try_cast::<crate::ImmutableString>()
+                    .try_cast::<ImmutableString>()
                 {
                     use crate::ModuleResolver;
 
@@ -2481,7 +2517,7 @@ impl Engine {
 
                     Ok(Dynamic::UNIT)
                 } else {
-                    Err(self.make_type_mismatch_err::<crate::ImmutableString>("", expr.position()))
+                    Err(self.make_type_mismatch_err::<ImmutableString>("", expr.position()))
                 }
             }
 

src/engine_api.rs

@@ -5,9 +5,11 @@ use crate::engine::{EvalContext, Imports, State};
 use crate::fn_native::{FnCallArgs, SendSync};
 use crate::fn_register::RegisterNativeFunction;
 use crate::optimize::OptimizationLevel;
+use crate::parser::ParseState;
 use crate::stdlib::{
     any::{type_name, TypeId},
     boxed::Box,
+    num::NonZeroUsize,
     string::String,
 };
 use crate::{
@@ -1156,8 +1158,22 @@ impl Engine {
         scripts: &[&str],
         optimization_level: OptimizationLevel,
     ) -> Result<AST, ParseError> {
-        let stream = self.lex_raw(scripts, None);
+        let (stream, buffer) = self.lex_raw(scripts, None);
-        self.parse(&mut stream.peekable(), scope, optimization_level)
+        let mut state = ParseState::new(
+            self,
+            buffer,
+            #[cfg(not(feature = "unchecked"))]
+            NonZeroUsize::new(self.max_expr_depth()),
+            #[cfg(not(feature = "unchecked"))]
+            #[cfg(not(feature = "no_function"))]
+            NonZeroUsize::new(self.max_function_expr_depth()),
+        );
+        self.parse(
+            &mut stream.peekable(),
+            &mut state,
+            scope,
+            optimization_level,
+        )
     }
     /// Read the contents of a file into a string.
     #[cfg(not(feature = "no_std"))]
@@ -1331,7 +1347,7 @@ impl Engine {
             .into());
         };
 
-        let stream = self.lex_raw(
+        let (stream, buffer) = self.lex_raw(
             &scripts,
             Some(if has_null {
                 |token| match token {
@@ -1344,8 +1360,22 @@ impl Engine {
             }),
         );
 
-        let ast =
+        let mut state = ParseState::new(
-            self.parse_global_expr(&mut stream.peekable(), &scope, OptimizationLevel::None)?;
+            self,
+            buffer,
+            #[cfg(not(feature = "unchecked"))]
+            NonZeroUsize::new(self.max_expr_depth()),
+            #[cfg(not(feature = "unchecked"))]
+            #[cfg(not(feature = "no_function"))]
+            NonZeroUsize::new(self.max_function_expr_depth()),
+        );
+
+        let ast = self.parse_global_expr(
+            &mut stream.peekable(),
+            &mut state,
+            &scope,
+            OptimizationLevel::None,
+        )?;
 
         // Handle null - map to ()
         if has_null {
@@ -1424,10 +1454,19 @@ impl Engine {
         script: &str,
     ) -> Result<AST, ParseError> {
         let scripts = [script];
-        let stream = self.lex_raw(&scripts, None);
+        let (stream, buffer) = self.lex_raw(&scripts, None);
 
         let mut peekable = stream.peekable();
-        self.parse_global_expr(&mut peekable, scope, self.optimization_level)
+        let mut state = ParseState::new(
+            self,
+            buffer,
+            #[cfg(not(feature = "unchecked"))]
+            NonZeroUsize::new(self.max_expr_depth()),
+            #[cfg(not(feature = "unchecked"))]
+            #[cfg(not(feature = "no_function"))]
+            NonZeroUsize::new(self.max_function_expr_depth()),
+        );
+        self.parse_global_expr(&mut peekable, &mut state, scope, self.optimization_level)
     }
     /// Evaluate a script file.
     ///
@@ -1585,10 +1624,24 @@ impl Engine {
         script: &str,
     ) -> Result<T, Box<EvalAltResult>> {
         let scripts = [script];
-        let stream = self.lex_raw(&scripts, None);
+        let (stream, buffer) = self.lex_raw(&scripts, None);
+        let mut state = ParseState::new(
+            self,
+            buffer,
+            #[cfg(not(feature = "unchecked"))]
+            NonZeroUsize::new(self.max_expr_depth()),
+            #[cfg(not(feature = "unchecked"))]
+            #[cfg(not(feature = "no_function"))]
+            NonZeroUsize::new(self.max_function_expr_depth()),
+        );
 
         // No need to optimize a lone expression
-        let ast = self.parse_global_expr(&mut stream.peekable(), scope, OptimizationLevel::None)?;
+        let ast = self.parse_global_expr(
+            &mut stream.peekable(),
+            &mut state,
+            scope,
+            OptimizationLevel::None,
+        )?;
 
         self.eval_ast_with_scope(scope, &ast)
     }
@@ -1726,8 +1779,24 @@ impl Engine {
         script: &str,
     ) -> Result<(), Box<EvalAltResult>> {
         let scripts = [script];
-        let stream = self.lex_raw(&scripts, None);
+        let (stream, buffer) = self.lex_raw(&scripts, None);
-        let ast = self.parse(&mut stream.peekable(), scope, self.optimization_level)?;
+        let mut state = ParseState::new(
+            self,
+            buffer,
+            #[cfg(not(feature = "unchecked"))]
+            NonZeroUsize::new(self.max_expr_depth()),
+            #[cfg(not(feature = "unchecked"))]
+            #[cfg(not(feature = "no_function"))]
+            NonZeroUsize::new(self.max_function_expr_depth()),
+        );
+
+        let ast = self.parse(
+            &mut stream.peekable(),
+            &mut state,
+            scope,
+            self.optimization_level,
+        )?;
+
         self.consume_ast_with_scope(scope, &ast)
     }
     /// Evaluate an AST, but throw away the result and only return error (if any).

src/optimize.rs

@@ -609,6 +609,16 @@ fn optimize_expr(expr: &mut Expr, state: &mut State) {
     match expr {
         // {}
         Expr::Stmt(x) if x.statements.is_empty() => { state.set_dirty(); *expr = Expr::Unit(x.pos) }
+        // { Stmt(Expr) }
+        Expr::Stmt(x) if x.statements.len() == 1 && x.statements[0].is_pure() && matches!(x.statements[0], Stmt::Expr(_)) =>
+        {
+            state.set_dirty();
+            if let Stmt::Expr(e) = mem::take(&mut x.statements[0]) {
+                *expr = e;
+            } else {
+                unreachable!();                
+            }
+        }
         // { stmt; ... } - do not count promotion as dirty because it gets turned back into an array
         Expr::Stmt(x) => x.statements = optimize_stmt_block(mem::take(&mut x.statements).into_vec(), state, true, true, false).into(),
         // lhs.rhs
@@ -664,6 +674,59 @@ fn optimize_expr(expr: &mut Expr, state: &mut State) {
             // lhs[rhs]
             (lhs, rhs) => { optimize_expr(lhs, state); optimize_expr(rhs, state); }
         },
+        // ``
+        Expr::InterpolatedString(x) if x.is_empty() => {
+            state.set_dirty();
+            *expr = Expr::StringConstant(state.engine.empty_string.clone(), Position::NONE);
+        }
+        // `...`
+        Expr::InterpolatedString(x) if x.len() == 1 && matches!(x[0], Expr::StringConstant(_, _)) => {
+            state.set_dirty();
+            *expr = mem::take(&mut x[0]);
+        }
+        // `... ${ ... } ...`
+        Expr::InterpolatedString(x) => {
+            x.iter_mut().for_each(|expr| optimize_expr(expr, state));
+
+            let mut n= 0;
+
+            // Merge consecutive strings
+            while n < x.len()-1 {
+                match (mem::take(&mut x[n]), mem::take(&mut x[n+1])) {
+                    (Expr::StringConstant(mut s1, pos), Expr::StringConstant(s2, _)) => {
+                        s1 += s2;
+                        x[n] = Expr::StringConstant(s1, pos);
+                        x.remove(n+1);
+                        state.set_dirty();
+                    }
+                    (expr1, Expr::Unit(_))  => {
+                        x[n] = expr1;
+                        x.remove(n+1);
+                        state.set_dirty();
+                    }
+                    (Expr::Unit(_), expr2) => {
+                        x[n+1] = expr2;
+                        x.remove(n);
+                        state.set_dirty();
+                    }
+                    (expr1, Expr::StringConstant(s, _)) if s.is_empty() => {
+                        x[n] = expr1;
+                        x.remove(n+1);
+                        state.set_dirty();
+                    }
+                    (Expr::StringConstant(s, _), expr2) if s.is_empty()=> {
+                        x[n+1] = expr2;
+                        x.remove(n);
+                        state.set_dirty();
+                    }
+                    (expr1, expr2) => {
+                        x[n] = expr1;
+                        x[n+1] = expr2;
+                        n += 1;
+                    }
+                }
+            }
+        }
         // [ constant .. ]
         #[cfg(not(feature = "no_index"))]
         Expr::Array(_, _) if expr.is_constant() => {

src/packages/string_more.rs

@@ -23,9 +23,26 @@ mod string_functions {
     use crate::ImmutableString;
 
     #[rhai_fn(name = "+", name = "append")]
-    pub fn add_append(ctx: NativeCallContext, string: &str, mut item: Dynamic) -> ImmutableString {
+    pub fn add_append(
+        ctx: NativeCallContext,
+        string: ImmutableString,
+        mut item: Dynamic,
+    ) -> ImmutableString {
         let s = print_with_func(FUNC_TO_STRING, &ctx, &mut item);
-        format!("{}{}", string, s).into()
+
+        if s.is_empty() {
+            string
+        } else {
+            format!("{}{}", string, s).into()
+        }
+    }
+    #[rhai_fn(name = "+=")]
+    pub fn append(ctx: NativeCallContext, string: &mut ImmutableString, mut item: Dynamic) {
+        let s = print_with_func(FUNC_TO_STRING, &ctx, &mut item);
+
+        if !s.is_empty() {
+            string.make_mut().push_str(&s);
+        }
     }
     #[rhai_fn(name = "+", pure)]
     pub fn add_prepend(
@@ -33,8 +50,22 @@ mod string_functions {
         item: &mut Dynamic,
         string: &str,
     ) -> ImmutableString {
-        let s = print_with_func(FUNC_TO_STRING, &ctx, item);
+        let mut s = print_with_func(FUNC_TO_STRING, &ctx, item);
-        format!("{}{}", s, string).into()
+
+        if string.is_empty() {
+            s
+        } else {
+            s.make_mut().push_str(string);
+            s.into()
+        }
+    }
+    #[rhai_fn(name = "+=")]
+    pub fn prepend(ctx: NativeCallContext, item: &mut Dynamic, string: &str) {
+        let mut s = print_with_func(FUNC_TO_STRING, &ctx, item);
+
+        if !string.is_empty() {
+            s.make_mut().push_str(string);
+        }
     }
 
     #[rhai_fn(name = "+")]
@@ -137,13 +168,18 @@ mod string_functions {
             .unwrap_or(-1 as INT)
     }
 
-    pub fn sub_string(string: &str, start: INT, len: INT) -> ImmutableString {
+    pub fn sub_string(
+        ctx: NativeCallContext,
+        string: &str,
+        start: INT,
+        len: INT,
+    ) -> ImmutableString {
         let offset = if string.is_empty() || len <= 0 {
-            return "".to_string().into();
+            return ctx.engine().empty_string.clone().into();
         } else if start < 0 {
             0
         } else if start as usize >= string.chars().count() {
-            return "".to_string().into();
+            return ctx.engine().empty_string.clone().into();
         } else {
             start as usize
         };
@@ -165,9 +201,13 @@ mod string_functions {
             .into()
     }
     #[rhai_fn(name = "sub_string")]
-    pub fn sub_string_starting_from(string: &str, start: INT) -> ImmutableString {
+    pub fn sub_string_starting_from(
+        ctx: NativeCallContext,
+        string: &str,
+        start: INT,
+    ) -> ImmutableString {
         let len = string.len() as INT;
-        sub_string(string, start, len)
+        sub_string(ctx, string, start, len)
     }
 
     #[rhai_fn(name = "crop")]
@@ -341,9 +381,9 @@ mod string_functions {
             string.chars().map(Into::<Dynamic>::into).collect()
         }
         #[rhai_fn(name = "split")]
-        pub fn split_at(string: ImmutableString, start: INT) -> Array {
+        pub fn split_at(ctx: NativeCallContext, string: ImmutableString, start: INT) -> Array {
             if start <= 0 {
-                vec!["".into(), string.into()]
+                vec![ctx.engine().empty_string.clone().into(), string.into()]
             } else {
                 let prefix: String = string.chars().take(start as usize).collect();
                 let prefix_len = prefix.len();

src/parser.rs

@@ -11,6 +11,7 @@ use crate::optimize::optimize_into_ast;
 use crate::optimize::OptimizationLevel;
 use crate::stdlib::{
     boxed::Box,
+    cell::Cell,
     collections::BTreeMap,
     format,
     hash::{Hash, Hasher},
@@ -40,9 +41,11 @@ type FunctionsLib = BTreeMap<u64, Shared<ScriptFnDef>>;
 
 /// A type that encapsulates the current state of the parser.
 #[derive(Debug)]
-struct ParseState<'e> {
+pub struct ParseState<'e> {
     /// Reference to the scripting [`Engine`].
     engine: &'e Engine,
+    /// Input stream buffer containing the next character to read.
+    buffer: Shared<Cell<Option<char>>>,
     /// Interned strings.
     interned_strings: IdentifierBuilder,
     /// Encapsulates a local stack with variable names to simulate an actual runtime scope.
@@ -75,6 +78,7 @@ impl<'e> ParseState<'e> {
     #[inline(always)]
     pub fn new(
         engine: &'e Engine,
+        buffer: Shared<Cell<Option<char>>>,
         #[cfg(not(feature = "unchecked"))] max_expr_depth: Option<NonZeroUsize>,
         #[cfg(not(feature = "unchecked"))]
         #[cfg(not(feature = "no_function"))]
@@ -82,6 +86,7 @@ impl<'e> ParseState<'e> {
     ) -> Self {
         Self {
             engine,
+            buffer,
             #[cfg(not(feature = "unchecked"))]
             max_expr_depth,
             #[cfg(not(feature = "unchecked"))]
@@ -458,7 +463,7 @@ fn parse_index_chain(
             .into_err(*pos))
         }
         Expr::IntegerConstant(_, pos) => match lhs {
-            Expr::Array(_, _) | Expr::StringConstant(_, _) => (),
+            Expr::Array(_, _) | Expr::StringConstant(_, _) | Expr::InterpolatedString(_) => (),
 
             Expr::Map(_, _) => {
                 return Err(PERR::MalformedIndexExpr(
@@ -490,14 +495,14 @@ fn parse_index_chain(
         },
 
         // lhs[string]
-        Expr::StringConstant(_, pos) => match lhs {
+        Expr::StringConstant(_, _) | Expr::InterpolatedString(_) => match lhs {
             Expr::Map(_, _) => (),
 
-            Expr::Array(_, _) | Expr::StringConstant(_, _) => {
+            Expr::Array(_, _) | Expr::StringConstant(_, _) | Expr::InterpolatedString(_) => {
                 return Err(PERR::MalformedIndexExpr(
                     "Array or string expects numeric index, not a string".into(),
                 )
-                .into_err(*pos))
+                .into_err(idx_expr.position()))
             }
 
             #[cfg(not(feature = "no_float"))]
@@ -979,6 +984,7 @@ fn parse_primary(
         Token::Pipe | Token::Or if settings.allow_anonymous_fn => {
             let mut new_state = ParseState::new(
                 state.engine,
+                state.buffer.clone(),
                 #[cfg(not(feature = "unchecked"))]
                 state.max_function_expr_depth,
                 #[cfg(not(feature = "unchecked"))]
@@ -1010,6 +1016,50 @@ fn parse_primary(
             expr
         }
 
+        // Interpolated string
+        Token::InterpolatedString(_) => {
+            let mut segments: StaticVec<Expr> = Default::default();
+
+            if let (Token::InterpolatedString(s), pos) = input.next().unwrap() {
+                segments.push(Expr::StringConstant(s.into(), pos));
+            } else {
+                unreachable!();
+            }
+
+            loop {
+                let expr = match parse_block(input, state, lib, settings.level_up())? {
+                    block @ Stmt::Block(_, _) => Expr::Stmt(Box::new(block.into())),
+                    stmt => unreachable!("expecting Stmt::Block, but gets {:?}", stmt),
+                };
+                segments.push(expr);
+
+                // Make sure to parse the following as text
+                state.buffer.set(Some('`'));
+
+                match input.next().unwrap() {
+                    (Token::StringConstant(s), pos) => {
+                        if !s.is_empty() {
+                            segments.push(Expr::StringConstant(s.into(), pos));
+                        }
+                        // End the interpolated string if it is terminated by a back-tick.
+                        break;
+                    }
+                    (Token::InterpolatedString(s), pos) => {
+                        if !s.is_empty() {
+                            segments.push(Expr::StringConstant(s.into(), pos));
+                        }
+                    }
+                    (token, _) => unreachable!(
+                        "expected a string within an interpolated string literal, but gets {:?}",
+                        token
+                    ),
+                }
+            }
+
+            println!("Interpolated string: {:?}", segments);
+            Expr::InterpolatedString(Box::new(segments))
+        }
+
         // Array literal
         #[cfg(not(feature = "no_index"))]
         Token::LeftBracket => parse_array_literal(input, state, lib, settings.level_up())?,
@@ -1020,8 +1070,8 @@ fn parse_primary(
 
         // Identifier
         Token::Identifier(_) => {
-            let s = match input.next().unwrap().0 {
+            let s = match input.next().unwrap() {
-                Token::Identifier(s) => s,
+                (Token::Identifier(s), _) => s,
                 _ => unreachable!(),
             };
 
@@ -1067,8 +1117,8 @@ fn parse_primary(
 
         // Reserved keyword or symbol
         Token::Reserved(_) => {
-            let s = match input.next().unwrap().0 {
+            let s = match input.next().unwrap() {
-                Token::Reserved(s) => s,
+                (Token::Reserved(s), _) => s,
                 _ => unreachable!(),
             };
 
@@ -1101,14 +1151,10 @@ fn parse_primary(
             }
         }
 
-        Token::LexError(_) => {
+        Token::LexError(_) => match input.next().unwrap() {
-            let err = match input.next().unwrap().0 {
+            (Token::LexError(err), _) => return Err(err.into_err(settings.pos)),
-                Token::LexError(err) => err,
+            _ => unreachable!(),
-                _ => unreachable!(),
+        },
-            };
-
-            return Err(err.into_err(settings.pos));
-        }
 
         _ => {
             return Err(LexError::UnexpectedInput(token.syntax().to_string()).into_err(settings.pos))
@@ -1374,13 +1420,7 @@ fn make_assignment_stmt<'a>(
     let op_info = if op.is_empty() {
         None
     } else {
-        let op2 = &op[..op.len() - 1]; // extract operator without =
+        Some(OpAssignment::new(op))
-
-        Some(OpAssignment {
-            hash_op_assign: calc_fn_hash(empty(), &op, 2),
-            hash_op: calc_fn_hash(empty(), op2, 2),
-            op,
-        })
     };
 
     match &lhs {
@@ -1460,7 +1500,7 @@ fn parse_op_assignment_stmt(
     settings.pos = *token_pos;
 
     let op = match token {
-        Token::Equals => "".into(),
+        Token::Equals => "",
 
         Token::PlusAssign
         | Token::MinusAssign
@@ -1797,9 +1837,10 @@ fn parse_custom_syntax(
             // Add enough empty variable names to the stack.
             // Empty variable names act as a barrier so earlier variables will not be matched.
             // Variable searches stop at the first empty variable name.
+            let empty = state.get_identifier("");
             state.stack.resize(
                 state.stack.len() + delta as usize,
-                ("".into(), AccessMode::ReadWrite),
+                (empty, AccessMode::ReadWrite),
             );
         }
         delta if delta < 0 && state.stack.len() <= delta.abs() as usize => state.stack.clear(),
@@ -2502,6 +2543,7 @@ fn parse_stmt(
                 (Token::Fn, pos) => {
                     let mut new_state = ParseState::new(
                         state.engine,
+                        state.buffer.clone(),
                         #[cfg(not(feature = "unchecked"))]
                         state.max_function_expr_depth,
                         #[cfg(not(feature = "unchecked"))]
@@ -2930,18 +2972,11 @@ impl Engine {
     pub(crate) fn parse_global_expr(
         &self,
         input: &mut TokenStream,
+        state: &mut ParseState,
         scope: &Scope,
         optimization_level: OptimizationLevel,
     ) -> Result<AST, ParseError> {
         let mut functions = Default::default();
-        let mut state = ParseState::new(
-            self,
-            #[cfg(not(feature = "unchecked"))]
-            NonZeroUsize::new(self.max_expr_depth()),
-            #[cfg(not(feature = "unchecked"))]
-            #[cfg(not(feature = "no_function"))]
-            NonZeroUsize::new(self.max_function_expr_depth()),
-        );
 
         let settings = ParseSettings {
             allow_if_expr: false,
@@ -2954,7 +2989,7 @@ impl Engine {
             level: 0,
             pos: Position::NONE,
         };
-        let expr = parse_expr(input, &mut state, &mut functions, settings)?;
+        let expr = parse_expr(input, state, &mut functions, settings)?;
 
         assert!(functions.is_empty());
 
@@ -2978,17 +3013,10 @@ impl Engine {
     fn parse_global_level(
         &self,
         input: &mut TokenStream,
+        state: &mut ParseState,
     ) -> Result<(Vec<Stmt>, Vec<Shared<ScriptFnDef>>), ParseError> {
         let mut statements = Vec::with_capacity(16);
         let mut functions = BTreeMap::new();
-        let mut state = ParseState::new(
-            self,
-            #[cfg(not(feature = "unchecked"))]
-            NonZeroUsize::new(self.max_expr_depth()),
-            #[cfg(not(feature = "unchecked"))]
-            #[cfg(not(feature = "no_function"))]
-            NonZeroUsize::new(self.max_function_expr_depth()),
-        );
 
         while !input.peek().unwrap().0.is_eof() {
             let settings = ParseSettings {
@@ -3003,7 +3031,7 @@ impl Engine {
                 pos: Position::NONE,
             };
 
-            let stmt = parse_stmt(input, &mut state, &mut functions, settings)?;
+            let stmt = parse_stmt(input, state, &mut functions, settings)?;
 
             if stmt.is_noop() {
                 continue;
@@ -3046,10 +3074,11 @@ impl Engine {
     pub(crate) fn parse(
         &self,
         input: &mut TokenStream,
+        state: &mut ParseState,
         scope: &Scope,
         optimization_level: OptimizationLevel,
     ) -> Result<AST, ParseError> {
-        let (statements, lib) = self.parse_global_level(input)?;
+        let (statements, lib) = self.parse_global_level(input, state)?;
 
         Ok(
             // Optimize AST

src/token.rs

@@ -1,11 +1,14 @@
 //! Main module defining the lexer and parser.
 
+use std::iter::FusedIterator;
+
 use crate::engine::{
     Precedence, KEYWORD_DEBUG, KEYWORD_EVAL, KEYWORD_FN_PTR, KEYWORD_FN_PTR_CALL,
     KEYWORD_FN_PTR_CURRY, KEYWORD_IS_DEF_VAR, KEYWORD_PRINT, KEYWORD_THIS, KEYWORD_TYPE_OF,
 };
 use crate::stdlib::{
     borrow::Cow,
+    cell::Cell,
     char, fmt, format,
     iter::Peekable,
     num::NonZeroUsize,
@@ -13,7 +16,7 @@ use crate::stdlib::{
     str::{Chars, FromStr},
     string::{String, ToString},
 };
-use crate::{Engine, LexError, StaticVec, INT};
+use crate::{Engine, LexError, Shared, StaticVec, INT};
 
 #[cfg(not(feature = "no_float"))]
 use crate::ast::FloatWrapper;
@@ -209,6 +212,8 @@ pub enum Token {
     CharConstant(char),
     /// A string constant.
     StringConstant(String),
+    /// An interpolated string.
+    InterpolatedString(String),
     /// `{`
     LeftBrace,
     /// `}`
@@ -485,6 +490,7 @@ impl Token {
             #[cfg(feature = "decimal")]
             DecimalConstant(d) => d.to_string().into(),
             StringConstant(_) => "string".into(),
+            InterpolatedString(_) => "string".into(),
             CharConstant(c) => c.to_string().into(),
             Identifier(s) => s.clone().into(),
             Reserved(s) => s.clone().into(),
@@ -855,18 +861,30 @@ pub fn parse_string_literal(
     termination_char: char,
     continuation: bool,
     verbatim: bool,
-) -> Result<String, (LexError, Position)> {
+    allow_interpolation: bool,
+) -> Result<(String, bool), (LexError, Position)> {
     let mut result: smallvec::SmallVec<[char; 16]> = Default::default();
     let mut escape: smallvec::SmallVec<[char; 12]> = Default::default();
 
     let start = *pos;
     let mut skip_whitespace_until = 0;
+    let mut interpolated = false;
 
     loop {
         let next_char = stream.get_next().ok_or((LERR::UnterminatedString, start))?;
 
         pos.advance();
 
+        // String interpolation?
+        if allow_interpolation
+            && next_char == '$'
+            && escape.is_empty()
+            && stream.peek_next().map(|ch| ch == '{').unwrap_or(false)
+        {
+            interpolated = true;
+            break;
+        }
+
         if let Some(max) = state.max_string_size {
             if result.len() > max.get() {
                 return Err((LexError::StringTooLong(max.get()), *pos));
@@ -1000,7 +1018,7 @@ pub fn parse_string_literal(
         }
     }
 
-    Ok(s)
+    Ok((s, interpolated))
 }
 
 /// Consume the next character.
@@ -1296,10 +1314,11 @@ fn get_next_token_inner(
 
             // " - string literal
             ('"', _) => {
-                return parse_string_literal(stream, state, pos, c, true, false).map_or_else(
+                return parse_string_literal(stream, state, pos, c, true, false, false)
-                    |err| Some((Token::LexError(err.0), err.1)),
+                    .map_or_else(
-                    |out| Some((Token::StringConstant(out), start_pos)),
+                        |err| Some((Token::LexError(err.0), err.1)),
-                );
+                        |(result, _)| Some((Token::StringConstant(result), start_pos)),
+                    );
             }
             // ` - string literal
             ('`', _) => {
@@ -1320,9 +1339,15 @@ fn get_next_token_inner(
                     _ => (),
                 }
 
-                return parse_string_literal(stream, state, pos, c, false, true).map_or_else(
+                return parse_string_literal(stream, state, pos, c, false, true, true).map_or_else(
                     |err| Some((Token::LexError(err.0), err.1)),
-                    |out| Some((Token::StringConstant(out), start_pos)),
+                    |(result, interpolated)| {
+                        if interpolated {
+                            Some((Token::InterpolatedString(result), start_pos))
+                        } else {
+                            Some((Token::StringConstant(result), start_pos))
+                        }
+                    },
                 );
             }
 
@@ -1335,9 +1360,9 @@ fn get_next_token_inner(
             }
             ('\'', _) => {
                 return Some(
-                    parse_string_literal(stream, state, pos, c, false, false).map_or_else(
+                    parse_string_literal(stream, state, pos, c, false, false, false).map_or_else(
                         |err| (Token::LexError(err.0), err.1),
-                        |result| {
+                        |(result, _)| {
                             let mut chars = result.chars();
                             let first = chars.next().unwrap();
 
@@ -1765,6 +1790,10 @@ pub struct MultiInputsStream<'a> {
 impl InputStream for MultiInputsStream<'_> {
     #[inline(always)]
     fn unget(&mut self, ch: char) {
+        if self.buf.is_some() {
+            panic!("cannot unget two characters in a row");
+        }
+
         self.buf = Some(ch);
     }
     fn get_next(&mut self) -> Option<char> {
@@ -1813,6 +1842,8 @@ pub struct TokenIterator<'a> {
     state: TokenizeState,
     /// Current position.
     pos: Position,
+    /// Buffer containing the next character to read, if any.
+    buffer: Shared<Cell<Option<char>>>,
     /// Input character stream.
     stream: MultiInputsStream<'a>,
     /// A processor function that maps a token to another.
@@ -1823,6 +1854,11 @@ impl<'a> Iterator for TokenIterator<'a> {
     type Item = (Token, Position);
 
     fn next(&mut self) -> Option<Self::Item> {
+        if let Some(ch) = self.buffer.take() {
+            self.stream.unget(ch);
+            self.pos.rewind();
+        }
+
         let (token, pos) = match get_next_token(&mut self.stream, &mut self.state, &mut self.pos) {
             // {EOF}
             None => return None,
@@ -1901,12 +1937,17 @@ impl<'a> Iterator for TokenIterator<'a> {
     }
 }
 
+impl FusedIterator for TokenIterator<'_> {}
+
 impl Engine {
     /// _(INTERNALS)_ Tokenize an input text stream.
     /// Exported under the `internals` feature only.
     #[cfg(feature = "internals")]
     #[inline(always)]
-    pub fn lex<'a>(&'a self, input: impl IntoIterator<Item = &'a &'a str>) -> TokenIterator<'a> {
+    pub fn lex<'a>(
+        &'a self,
+        input: impl IntoIterator<Item = &'a &'a str>,
+    ) -> (TokenIterator<'a>, Shared<Cell<Option<char>>>) {
         self.lex_raw(input, None)
     }
     /// _(INTERNALS)_ Tokenize an input text stream with a mapping function.
@@ -1917,7 +1958,7 @@ impl Engine {
         &'a self,
         input: impl IntoIterator<Item = &'a &'a str>,
         map: fn(Token) -> Token,
-    ) -> TokenIterator<'a> {
+    ) -> (TokenIterator<'a>, Shared<Cell<Option<char>>>) {
         self.lex_raw(input, Some(map))
     }
     /// Tokenize an input text stream with an optional mapping function.
@@ -1926,27 +1967,34 @@ impl Engine {
         &'a self,
         input: impl IntoIterator<Item = &'a &'a str>,
         map: Option<fn(Token) -> Token>,
-    ) -> TokenIterator<'a> {
+    ) -> (TokenIterator<'a>, Shared<Cell<Option<char>>>) {
-        TokenIterator {
+        let buffer: Shared<Cell<Option<char>>> = Cell::new(None).into();
-            engine: self,
+        let buffer2 = buffer.clone();
-            state: TokenizeState {
+
-                #[cfg(not(feature = "unchecked"))]
+        (
-                max_string_size: self.limits.max_string_size,
+            TokenIterator {
-                #[cfg(feature = "unchecked")]
+                engine: self,
-                max_string_size: None,
+                state: TokenizeState {
-                non_unary: false,
+                    #[cfg(not(feature = "unchecked"))]
-                comment_level: 0,
+                    max_string_size: self.limits.max_string_size,
-                end_with_none: false,
+                    #[cfg(feature = "unchecked")]
-                include_comments: false,
+                    max_string_size: None,
-                disable_doc_comments: self.disable_doc_comments,
+                    non_unary: false,
+                    comment_level: 0,
+                    end_with_none: false,
+                    include_comments: false,
+                    disable_doc_comments: self.disable_doc_comments,
+                },
+                pos: Position::new(1, 0),
+                buffer,
+                stream: MultiInputsStream {
+                    buf: None,
+                    streams: input.into_iter().map(|s| s.chars().peekable()).collect(),
+                    index: 0,
+                },
+                map,
             },
-            pos: Position::new(1, 0),
+            buffer2,
-            stream: MultiInputsStream {
+        )
-                buf: None,
-                streams: input.into_iter().map(|s| s.chars().peekable()).collect(),
-                index: 0,
-            },
-            map,
-        }
     }
 }

tests/string.rs

@@ -310,3 +310,62 @@ fn test_string_split() -> Result<(), Box<EvalAltResult>> {
 
     Ok(())
 }
+
+#[test]
+fn test_string_interpolated() -> Result<(), Box<EvalAltResult>> {
+    let engine = Engine::new();
+
+    assert_eq!(
+        engine.eval::<String>(
+            r"
+                let x = 40;
+                `hello ${x+2} worlds!`
+            "
+        )?,
+        "hello 42 worlds!"
+    );
+
+    assert_eq!(
+        engine.eval::<String>(
+            r"
+                const x = 42;
+                `hello ${x} worlds!`
+            "
+        )?,
+        "hello 42 worlds!"
+    );
+
+    assert_eq!(engine.eval::<String>("`hello ${}world!`")?, "hello world!");
+
+    assert_eq!(
+        engine.eval::<String>(
+            r"
+                const x = 42;
+                `${x} worlds!`
+            "
+        )?,
+        "42 worlds!"
+    );
+
+    assert_eq!(
+        engine.eval::<String>(
+            r"
+                const x = 42;
+                `hello ${x}`
+            "
+        )?,
+        "hello 42"
+    );
+
+    assert_eq!(
+        engine.eval::<String>(
+            r"
+                const x = 20;
+                `hello ${let y = x + 1; `${y * 2}`} worlds!`
+            "
+        )?,
+        "hello 42 worlds!"
+    );
+
+    Ok(())
+}