Add support for line continuation and multi-line string literals.

CHANGELOG.md

@@ -16,7 +16,14 @@ an object map is small.
 `HashMap` and `BTreeMap` have almost identical public API's so this change is unlikely to break
 existing code.
 
-Im addition, all function signature/metadata methods are now grouped under the umbrella `metadata` feature.
+[`SmartString`](https://crates.io/crates/smartstring) is used to store identifiers (which tends to
+be short, fewer than 23 characters, and ASCII-based) because they can usually be stored inline.
+`Map` keys now also use [`SmartString`](https://crates.io/crates/smartstring).
+
+In addition, there is now support for line continuation in strings (put `\` at the end of line) as
+well as multi-line literal strings (wrapped by back-ticks: <code>\`...\`</code>).
+
+Finally, all function signature/metadata methods are now grouped under the umbrella `metadata` feature.
 This avoids spending precious resources maintaining metadata for functions for the vast majority of
 use cases where such information is not required.
 
@@ -24,7 +31,6 @@ use cases where such information is not required.
 Breaking changes
 ----------------
 
-* `Map` is now an alias to `BTreeMap` instead of `HashMap` because most object maps hold few properties.
 * The traits `RegisterFn` and `RegisterResultFn` are removed.  `Engine::register_fn` and `Engine::register_result_fn` are now implemented directly on `Engine`.
 * `FnPtr::call_dynamic` now takes `&NativeCallContext` instead of consuming it.
 * All `Module::set_fn_XXX` methods are removed, in favor of `Module::set_native_fn`.
@@ -35,6 +41,13 @@ Breaking changes
 * The shebang `#!` is now a reserved symbol.
 * Shebangs at the very beginning of script files are skipped when loading them.
 * [`smartstring`](https://crates.io/crates/smartstring) is used for identifiers by default. Currently, a PR branch is pulled because it breaks on `no-std` builds. The official crate will be used once `smartstring` is fixed to support `no-std`.
+* `Map` is now an alias to `BTreeMap<SmartString, Dynamic>` instead of `HashMap` because most object maps hold few properties.
+
+New features
+------------
+
+* Line continuation (via `\`) and multi-line literal strings (wrapped with <code>\`</code>) support are added.
+* Rhai scripts can now start with a shebang `#!` which is ignored.
 
 Enhancements
 ------------
@@ -42,7 +55,6 @@ Enhancements
 * Replaced all `HashMap` usage with `BTreeMap` for better performance because collections in Rhai are tiny.
 * `Engine::register_result_fn` no longer requires the successful return type to be `Dynamic`.  It can now be any clonable type.
 * `#[rhai_fn(return_raw)]` can now return `Result<T, Box<EvalAltResult>>` where `T` is any clonable type instead of `Result<Dynamic, Box<EvalAltResult>>`.
-* Rhai scripts can now start with a shebang `#!`.
 
 
 Version 0.19.14

src/token.rs

@@ -842,7 +842,7 @@ pub trait InputStream {
     fn peek_next(&mut self) -> Option<char>;
 }
 
-/// _(INTERNALS)_ Parse a string literal wrapped by `enclosing_char`.
+/// _(INTERNALS)_ Parse a string literal ended by `termination_char`.
 /// Exported under the `internals` feature only.
 ///
 /// # Volatile API
@@ -852,12 +852,15 @@ pub fn parse_string_literal(
     stream: &mut impl InputStream,
     state: &mut TokenizeState,
     pos: &mut Position,
-    enclosing_char: char,
+    termination_char: char,
+    continuation: bool,
+    verbatim: bool,
 ) -> Result<String, (LexError, Position)> {
     let mut result: smallvec::SmallVec<[char; 16]> = Default::default();
     let mut escape: smallvec::SmallVec<[char; 12]> = Default::default();
 
     let start = *pos;
+    let mut skip_whitespace_until = 0;
 
     loop {
         let next_char = stream.get_next().ok_or((LERR::UnterminatedString, start))?;
@@ -871,8 +874,10 @@ pub fn parse_string_literal(
         }
 
         match next_char {
+            // \r - ignore if followed by \n
+            '\r' if stream.peek_next().unwrap_or('\0') == '\n' => {}
             // \...
-            '\\' if escape.is_empty() => {
+            '\\' if escape.is_empty() && !verbatim => {
                 escape.push('\\');
             }
             // \\
@@ -937,18 +942,37 @@ pub fn parse_string_literal(
                 })?);
             }
 
-            // \{enclosing_char} - escaped
+            // \{termination_char} - escaped
-            ch if enclosing_char == ch && !escape.is_empty() => {
+            _ if termination_char == next_char && !escape.is_empty() => {
                 escape.clear();
-                result.push(ch)
+                result.push(next_char)
             }
 
             // Close wrapper
-            ch if enclosing_char == ch && escape.is_empty() => break,
+            _ if termination_char == next_char && escape.is_empty() => break,
+
+            // Line continuation
+            '\n' if continuation && !escape.is_empty() => {
+                escape.clear();
+                pos.new_line();
+                skip_whitespace_until = start.position().unwrap() + 1;
+            }
+
+            // New-line cannot be escaped
+            // Cannot have new-lines inside non-multi-line string literals
+            '\n' if !escape.is_empty() || !verbatim => {
+                pos.rewind();
+                return Err((LERR::UnterminatedString, start));
+            }
+
+            '\n' => {
+                pos.new_line();
+                result.push(next_char);
+            }
 
             // Unknown escape sequence
-            ch if !escape.is_empty() => {
+            _ if !escape.is_empty() => {
-                escape.push(ch);
+                escape.push(next_char);
 
                 return Err((
                     LERR::MalformedEscapeSequence(escape.into_iter().collect()),
@@ -956,16 +980,14 @@ pub fn parse_string_literal(
                 ));
             }
 
-            // Cannot have new-lines inside string literals
+            // Whitespace to skip
-            '\n' => {
+            _ if next_char.is_whitespace() && pos.position().unwrap() < skip_whitespace_until => {}
-                pos.rewind();
-                return Err((LERR::UnterminatedString, start));
-            }
 
             // All other characters
-            ch => {
+            _ => {
                 escape.clear();
-                result.push(ch);
+                result.push(next_char);
+                skip_whitespace_until = 0;
             }
         }
     }
@@ -1272,12 +1294,15 @@ fn get_next_token_inner(
                 return get_identifier(stream, pos, start_pos, c);
             }
 
-            // " - string literal
+            // " or ` - string literal
-            ('"', _) => {
+            ('"', _) | ('`', _) => {
-                return parse_string_literal(stream, state, pos, '"').map_or_else(
+                let multi_line = c == '`';
-                    |err| Some((Token::LexError(err.0), err.1)),
+
-                    |out| Some((Token::StringConstant(out), start_pos)),
+                return parse_string_literal(stream, state, pos, c, !multi_line, multi_line)
-                )
+                    .map_or_else(
+                        |err| Some((Token::LexError(err.0), err.1)),
+                        |out| Some((Token::StringConstant(out), start_pos)),
+                    );
             }
 
             // ' - character literal
@@ -1288,19 +1313,21 @@ fn get_next_token_inner(
                 ))
             }
             ('\'', _) => {
-                return Some(parse_string_literal(stream, state, pos, '\'').map_or_else(
+                return Some(
-                    |err| (Token::LexError(err.0), err.1),
+                    parse_string_literal(stream, state, pos, c, false, false).map_or_else(
-                    |result| {
+                        |err| (Token::LexError(err.0), err.1),
-                        let mut chars = result.chars();
+                        |result| {
-                        let first = chars.next().unwrap();
+                            let mut chars = result.chars();
+                            let first = chars.next().unwrap();
 
-                        if chars.next().is_some() {
+                            if chars.next().is_some() {
-                            (Token::LexError(LERR::MalformedChar(result)), start_pos)
+                                (Token::LexError(LERR::MalformedChar(result)), start_pos)
-                        } else {
+                            } else {
-                            (Token::CharConstant(first), start_pos)
+                                (Token::CharConstant(first), start_pos)
-                        }
+                            }
-                    },
+                        },
-                ))
+                    ),
+                )
             }
 
             // Braces

tests/string.rs

@@ -8,6 +8,14 @@ fn test_string() -> Result<(), Box<EvalAltResult>> {
         engine.eval::<String>(r#""Test string: \u2764""#)?,
         "Test string: ❤"
     );
+    assert_eq!(
+        engine.eval::<String>("   \"Test string: \\u2764\\\n     hello, world!\"")?,
+        "Test string: ❤ hello, world!"
+    );
+    assert_eq!(
+        engine.eval::<String>("     `Test string: \\u2764\nhello,\\nworld!`")?,
+        "Test string: \\u2764\nhello,\\nworld!"
+    );
     assert_eq!(
         engine.eval::<String>(r#""Test string: \x58""#)?,
         "Test string: X"