From f8c14ba1c4794d456c50eb6854c963f6b007c5cd Mon Sep 17 00:00:00 2001
From: Stephen Chung <schungx@live.com>
Date: Tue, 15 Dec 2020 19:23:30 +0800
Subject: [PATCH] Add look-ahead to custom syntax parser.

---
 RELEASES.md     |  2 ++
 src/parser.rs   | 16 +++++++++-------
 src/syntax.rs   |  8 ++++----
 tests/syntax.rs |  2 +-
 4 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/RELEASES.md b/RELEASES.md
index 32b16d16..d442c231 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -23,6 +23,7 @@ Breaking changes
 * `Engine::on_progress` now takes `u64` instead of `&u64`.
 * The closure for `Engine::on_debug` now takes an additional `Position` parameter.
 * `AST::iter_functions` now returns `ScriptFnMetadata`.
+* The parser function passed to `Engine::register_custom_syntax_raw` now takes an additional parameter containing the _look-ahead_ symbol.
 
 New features
 ------------
@@ -34,6 +35,7 @@ Enhancements
 
 * Capturing a constant variable in a closure is now supported, with no cloning.
 * Provides position info for `debug` statements.
+* A _look-ahead_ symbol is provided to custom syntax parsers, which can be used to parse variable-length symbol streams.
 
 
 Version 0.19.7
diff --git a/src/parser.rs b/src/parser.rs
index e3b275da..afd0dd5f 100644
--- a/src/parser.rs
+++ b/src/parser.rs
@@ -1831,15 +1831,17 @@ fn parse_custom_syntax(
     tokens.push(key.into());
 
     loop {
-        settings.pos = input.peek().unwrap().1;
+        let (fwd_token, fwd_pos) = input.peek().unwrap();
+        settings.pos = *fwd_pos;
         let settings = settings.level_up();
 
-        let required_token =
-            if let Some(seg) = parse_func(&segments).map_err(|err| err.0.into_err(settings.pos))? {
-                seg
-            } else {
-                break;
-            };
+        let required_token = if let Some(seg) = parse_func(&segments, fwd_token.syntax().as_ref())
+            .map_err(|err| err.0.into_err(settings.pos))?
+        {
+            seg
+        } else {
+            break;
+        };
 
         match required_token.as_str() {
             MARKER_IDENT => match input.next().unwrap() {
diff --git a/src/syntax.rs b/src/syntax.rs
index a8befa39..4ceec8f9 100644
--- a/src/syntax.rs
+++ b/src/syntax.rs
@@ -26,11 +26,11 @@ pub type FnCustomSyntaxEval =
 /// A general expression parsing trait object.
 #[cfg(not(feature = "sync"))]
 pub type FnCustomSyntaxParse =
-    dyn Fn(&[ImmutableString]) -> Result<Option<ImmutableString>, ParseError>;
+    dyn Fn(&[ImmutableString], &str) -> Result<Option<ImmutableString>, ParseError>;
 /// A general expression parsing trait object.
 #[cfg(feature = "sync")]
 pub type FnCustomSyntaxParse =
-    dyn Fn(&[ImmutableString]) -> Result<Option<ImmutableString>, ParseError> + Send + Sync;
+    dyn Fn(&[ImmutableString], &str) -> Result<Option<ImmutableString>, ParseError> + Send + Sync;
 
 /// An expression sub-tree in an [`AST`][crate::AST].
 #[derive(Debug, Clone)]
@@ -185,7 +185,7 @@ impl Engine {
         self.register_custom_syntax_raw(
             key,
             // Construct the parsing function
-            move |stream| {
+            move |stream, _| {
                 if stream.len() >= segments.len() {
                     Ok(None)
                 } else {
@@ -213,7 +213,7 @@ impl Engine {
     pub fn register_custom_syntax_raw(
         &mut self,
         key: impl Into<ImmutableString>,
-        parse: impl Fn(&[ImmutableString]) -> Result<Option<ImmutableString>, ParseError>
+        parse: impl Fn(&[ImmutableString], &str) -> Result<Option<ImmutableString>, ParseError>
             + SendSync
             + 'static,
         new_vars: isize,
diff --git a/tests/syntax.rs b/tests/syntax.rs
index 2f35eca3..f2049e86 100644
--- a/tests/syntax.rs
+++ b/tests/syntax.rs
@@ -93,7 +93,7 @@ fn test_custom_syntax_raw() -> Result<(), Box<EvalAltResult>> {
 
     engine.register_custom_syntax_raw(
         "hello",
-        |stream| match stream.len() {
+        |stream, _| match stream.len() {
             0 => unreachable!(),
             1 => Ok(Some("$ident$".into())),
             2 => match stream[1].as_str() {