Add state to custom syntax.

CHANGELOG.md

@@ -1,6 +1,18 @@
 Rhai Release Notes
 ==================
 
+Version 1.11.0
+==============
+
+New features
+------------
+
+### Custom syntax with state
+
+* [`Engine::register_custom_syntax_with_state_raw`] is added. The custom syntax parser and implementation functions take on an additional parameter that holds a user-defined custom _state_ which should substantially simplify writing some custom parsers.
+* [`Engine::register_custom_syntax_raw`] is deprecated.
+
+
 Version 1.10.0
 ==============
 

src/api/custom_syntax.rs

@@ -7,8 +7,8 @@ use crate::parser::ParseResult;
 use crate::tokenizer::{is_valid_identifier, Token};
 use crate::types::dynamic::Variant;
 use crate::{
-    reify, Engine, EvalContext, Identifier, ImmutableString, LexError, Position, RhaiResult,
+    reify, Dynamic, Engine, EvalContext, Identifier, ImmutableString, LexError, Position,
-    StaticVec,
+    RhaiResult, StaticVec,
 };
 use std::ops::Deref;
 #[cfg(feature = "no_std")]
@@ -39,19 +39,21 @@ pub mod markers {
 
 /// A general expression evaluation trait object.
 #[cfg(not(feature = "sync"))]
-pub type FnCustomSyntaxEval = dyn Fn(&mut EvalContext, &[Expression]) -> RhaiResult;
+pub type FnCustomSyntaxEval = dyn Fn(&mut EvalContext, &[Expression], &Dynamic) -> RhaiResult;
 /// A general expression evaluation trait object.
 #[cfg(feature = "sync")]
-pub type FnCustomSyntaxEval = dyn Fn(&mut EvalContext, &[Expression]) -> RhaiResult + Send + Sync;
+pub type FnCustomSyntaxEval =
+    dyn Fn(&mut EvalContext, &[Expression], &Dynamic) -> RhaiResult + Send + Sync;
 
 /// A general expression parsing trait object.
 #[cfg(not(feature = "sync"))]
 pub type FnCustomSyntaxParse =
-    dyn Fn(&[ImmutableString], &str) -> ParseResult<Option<ImmutableString>>;
+    dyn Fn(&[ImmutableString], &str, &mut Dynamic) -> ParseResult<Option<ImmutableString>>;
 /// A general expression parsing trait object.
 #[cfg(feature = "sync")]
-pub type FnCustomSyntaxParse =
+pub type FnCustomSyntaxParse = dyn Fn(&[ImmutableString], &str, &mut Dynamic) -> ParseResult<Option<ImmutableString>>
-    dyn Fn(&[ImmutableString], &str) -> ParseResult<Option<ImmutableString>> + Send + Sync;
+    + Send
+    + Sync;
 
 /// An expression sub-tree in an [`AST`][crate::AST].
 #[derive(Debug, Clone)]
@@ -297,10 +299,10 @@ impl Engine {
         // The first keyword is the discriminator
         let key = segments[0].clone();
 
-        self.register_custom_syntax_raw(
+        self.register_custom_syntax_with_state_raw(
             key,
             // Construct the parsing function
-            move |stream, _| {
+            move |stream, _, _| {
                 if stream.len() >= segments.len() {
                     Ok(None)
                 } else {
@@ -308,12 +310,12 @@ impl Engine {
                 }
             },
             scope_may_be_changed,
-            func,
+            move |context, expressions, _| func(context, expressions),
         );
 
         Ok(self)
     }
-    /// Register a custom syntax with the [`Engine`].
+    /// Register a custom syntax with the [`Engine`] with custom user-defined state.
     ///
     /// Not available under `no_custom_syntax`.
     ///
@@ -328,30 +330,31 @@ impl Engine {
     /// All custom keywords used as symbols must be manually registered via [`Engine::register_custom_operator`].
     /// Otherwise, they won't be recognized.
     ///
-    /// # Implementation Function Signature
+    /// # Parsing Function Signature
     ///
-    /// The implementation function has the following signature:
+    /// The parsing function has the following signature:
     ///
-    /// `Fn(symbols: &[ImmutableString], look_ahead: &str) -> Result<Option<ImmutableString>, ParseError>`
+    /// `Fn(symbols: &[ImmutableString], look_ahead: &str, state: &mut Dynamic) -> Result<Option<ImmutableString>, ParseError>`
     ///
     /// where:
     /// * `symbols`: a slice of symbols that have been parsed so far, possibly containing `$expr$` and/or `$block$`;
     ///   `$ident$` and other literal markers are replaced by the actual text
     /// * `look_ahead`: a string slice containing the next symbol that is about to be read
+    /// * `state`: a [`Dynamic`] value that contains a user-defined state
     ///
     /// ## Return value
     ///
     /// * `Ok(None)`: parsing complete and there are no more symbols to match.
     /// * `Ok(Some(symbol))`: the next symbol to match, which can also be `$expr$`, `$ident$` or `$block$`.
     /// * `Err(ParseError)`: error that is reflected back to the [`Engine`], normally `ParseError(ParseErrorType::BadInput(LexError::ImproperSymbol(message)), Position::NONE)` to indicate a syntax error, but it can be any [`ParseError`][crate::ParseError].
-    pub fn register_custom_syntax_raw(
+    pub fn register_custom_syntax_with_state_raw(
         &mut self,
         key: impl Into<Identifier>,
-        parse: impl Fn(&[ImmutableString], &str) -> ParseResult<Option<ImmutableString>>
+        parse: impl Fn(&[ImmutableString], &str, &mut Dynamic) -> ParseResult<Option<ImmutableString>>
             + SendSync
             + 'static,
         scope_may_be_changed: bool,
-        func: impl Fn(&mut EvalContext, &[Expression]) -> RhaiResult + SendSync + 'static,
+        func: impl Fn(&mut EvalContext, &[Expression], &Dynamic) -> RhaiResult + SendSync + 'static,
     ) -> &mut Self {
         self.custom_syntax.insert(
             key.into(),

src/api/deprecated.rs

@@ -258,6 +258,68 @@ impl Engine {
     ) -> &mut Self {
         self.register_indexer_set(set_fn)
     }
+    /// Register a custom syntax with the [`Engine`].
+    ///
+    /// Not available under `no_custom_syntax`.
+    ///
+    /// # Deprecated
+    ///
+    /// This method is deprecated.
+    /// Use [`register_custom_syntax_with_state_raw`][Engine::register_custom_syntax_with_state_raw] instead.
+    ///
+    /// This method will be removed in the next major version.
+    ///
+    /// # WARNING - Low Level API
+    ///
+    /// This function is very low level.
+    ///
+    /// * `scope_may_be_changed` specifies variables have been added/removed by this custom syntax.
+    /// * `parse` is the parsing function.
+    /// * `func` is the implementation function.
+    ///
+    /// All custom keywords used as symbols must be manually registered via [`Engine::register_custom_operator`].
+    /// Otherwise, they won't be recognized.
+    ///
+    /// # Parsing Function Signature
+    ///
+    /// The parsing function has the following signature:
+    ///
+    /// `Fn(symbols: &[ImmutableString], look_ahead: &str) -> Result<Option<ImmutableString>, ParseError>`
+    ///
+    /// where:
+    /// * `symbols`: a slice of symbols that have been parsed so far, possibly containing `$expr$` and/or `$block$`;
+    ///   `$ident$` and other literal markers are replaced by the actual text
+    /// * `look_ahead`: a string slice containing the next symbol that is about to be read
+    ///
+    /// ## Return value
+    ///
+    /// * `Ok(None)`: parsing complete and there are no more symbols to match.
+    /// * `Ok(Some(symbol))`: the next symbol to match, which can also be `$expr$`, `$ident$` or `$block$`.
+    /// * `Err(ParseError)`: error that is reflected back to the [`Engine`], normally `ParseError(ParseErrorType::BadInput(LexError::ImproperSymbol(message)), Position::NONE)` to indicate a syntax error, but it can be any [`ParseError`][crate::ParseError].
+    #[deprecated(
+        since = "1.11.0",
+        note = "use `register_custom_syntax_with_state_raw` instead"
+    )]
+    #[inline(always)]
+    #[cfg(not(feature = "no_custom_syntax"))]
+    pub fn register_custom_syntax_raw(
+        &mut self,
+        key: impl Into<Identifier>,
+        parse: impl Fn(&[ImmutableString], &str) -> crate::parser::ParseResult<Option<ImmutableString>>
+            + crate::func::SendSync
+            + 'static,
+        scope_may_be_changed: bool,
+        func: impl Fn(&mut crate::EvalContext, &[crate::Expression]) -> RhaiResult
+            + crate::func::SendSync
+            + 'static,
+    ) -> &mut Self {
+        self.register_custom_syntax_with_state_raw(
+            key,
+            move |keywords, look_ahead, _| parse(keywords, look_ahead),
+            scope_may_be_changed,
+            move |context, expressions, _| func(context, expressions),
+        )
+    }
 }
 
 impl Dynamic {

src/ast/expr.rs

@@ -61,6 +61,8 @@ pub struct CustomExpr {
     pub inputs: StaticVec<Expr>,
     /// List of tokens actually parsed.
     pub tokens: StaticVec<ImmutableString>,
+    /// State value.
+    pub state: Dynamic,
     /// Is the current [`Scope`][crate::Scope] possibly modified by this custom statement
     /// (e.g. introducing a new variable)?
     pub scope_may_be_changed: bool,

src/eval/expr.rs

@@ -596,7 +596,7 @@ impl Engine {
                 let mut context =
                     EvalContext::new(self, scope, global, Some(caches), lib, this_ptr, level);
 
-                let result = (custom_def.func)(&mut context, &expressions);
+                let result = (custom_def.func)(&mut context, &expressions, &custom.state);
 
                 self.check_return_value(result, expr.start_position())
             }

src/parser.rs

@@ -2477,6 +2477,7 @@ impl Engine {
             state.stack.push(marker, ());
         }
 
+        let mut user_state = Dynamic::UNIT;
         let parse_func = &*syntax.parse;
         let mut required_token: ImmutableString = key.into();
 
@@ -2488,7 +2489,7 @@ impl Engine {
             settings.pos = *fwd_pos;
             let settings = settings.level_up();
 
-            required_token = match parse_func(&segments, &*fwd_token.syntax()) {
+            required_token = match parse_func(&segments, &*fwd_token.syntax(), &mut user_state) {
                 Ok(Some(seg))
                     if seg.starts_with(CUSTOM_SYNTAX_MARKER_SYNTAX_VARIANT)
                         && seg.len() > CUSTOM_SYNTAX_MARKER_SYNTAX_VARIANT.len() =>
@@ -2624,6 +2625,7 @@ impl Engine {
             crate::ast::CustomExpr {
                 inputs,
                 tokens,
+                state: user_state,
                 scope_may_be_changed: syntax.scope_may_be_changed,
                 self_terminated,
             }

tests/custom_syntax.rs

@@ -254,14 +254,17 @@ fn test_custom_syntax() -> Result<(), Box<EvalAltResult>> {
 fn test_custom_syntax_raw() -> Result<(), Box<EvalAltResult>> {
     let mut engine = Engine::new();
 
-    engine.register_custom_syntax_raw(
+    engine.register_custom_syntax_with_state_raw(
         "hello",
-        |stream, _| match stream.len() {
+        |stream, _, state| match stream.len() {
             0 => unreachable!(),
             1 => Ok(Some("$ident$".into())),
             2 => match stream[1].as_str() {
                 "world" => Ok(Some("$$hello".into())),
-                "kitty" => Ok(None),
+                "kitty" => {
+                    *state = (42 as INT).into();
+                    Ok(None)
+                }
                 s => Err(LexError::ImproperSymbol(s.to_string(), String::new())
                     .into_err(Position::NONE)
                     .into()),
@@ -269,7 +272,7 @@ fn test_custom_syntax_raw() -> Result<(), Box<EvalAltResult>> {
             _ => unreachable!(),
         },
         true,
-        |context, inputs| {
+        |context, inputs, state| {
             context.scope_mut().push("foo", 999 as INT);
 
             Ok(match inputs[0].get_string_value().unwrap() {
@@ -277,14 +280,14 @@ fn test_custom_syntax_raw() -> Result<(), Box<EvalAltResult>> {
                     if inputs
                         .last()
                         .unwrap()
-                        .get_literal_value::<ImmutableString>()
+                        .get_string_value()
                         .map_or(false, |s| s == "$$hello") =>
                 {
                     0 as INT
                 }
                 "world" => 123 as INT,
                 "kitty" if inputs.len() > 1 => 999 as INT,
-                "kitty" => 42 as INT,
+                "kitty" => state.as_int().unwrap(),
                 _ => unreachable!(),
             }
             .into())
@@ -313,9 +316,9 @@ fn test_custom_syntax_raw() -> Result<(), Box<EvalAltResult>> {
 fn test_custom_syntax_raw2() -> Result<(), Box<EvalAltResult>> {
     let mut engine = Engine::new();
 
-    engine.register_custom_syntax_raw(
+    engine.register_custom_syntax_with_state_raw(
         "#",
-        |symbols, lookahead| match symbols.len() {
+        |symbols, lookahead, _| match symbols.len() {
             1 if lookahead == "-" => Ok(Some("$symbol$".into())),
             1 => Ok(Some("$int$".into())),
             2 if symbols[1] == "-" => Ok(Some("$int$".into())),
@@ -324,7 +327,7 @@ fn test_custom_syntax_raw2() -> Result<(), Box<EvalAltResult>> {
             _ => unreachable!(),
         },
         false,
-        move |_, inputs| {
+        move |_, inputs, _| {
             let id = if inputs.len() == 2 {
                 -inputs[1].get_literal_value::<INT>().unwrap()
             } else {