Custom syntax parser function takes &[String].

RELEASES.md

@@ -5,14 +5,22 @@ Rhai Release Notes
 Version 0.19.4
 ==============
 
+This version adds a low-level API for more flexibility when defining custom syntax.
+
 Bug fixes
 ---------
 
 * Fixes `Send + Sync` for `EvalAltResult` under the `sync` feature. Bug introduced with `0.19.3`.
 
+Breaking changes
+----------------
+
+* Custom syntax can no longer start with a keyword (even a _reserved_ one), even if it has been disabled. That is to avoid breaking scripts later when the keyword is no longer disabled.
+
 New features
 ------------
 
+* Low-level API for custom syntax allowing more flexibility in designing the syntax.
 * `Module::fill_with` to poly-fill a module with another.
 
 

doc/src/engine/custom-syntax.md

@@ -263,6 +263,7 @@ second symbol calls a particular command:
 
 ```rust
 // The following simulates a command-style syntax, all starting with 'perform'.
+perform hello world;        // A fixed sequence of symbols
 perform action 42;          // Perform a system action with a parameter
 perform update system;      // Update the system
 perform check all;          // Check all system settings
@@ -278,34 +279,52 @@ Use `Engine::register_custom_syntax_raw` to register a custom syntax _parser_
 together with the implementation function:
 
 ```rust
-// Register the custom syntax (sample): exec |x| -> { x += 1 } while x < 0;
 engine.register_custom_syntax_raw(
-    "exec",
+    "perform",
     |stream| match stream.len() {
-        1 => Ok(Some("|".into())),
+        // perform ...
-        2 => Ok(Some("$ident$".into())),
+        1 => Ok(Some("$ident$".into())),
-        3 => Ok(Some("|".into())),
+        // perform command ...
-        4 => Ok(Some("->".into())),
+        2 => match stream[1].as_str() {
-        5 => Ok(Some("$block$".into())),
+            "action" => Ok(Some("$expr$".into())),
-        6 => Ok(Some("while".into())),
+            "hello" => Ok(Some("world".into())),
-        7 => Ok(Some("$expr$".into())),
+            "update" | "check" | "add" | "remove" => Ok(Some("$ident$".into())),
-        8 => Ok(None)
+            "cleanup" => Ok(None),
+            cmd => Err(ParseError(Box::new(ParseErrorType::BadInput(
+                format!("Improper command: {}", cmd))),
+                Position::none(),
+            )),
+        },
+        // perform command arg ...
+        3 => match (stream[1].as_str(), stream[2].as_str()) {
+            ("action", _) => Ok(None),
+            ("hello", "world") => Ok(None),
+            ("update", arg) if arg == "system" => Ok(None),
+            ("update", arg) if arg == "client" => Ok(None),
+            ("check", arg) => Ok(None),
+            ("add", arg) => Ok(None),
+            ("remove", arg) => Ok(None),
+            (cmd, arg) => Err(ParseError(Box::new(ParseErrorType::BadInput(
+                format!("Invalid argument for command {}: {}", cmd, arg))),
+                Position::none(),
+            )),
+        },
         _ => unreachable!(),
-    }
+    },
-    1,  // the number of new variables declared within this custom syntax
+    0, // the number of new variables declared within this custom syntax
-    implementation_func     // implementation function as above
+    implementation_func
-)?;
+);
 ```
 
 ### Function Signature
 
 The custom syntax parser has the following signature:
 
-> `Fn(stream: &[&String]) -> Result<Option<ImmutableString>, ParseError>`
+> `Fn(stream: &[String]) -> Result<Option<ImmutableString>, ParseError>`
 
 where:
 
-* `stream: &[&String]` - a slice of symbols that have been parsed so far, perhaps containing the following:
+* `stream: &[String]` - a slice of symbols that have been parsed so far, perhaps containing the following:
   * `$expr$` - an expression
   * `$block$` - a statement block
 
@@ -318,4 +337,4 @@ The return value is `Result<Option<ImmutableString>, ParseError>` where:
 * `Ok(Some(symbol))` - next symbol to match.
 
 * `Err(ParseError)` - error is reflected back to the [`Engine`].
-  Normally this is `ParseErrorType::ImproperSymbol` to indicate that there is a syntax error, but it can be any error.
+  Normally this is `ParseErrorType::BadInput` to indicate that there is a syntax error, but it can be any error.

src/parser.rs

@@ -2681,7 +2681,7 @@ fn parse_custom_syntax(
     state: &mut ParseState,
     lib: &mut FunctionsLib,
     mut settings: ParseSettings,
-    key: String,
+    key: &str,
     syntax: &CustomSyntax,
     pos: Position,
 ) -> Result<Expr, ParseError> {
@@ -2703,25 +2703,24 @@ fn parse_custom_syntax(
     }
 
     let mut segments: StaticVec<_> = Default::default();
-    segments.push(key);
+    segments.push(key.to_string());
 
     loop {
         settings.pos = input.peek().unwrap().1;
+        let settings = settings.level_up();
 
-        let token = if let Some(seg) = (syntax.parse)(&segments.iter().collect::<StaticVec<_>>())
+        let parse_func = &syntax.parse;
-            .map_err(|err| err.0.into_err(settings.pos))?
+        let token =
-        {
+            if let Some(seg) = parse_func(&segments).map_err(|err| err.0.into_err(settings.pos))? {
                 seg
             } else {
                 break;
             };
 
-        let settings = settings.level_up();
-
         match token.as_str() {
             MARKER_IDENT => match input.next().unwrap() {
                 (Token::Identifier(s), pos) => {
-                    segments.push(s.to_string());
+                    segments.push(s.clone());
                     exprs.push(Expr::Variable(Box::new(((s, pos), None, 0, None))));
                 }
                 (Token::Reserved(s), pos) if is_valid_identifier(s.chars()) => {
@@ -2731,18 +2730,18 @@ fn parse_custom_syntax(
             },
             MARKER_EXPR => {
                 exprs.push(parse_expr(input, state, lib, settings)?);
-                segments.push(MARKER_EXPR.to_string());
+                segments.push(MARKER_EXPR.into());
             }
             MARKER_BLOCK => {
                 let stmt = parse_block(input, state, lib, settings)?;
                 let pos = stmt.position();
                 exprs.push(Expr::Stmt(Box::new((stmt, pos))));
-                segments.push(MARKER_BLOCK.to_string());
+                segments.push(MARKER_BLOCK.into());
             }
             s => match input.peek().unwrap() {
                 (Token::LexError(err), pos) => return Err(err.into_err(*pos)),
                 (t, _) if t.syntax().as_ref() == s => {
-                    segments.push(input.next().unwrap().0.syntax().into_owned());
+                    segments.push(t.syntax().into_owned());
                 }
                 (_, pos) => {
                     return Err(PERR::MissingToken(
@@ -2782,7 +2781,6 @@ fn parse_expr(
             Token::Custom(key) | Token::Reserved(key) | Token::Identifier(key) => {
                 match state.engine.custom_syntax.get_key_value(key) {
                     Some((key, syntax)) => {
-                        let key = key.to_string();
                         input.next().unwrap();
                         return parse_custom_syntax(
                             input, state, lib, settings, key, syntax, token_pos,

src/syntax.rs

@@ -27,11 +27,11 @@ pub type FnCustomSyntaxEval =
 
 /// A general expression parsing trait object.
 #[cfg(not(feature = "sync"))]
-pub type FnCustomSyntaxParse = dyn Fn(&[&String]) -> Result<Option<ImmutableString>, ParseError>;
+pub type FnCustomSyntaxParse = dyn Fn(&[String]) -> Result<Option<ImmutableString>, ParseError>;
 /// A general expression parsing trait object.
 #[cfg(feature = "sync")]
 pub type FnCustomSyntaxParse =
-    dyn Fn(&[&String]) -> Result<Option<ImmutableString>, ParseError> + Send + Sync;
+    dyn Fn(&[String]) -> Result<Option<ImmutableString>, ParseError> + Send + Sync;
 
 /// An expression sub-tree in an AST.
 #[derive(Debug, Clone, Hash)]
@@ -204,7 +204,7 @@ impl Engine {
     pub fn register_custom_syntax_raw(
         &mut self,
         key: impl Into<ImmutableString>,
-        parse: impl Fn(&[&String]) -> Result<Option<ImmutableString>, ParseError> + SendSync + 'static,
+        parse: impl Fn(&[String]) -> Result<Option<ImmutableString>, ParseError> + SendSync + 'static,
         new_vars: isize,
         func: impl Fn(&mut EvalContext, &[Expression]) -> Result<Dynamic, Box<EvalAltResult>>
             + SendSync