Support $symbol$ in custom syntax.

CHANGELOG.md

@@ -7,6 +7,7 @@ Version 1.1.0
 Enhancements
 ------------
 
+* `$symbol$` is supported in custom syntax to match any symbol.
 * `parse_float()`, `PI()` and `E()` now defer to `Decimal` under `no_float` if `decimal` is turned on.
 
 

src/ast.rs

@@ -1552,8 +1552,8 @@ impl OpAssignment<'_> {
         let op_raw = op
             .map_op_assignment()
             .expect("never fails because token must be an op-assignment operator")
-            .keyword_syntax();
+            .literal_syntax();
-        let op_assignment = op.keyword_syntax();
+        let op_assignment = op.literal_syntax();
 
         Self {
             hash_op_assign: calc_fn_hash(op_assignment, 2),

src/custom_syntax.rs

@@ -14,21 +14,26 @@ use std::any::TypeId;
 #[cfg(feature = "no_std")]
 use std::prelude::v1::*;
 
-/// Special marker for matching an expression.
+/// Collection of special markers for custom syntax definition.
-pub const CUSTOM_SYNTAX_MARKER_EXPR: &str = "$expr$";
+pub mod markers {
-/// Special marker for matching a statements block.
+    /// Special marker for matching an expression.
-pub const CUSTOM_SYNTAX_MARKER_BLOCK: &str = "$block$";
+    pub const CUSTOM_SYNTAX_MARKER_EXPR: &str = "$expr$";
-/// Special marker for matching an identifier.
+    /// Special marker for matching a statements block.
-pub const CUSTOM_SYNTAX_MARKER_IDENT: &str = "$ident$";
+    pub const CUSTOM_SYNTAX_MARKER_BLOCK: &str = "$block$";
-/// Special marker for matching a string literal.
+    /// Special marker for matching an identifier.
-pub const CUSTOM_SYNTAX_MARKER_STRING: &str = "$string$";
+    pub const CUSTOM_SYNTAX_MARKER_IDENT: &str = "$ident$";
-/// Special marker for matching an integer number.
+    /// Special marker for matching a single symbol.
-pub const CUSTOM_SYNTAX_MARKER_INT: &str = "$int$";
+    pub const CUSTOM_SYNTAX_MARKER_SYMBOL: &str = "$symbol$";
-/// Special marker for matching a floating-point number.
+    /// Special marker for matching a string literal.
-#[cfg(not(feature = "no_float"))]
+    pub const CUSTOM_SYNTAX_MARKER_STRING: &str = "$string$";
-pub const CUSTOM_SYNTAX_MARKER_FLOAT: &str = "$float$";
+    /// Special marker for matching an integer number.
-/// Special marker for matching a boolean value.
+    pub const CUSTOM_SYNTAX_MARKER_INT: &str = "$int$";
-pub const CUSTOM_SYNTAX_MARKER_BOOL: &str = "$bool$";
+    /// Special marker for matching a floating-point number.
+    #[cfg(not(feature = "no_float"))]
+    pub const CUSTOM_SYNTAX_MARKER_FLOAT: &str = "$float$";
+    /// Special marker for matching a boolean value.
+    pub const CUSTOM_SYNTAX_MARKER_BOOL: &str = "$bool$";
+}
 
 /// A general expression evaluation trait object.
 #[cfg(not(feature = "sync"))]
@@ -184,6 +189,8 @@ impl Engine {
         scope_may_be_changed: bool,
         func: impl Fn(&mut EvalContext, &[Expression]) -> RhaiResult + SendSync + 'static,
     ) -> Result<&mut Self, ParseError> {
+        use markers::*;
+
         let mut segments: StaticVec<ImmutableString> = Default::default();
 
         for s in keywords {
@@ -199,6 +206,7 @@ impl Engine {
             let seg = match s {
                 // Markers not in first position
                 CUSTOM_SYNTAX_MARKER_IDENT
+                | CUSTOM_SYNTAX_MARKER_SYMBOL
                 | CUSTOM_SYNTAX_MARKER_EXPR
                 | CUSTOM_SYNTAX_MARKER_BLOCK
                 | CUSTOM_SYNTAX_MARKER_BOOL
@@ -226,7 +234,7 @@ impl Engine {
                 s if segments.is_empty()
                     && token
                         .as_ref()
-                        .map_or(false, |v| v.is_keyword() || v.is_reserved()) =>
+                        .map_or(false, |v| v.is_standard_keyword() || v.is_reserved()) =>
                 {
                     return Err(LexError::ImproperSymbol(
                         s.to_string(),

src/engine_api.rs

@@ -1357,9 +1357,9 @@ impl Engine {
 
         // Trims the JSON string and add a '#' in front
         let json_text = json.trim_start();
-        let scripts = if json_text.starts_with(Token::MapStart.keyword_syntax()) {
+        let scripts = if json_text.starts_with(Token::MapStart.literal_syntax()) {
             [json_text, ""]
-        } else if json_text.starts_with(Token::LeftBrace.keyword_syntax()) {
+        } else if json_text.starts_with(Token::LeftBrace.literal_syntax()) {
             ["#", json_text]
         } else {
             return Err(crate::ParseErrorType::MissingToken(

src/engine_settings.rs

@@ -307,13 +307,13 @@ impl Engine {
             None | Some(Token::Reserved(_)) | Some(Token::Custom(_)) => (),
             // Active standard keywords cannot be made custom
             // Disabled keywords are OK
-            Some(token) if token.is_keyword() => {
+            Some(token) if token.is_standard_keyword() => {
                 if !self.disabled_symbols.contains(token.syntax().as_ref()) {
                     return Err(format!("'{}' is a reserved keyword", keyword.as_ref()));
                 }
             }
             // Active standard symbols cannot be made custom
-            Some(token) if token.is_symbol() => {
+            Some(token) if token.is_standard_symbol() => {
                 if !self.disabled_symbols.contains(token.syntax().as_ref()) {
                     return Err(format!("'{}' is a reserved operator", keyword.as_ref()));
                 }

src/error_parsing.rs

@@ -235,6 +235,8 @@ impl fmt::Display for ParseErrorType {
             Self::MismatchedType(r, a) => write!(f, "Expecting {}, not {}", r, a),
             Self::ExprExpected(s) => write!(f, "Expecting {} expression", s),
             Self::MissingToken(token, s) => write!(f, "Expecting '{}' {}", token, s),
+
+            Self::MissingSymbol(s) if s.is_empty() => f.write_str("Expecting a symbol"),
             Self::MissingSymbol(s) => f.write_str(s),
 
             Self::AssignmentToConstant(s) => match s.as_str() {

src/parse.rs

@@ -4,10 +4,7 @@ use crate::ast::{
     BinaryExpr, CustomExpr, Expr, FnCallExpr, FnCallHashes, Ident, OpAssignment, ReturnType,
     ScriptFnDef, Stmt, StmtBlock, AST_OPTION_FLAGS::*,
 };
-use crate::custom_syntax::{
+use crate::custom_syntax::{markers::*, CustomSyntax};
-    CustomSyntax, CUSTOM_SYNTAX_MARKER_BLOCK, CUSTOM_SYNTAX_MARKER_BOOL, CUSTOM_SYNTAX_MARKER_EXPR,
-    CUSTOM_SYNTAX_MARKER_IDENT, CUSTOM_SYNTAX_MARKER_INT, CUSTOM_SYNTAX_MARKER_STRING,
-};
 use crate::dynamic::AccessMode;
 use crate::engine::{Precedence, KEYWORD_THIS, OP_CONTAINS};
 use crate::fn_hash::get_hasher;
@@ -17,8 +14,8 @@ use crate::token::{
     is_keyword_function, is_valid_identifier, Token, TokenStream, TokenizerControl,
 };
 use crate::{
-    calc_fn_hash, calc_qualified_fn_hash, calc_qualified_var_hash, Engine, Identifier, LexError,
+    calc_fn_hash, calc_qualified_fn_hash, calc_qualified_var_hash, Engine, Identifier,
-    ParseError, ParseErrorType, Position, Scope, Shared, StaticVec, AST,
+    ImmutableString, LexError, ParseError, ParseErrorType, Position, Scope, Shared, StaticVec, AST,
 };
 #[cfg(feature = "no_std")]
 use std::prelude::v1::*;
@@ -29,7 +26,7 @@ use std::{
 };
 
 #[cfg(not(feature = "no_float"))]
-use crate::{custom_syntax::CUSTOM_SYNTAX_MARKER_FLOAT, FLOAT};
+use crate::{custom_syntax::markers::CUSTOM_SYNTAX_MARKER_FLOAT, FLOAT};
 
 #[cfg(not(feature = "no_function"))]
 use crate::FnAccess;
@@ -389,6 +386,20 @@ fn parse_var_name(input: &mut TokenStream) -> Result<(String, Position), ParseEr
     }
 }
 
+/// Parse a symbol.
+fn parse_symbol(input: &mut TokenStream) -> Result<(String, Position), ParseError> {
+    match input.next().expect(NEVER_ENDS) {
+        // Symbol
+        (token, pos) if token.is_standard_symbol() => Ok((token.literal_syntax().into(), pos)),
+        // Reserved symbol
+        (Token::Reserved(s), pos) if !is_valid_identifier(s.chars()) => Ok((s, pos)),
+        // Bad identifier
+        (Token::LexError(err), pos) => Err(err.into_err(pos)),
+        // Not a symbol
+        (_, pos) => Err(PERR::MissingSymbol(Default::default()).into_err(pos)),
+    }
+}
+
 /// Parse `(` expr `)`
 fn parse_paren_expr(
     input: &mut TokenStream,
@@ -2016,6 +2027,13 @@ fn parse_custom_syntax(
                 tokens.push(state.get_identifier(CUSTOM_SYNTAX_MARKER_IDENT));
                 keywords.push(Expr::Variable(None, pos, (None, None, name).into()));
             }
+            CUSTOM_SYNTAX_MARKER_SYMBOL => {
+                let (symbol, pos) = parse_symbol(input)?;
+                let symbol: ImmutableString = state.get_identifier(symbol).into();
+                segments.push(symbol.clone());
+                tokens.push(state.get_identifier(CUSTOM_SYNTAX_MARKER_SYMBOL));
+                keywords.push(Expr::StringConstant(symbol, pos));
+            }
             CUSTOM_SYNTAX_MARKER_EXPR => {
                 keywords.push(parse_expr(input, state, lib, settings)?);
                 let keyword = state.get_identifier(CUSTOM_SYNTAX_MARKER_EXPR);
@@ -2034,9 +2052,8 @@ fn parse_custom_syntax(
             CUSTOM_SYNTAX_MARKER_BOOL => match input.next().expect(NEVER_ENDS) {
                 (b @ Token::True, pos) | (b @ Token::False, pos) => {
                     keywords.push(Expr::BoolConstant(b == Token::True, pos));
-                    let keyword = state.get_identifier(CUSTOM_SYNTAX_MARKER_BOOL);
+                    segments.push(state.get_identifier(b.literal_syntax()).into());
-                    segments.push(keyword.clone().into());
+                    tokens.push(state.get_identifier(CUSTOM_SYNTAX_MARKER_BOOL));
-                    tokens.push(keyword);
                 }
                 (_, pos) => {
                     return Err(
@@ -2048,9 +2065,8 @@ fn parse_custom_syntax(
             CUSTOM_SYNTAX_MARKER_INT => match input.next().expect(NEVER_ENDS) {
                 (Token::IntegerConstant(i), pos) => {
                     keywords.push(Expr::IntegerConstant(i, pos));
-                    let keyword = state.get_identifier(CUSTOM_SYNTAX_MARKER_INT);
+                    segments.push(i.to_string().into());
-                    segments.push(keyword.clone().into());
+                    tokens.push(state.get_identifier(CUSTOM_SYNTAX_MARKER_INT));
-                    tokens.push(keyword);
                 }
                 (_, pos) => {
                     return Err(
@@ -2063,9 +2079,8 @@ fn parse_custom_syntax(
             CUSTOM_SYNTAX_MARKER_FLOAT => match input.next().expect(NEVER_ENDS) {
                 (Token::FloatConstant(f), pos) => {
                     keywords.push(Expr::FloatConstant(f, pos));
-                    let keyword = state.get_identifier(CUSTOM_SYNTAX_MARKER_FLOAT);
+                    segments.push(f.to_string().into());
-                    segments.push(keyword.clone().into());
+                    tokens.push(state.get_identifier(CUSTOM_SYNTAX_MARKER_FLOAT));
-                    tokens.push(keyword);
                 }
                 (_, pos) => {
                     return Err(PERR::MissingSymbol(
@@ -2076,10 +2091,10 @@ fn parse_custom_syntax(
             },
             CUSTOM_SYNTAX_MARKER_STRING => match input.next().expect(NEVER_ENDS) {
                 (Token::StringConstant(s), pos) => {
-                    keywords.push(Expr::StringConstant(state.get_identifier(s).into(), pos));
+                    let s: ImmutableString = state.get_identifier(s).into();
-                    let keyword = state.get_identifier(CUSTOM_SYNTAX_MARKER_STRING);
+                    keywords.push(Expr::StringConstant(s.clone(), pos));
-                    segments.push(keyword.clone().into());
+                    segments.push(s);
-                    tokens.push(keyword);
+                    tokens.push(state.get_identifier(CUSTOM_SYNTAX_MARKER_STRING));
                 }
                 (_, pos) => {
                     return Err(PERR::MissingSymbol("Expecting a string".to_string()).into_err(pos))

src/token.rs

@@ -483,9 +483,9 @@ pub enum Token {
 }
 
 impl Token {
-    /// Get the syntax of the token if it is a keyword.
+    /// Get the literal syntax of the token.
     #[must_use]
-    pub const fn keyword_syntax(&self) -> &'static str {
+    pub const fn literal_syntax(&self) -> &'static str {
         use Token::*;
 
         match self {
@@ -595,7 +595,7 @@ impl Token {
 
             EOF => "{EOF}".into(),
 
-            token => token.keyword_syntax().into(),
+            token => token.literal_syntax().into(),
         }
     }
 
@@ -912,7 +912,7 @@ impl Token {
 
     /// Is this token a standard symbol used in the language?
     #[must_use]
-    pub const fn is_symbol(&self) -> bool {
+    pub const fn is_standard_symbol(&self) -> bool {
         use Token::*;
 
         match self {
@@ -928,10 +928,10 @@ impl Token {
         }
     }
 
-    /// Is this token an active standard keyword?
+    /// Is this token a standard keyword?
     #[inline]
     #[must_use]
-    pub const fn is_keyword(&self) -> bool {
+    pub const fn is_standard_keyword(&self) -> bool {
         use Token::*;
 
         match self {
@@ -948,7 +948,7 @@ impl Token {
         }
     }
 
-    /// Is this token a reserved symbol?
+    /// Is this token a reserved keyword or symbol?
     #[inline(always)]
     #[must_use]
     pub const fn is_reserved(&self) -> bool {

tests/custom_syntax.rs

@@ -1,4 +1,6 @@
-use rhai::{Dynamic, Engine, EvalAltResult, LexError, ParseErrorType, Position, INT};
+use rhai::{
+    Dynamic, Engine, EvalAltResult, ImmutableString, LexError, ParseErrorType, Position, INT,
+};
 
 #[test]
 fn test_custom_syntax() -> Result<(), Box<EvalAltResult>> {
@@ -19,21 +21,32 @@ fn test_custom_syntax() -> Result<(), Box<EvalAltResult>> {
 
     engine.register_custom_syntax(
         &[
-            "exec", "[", "$ident$", ";", "$int$", "]", "->", "$block$", "while", "$expr$",
+            "exec", "[", "$ident$", "$symbol$", "$int$", "]", "->", "$block$", "while", "$expr$",
         ],
         true,
         |context, inputs| {
             let var_name = inputs[0].get_variable_name().unwrap().to_string();
-            let max = inputs[1].get_literal_value::<INT>().unwrap();
+            let op = inputs[1].get_literal_value::<ImmutableString>().unwrap();
-            let stmt = &inputs[2];
+            let max = inputs[2].get_literal_value::<INT>().unwrap();
-            let condition = &inputs[3];
+            let stmt = &inputs[3];
+            let condition = &inputs[4];
 
             context.scope_mut().push(var_name.clone(), 0 as INT);
 
             let mut count: INT = 0;
 
             loop {
-                if count >= max {
+                let done = match op.as_str() {
+                    "<" => count >= max,
+                    "<=" => count > max,
+                    ">" => count <= max,
+                    ">=" => count < max,
+                    "==" => count != max,
+                    "!=" => count == max,
+                    _ => return Err(format!("Unsupported operator: {}", op).into()),
+                };
+
+                if done {
                     break;
                 }
 
@@ -64,11 +77,18 @@ fn test_custom_syntax() -> Result<(), Box<EvalAltResult>> {
         },
     )?;
 
+    assert!(matches!(
+        *engine
+            .consume("let foo = (exec [x<<15] -> { x += 2 } while x < 42) * 10;")
+            .expect_err("should error"),
+        EvalAltResult::ErrorRuntime(_, _)
+    ));
+
     assert_eq!(
         engine.eval::<INT>(
             "
                 let x = 0;
-                let foo = (exec [x;15] -> { x += 2 } while x < 42) * 10;
+                let foo = (exec [x<15] -> { x += 2 } while x < 42) * 10;
                 foo
             "
         )?,
@@ -78,7 +98,7 @@ fn test_custom_syntax() -> Result<(), Box<EvalAltResult>> {
         engine.eval::<INT>(
             "
                 let x = 0;
-                exec [x;100] -> { x += 1 } while x < 42;
+                exec [x<100] -> { x += 1 } while x < 42;
                 x
             "
         )?,
@@ -87,7 +107,7 @@ fn test_custom_syntax() -> Result<(), Box<EvalAltResult>> {
     assert_eq!(
         engine.eval::<INT>(
             "
-                exec [x;100] -> { x += 1 } while x < 42;
+                exec [x<100] -> { x += 1 } while x < 42;
                 x
             "
         )?,
@@ -97,7 +117,7 @@ fn test_custom_syntax() -> Result<(), Box<EvalAltResult>> {
         engine.eval::<INT>(
             "
                 let foo = 123;
-                exec [x;15] -> { x += 1 } while x < 42;
+                exec [x<15] -> { x += 1 } while x < 42;
                 foo + x + x1 + x2 + x3
             "
         )?,