Enhance custom operators.

RELEASES.md

@@ -20,6 +20,8 @@ Enhancements
 
 * `Scope` is now `Clone + Hash`.
 * `Engine::register_static_module` now supports sub-module paths (e.g. `foo::bar::baz`).
+* `Engine::register_custom_operator` now accepts reserved symbols.
+* `Engine::register_custom_operator` now returns an error if given a precedence of zero.
 
 
 Version 0.19.8

src/engine_settings.rs

@@ -1,7 +1,7 @@
 //! Configuration settings for [`Engine`].
 
 use crate::stdlib::{format, num::NonZeroU8, string::String};
-use crate::token::{is_valid_identifier, Token};
+use crate::token::Token;
 use crate::Engine;
 
 #[cfg(not(feature = "no_module"))]
@@ -252,17 +252,29 @@ impl Engine {
         if precedence.is_none() {
             return Err("precedence cannot be zero".into());
         }
-        if !is_valid_identifier(keyword.chars()) {
-            return Err(format!("not a valid identifier: '{}'", keyword).into());
-        }
 
         match Token::lookup_from_syntax(keyword) {
             // Standard identifiers, reserved keywords and custom keywords are OK
             None | Some(Token::Reserved(_)) | Some(Token::Custom(_)) => (),
-            // Disabled keywords are also OK
-            Some(token) if !self.disabled_symbols.contains(token.syntax().as_ref()) => (),
             // Active standard keywords cannot be made custom
-            Some(_) => return Err(format!("'{}' is a reserved keyword", keyword).into()),
+            // Disabled keywords are OK
+            Some(token) if token.is_keyword() => {
+                if !self.disabled_symbols.contains(token.syntax().as_ref()) {
+                    return Err(format!("'{}' is a reserved keyword", keyword).into());
+                }
+            }
+            // Active standard operators cannot be made custom
+            Some(token) if token.is_operator() => {
+                if !self.disabled_symbols.contains(token.syntax().as_ref()) {
+                    return Err(format!("'{}' is a reserved operator", keyword).into());
+                }
+            }
+            // Active standard symbols cannot be made custom
+            Some(token) if !self.disabled_symbols.contains(token.syntax().as_ref()) => {
+                return Err(format!("'{}' is a reserved symbol", keyword).into())
+            }
+            // Disabled symbols are OK
+            Some(_) => (),
         }
 
         // Add to custom keywords

src/parser.rs

@@ -1665,15 +1665,24 @@ fn parse_binary_op(
 
     loop {
         let (current_op, current_pos) = input.peek().unwrap();
-        let precedence = if let Token::Custom(c) = current_op {
+        let precedence = match current_op {
-            // Custom operators
+            Token::Custom(c) => {
-            if let Some(Some(p)) = state.engine.custom_keywords.get(c) {
+                if state
-                *p
+                    .engine
+                    .custom_keywords
+                    .get(c)
+                    .map(Option::is_some)
+                    .unwrap_or(false)
+                {
+                    state.engine.custom_keywords.get(c).unwrap().unwrap().get()
                 } else {
                     return Err(PERR::Reserved(c.clone()).into_err(*current_pos));
                 }
-        } else {
+            }
-            current_op.precedence()
+            Token::Reserved(c) if !is_valid_identifier(c.chars()) => {
+                return Err(PERR::UnknownOperator(c.into()).into_err(*current_pos))
+            }
+            _ => current_op.precedence(),
         };
         let bind_right = current_op.is_bind_right();
 
@@ -1698,15 +1707,24 @@ fn parse_binary_op(
         let rhs = parse_unary(input, state, lib, settings)?;
 
         let (next_op, next_pos) = input.peek().unwrap();
-        let next_precedence = if let Token::Custom(c) = next_op {
+        let next_precedence = match next_op {
-            // Custom operators
+            Token::Custom(c) => {
-            if let Some(Some(p)) = state.engine.custom_keywords.get(c) {
+                if state
-                *p
+                    .engine
+                    .custom_keywords
+                    .get(c)
+                    .map(Option::is_some)
+                    .unwrap_or(false)
+                {
+                    state.engine.custom_keywords.get(c).unwrap().unwrap().get()
                 } else {
                     return Err(PERR::Reserved(c.clone()).into_err(*next_pos));
                 }
-        } else {
+            }
-            next_op.precedence()
+            Token::Reserved(c) if !is_valid_identifier(c.chars()) => {
+                return Err(PERR::UnknownOperator(c.into()).into_err(*next_pos))
+            }
+            _ => next_op.precedence(),
         };
 
         // Bind to right if the next operator has higher precedence
@@ -1809,11 +1827,24 @@ fn parse_binary_op(
                 make_dot_expr(state, current_lhs, rhs, pos)?
             }
 
-            Token::Custom(s) if state.engine.custom_keywords.contains_key(&s) => {
+            Token::Custom(s)
+                if state
+                    .engine
+                    .custom_keywords
+                    .get(&s)
+                    .map(Option::is_some)
+                    .unwrap_or(false) =>
+            {
+                let hash_script = if is_valid_identifier(s.chars()) {
                     // Accept non-native functions for custom operators
+                    calc_script_fn_hash(empty(), &s, 2)
+                } else {
+                    None
+                };
+
                 Expr::FnCall(
                     Box::new(FnCallExpr {
-                        hash_script: calc_script_fn_hash(empty(), &s, 2),
+                        hash_script,
                         args,
                         ..op_base
                     }),

src/syntax.rs

@@ -121,20 +121,26 @@ impl Engine {
                 continue;
             }
 
+            let token = Token::lookup_from_syntax(s);
+
             let seg = match s {
                 // Markers not in first position
                 MARKER_IDENT | MARKER_EXPR | MARKER_BLOCK if !segments.is_empty() => s.into(),
                 // Standard or reserved keyword/symbol not in first position
-                s if !segments.is_empty() && Token::lookup_from_syntax(s).is_some() => {
+                s if !segments.is_empty() && token.is_some() => {
-                    // Make it a custom keyword/symbol
+                    // Make it a custom keyword/symbol if it is disabled or reserved
-                    if !self.custom_keywords.contains_key(s) {
+                    if (self.disabled_symbols.contains(s)
+                        || matches!(token, Some(Token::Reserved(_))))
+                        && !self.custom_keywords.contains_key(s)
+                    {
                         self.custom_keywords.insert(s.into(), None);
                     }
                     s.into()
                 }
                 // Standard keyword in first position
                 s if segments.is_empty()
-                    && Token::lookup_from_syntax(s)
+                    && token
+                        .as_ref()
                         .map(|v| v.is_keyword() || v.is_reserved())
                         .unwrap_or(false) =>
                 {
@@ -151,7 +157,11 @@ impl Engine {
                 }
                 // Identifier in first position
                 s if segments.is_empty() && is_valid_identifier(s.chars()) => {
-                    if !self.custom_keywords.contains_key(s) {
+                    // Make it a custom keyword/symbol if it is disabled or reserved
+                    if (self.disabled_symbols.contains(s)
+                        || matches!(token, Some(Token::Reserved(_))))
+                        && !self.custom_keywords.contains_key(s)
+                    {
                         self.custom_keywords.insert(s.into(), None);
                     }
                     s.into()

src/token.rs

@@ -532,10 +532,10 @@ impl Token {
             #[cfg(feature = "no_module")]
             "import" | "export" | "as" => Reserved(syntax.into()),
 
-            "===" | "!==" | "->" | "<-" | ":=" | "::<" | "(*" | "*)" | "#" | "public" | "new"
+            "===" | "!==" | "->" | "<-" | ":=" | "**" | "::<" | "(*" | "*)" | "#" | "public"
-            | "use" | "module" | "package" | "var" | "static" | "begin" | "end" | "shared"
+            | "new" | "use" | "module" | "package" | "var" | "static" | "begin" | "end"
-            | "with" | "each" | "then" | "goto" | "unless" | "exit" | "match" | "case"
+            | "shared" | "with" | "each" | "then" | "goto" | "unless" | "exit" | "match"
-            | "default" | "void" | "null" | "nil" | "spawn" | "thread" | "go" | "sync"
+            | "case" | "default" | "void" | "null" | "nil" | "spawn" | "thread" | "go" | "sync"
             | "async" | "await" | "yield" => Reserved(syntax.into()),
 
             KEYWORD_PRINT | KEYWORD_DEBUG | KEYWORD_TYPE_OF | KEYWORD_EVAL | KEYWORD_FN_PTR
@@ -1742,27 +1742,21 @@ impl<'a> Iterator for TokenIterator<'a, '_> {
             Some((Token::Identifier(s), pos)) if self.engine.custom_keywords.contains_key(&s) => {
                 Some((Token::Custom(s), pos))
             }
-            // Custom standard keyword - must be disabled
+            // Custom standard keyword/symbol - must be disabled
-            Some((token, pos)) if token.is_keyword() && self.engine.custom_keywords.contains_key(token.syntax().as_ref()) => {
+            Some((token, pos)) if self.engine.custom_keywords.contains_key(token.syntax().as_ref()) => {
                 if self.engine.disabled_symbols.contains(token.syntax().as_ref()) {
-                    // Disabled standard keyword
+                    // Disabled standard keyword/symbol
                     Some((Token::Custom(token.syntax().into()), pos))
                 } else {
                     // Active standard keyword - should never be a custom keyword!
-                    unreachable!()
+                    unreachable!("{:?}", token)
                 }
             }
-            // Disabled operator
+            // Disabled symbol
-            Some((token, pos)) if token.is_operator() && self.engine.disabled_symbols.contains(token.syntax().as_ref()) => {
+            Some((token, pos)) if self.engine.disabled_symbols.contains(token.syntax().as_ref()) => {
-                Some((
-                    Token::LexError(LexError::UnexpectedInput(token.syntax().into())),
-                    pos,
-                ))
-            }
-            // Disabled standard keyword
-            Some((token, pos)) if token.is_keyword() && self.engine.disabled_symbols.contains(token.syntax().as_ref()) => {
                 Some((Token::Reserved(token.syntax().into()), pos))
             }
+            // Normal symbol
             r => r,
         };
 

tests/tokens.rs

@@ -21,12 +21,17 @@ fn test_tokens_disabled() {
             .compile("let x = 40 + 2; x += 1;")
             .expect_err("should error")
             .0,
-        ParseErrorType::BadInput(LexError::UnexpectedInput("+=".to_string()))
+        ParseErrorType::UnknownOperator("+=".to_string())
     );
+
+    assert!(matches!(
+        *engine.compile("let x = += 0;").expect_err("should error").0,
+        ParseErrorType::BadInput(LexError::UnexpectedInput(err)) if err == "+="
+    ));
 }
 
 #[test]
-fn test_tokens_custom_operator() -> Result<(), Box<EvalAltResult>> {
+fn test_tokens_custom_operator_identifiers() -> Result<(), Box<EvalAltResult>> {
     let mut engine = Engine::new();
 
     // Register a custom operator called `foo` and give it
@@ -55,6 +60,29 @@ fn test_tokens_custom_operator() -> Result<(), Box<EvalAltResult>> {
     Ok(())
 }
 
+#[test]
+fn test_tokens_custom_operator_symbol() -> Result<(), Box<EvalAltResult>> {
+    let mut engine = Engine::new();
+
+    // Register a custom operator `#` and give it
+    // a precedence of 160 (i.e. between +|- and *|/).
+    engine.register_custom_operator("#", 160).unwrap();
+
+    // Register a binary function named `#`
+    engine.register_fn("#", |x: INT, y: INT| (x * y) - (x + y));
+
+    assert_eq!(engine.eval_expression::<INT>("1 + 2 * 3 # 4 - 5 / 6")?, 15);
+
+    // Register a custom operator named `=>`
+    assert!(engine.register_custom_operator("=>", 160).is_err());
+    engine.disable_symbol("=>");
+    engine.register_custom_operator("=>", 160).unwrap();
+    engine.register_fn("=>", |x: INT, y: INT| (x * y) - (x + y));
+    assert_eq!(engine.eval_expression::<INT>("1 + 2 * 3 => 4 - 5 / 6")?, 15);
+
+    Ok(())
+}
+
 #[test]
 fn test_tokens_unicode_xid_ident() -> Result<(), Box<EvalAltResult>> {
     let engine = Engine::new();