diff --git a/README.md b/README.md index a88c574d..e448ba69 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Features * Rugged - protected against malicious attacks (such as [stack-overflow](https://schungx.github.io/rhai/safety/max-call-stack.html), [over-sized data](https://schungx.github.io/rhai/safety/max-string-size.html), and [runaway scripts](https://schungx.github.io/rhai/safety/max-operations.html) etc.) that may come from untrusted third-party user-land scripts. * Track script evaluation [progress](https://schungx.github.io/rhai/safety/progress.html) and manually terminate a script run. * [Function overloading](https://schungx.github.io/rhai/language/overload.html). -* [Operator overloading](https://schungx.github.io/rhai/rust/operators.html). +* [Operator overloading](https://schungx.github.io/rhai/rust/operators.html) and [custom operators](https://schungx.github.io/rhai/engine/custom-op.html). * Dynamic dispatch via [function pointers](https://schungx.github.io/rhai/language/fn-ptr.html). * Some support for [object-oriented programming (OOP)](https://schungx.github.io/rhai/language/oop.html). * Organize code base with dynamically-loadable [modules](https://schungx.github.io/rhai/language/modules.html). diff --git a/RELEASES.md b/RELEASES.md index 4306c015..7b9d8230 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -6,8 +6,9 @@ Version 0.17.0 This version adds: -* [`serde`](https://crates.io/crates/serde) support for working with `Dynamic` values (particularly _object maps_) -* Ability to surgically disable keywords and/or operators in the language +* [`serde`](https://crates.io/crates/serde) support for working with `Dynamic` values (particularly _object maps_). +* Ability to surgically disable keywords and/or operators in the language. +* Ability to define custom operators (which must be valid identifiers). Breaking changes ---------------- @@ -20,6 +21,7 @@ New features * New `serde` feature to allow serializating/deserializating to/from `Dynamic` values using [`serde`](https://crates.io/crates/serde). This is particularly useful when converting a Rust `struct` to a `Dynamic` _object map_ and back. * `Engine::disable_symbol` to surgically disable keywords and/or operators. +* `Engine::register_custom_operator` to define a custom operator. Version 0.16.1 diff --git a/doc/src/SUMMARY.md b/doc/src/SUMMARY.md index 39c9fb68..6c483cde 100644 --- a/doc/src/SUMMARY.md +++ b/doc/src/SUMMARY.md @@ -95,18 +95,19 @@ The Rhai Scripting Language 8. [Maximum Call Stack Depth](safety/max-call-stack.md) 9. [Maximum Statement Depth](safety/max-stmt-depth.md) 8. [Advanced Topics](advanced.md) - 1. [Disable Keywords and/or Operators](engine/disable.md) - 2. [Object-Oriented Programming (OOP)](language/oop.md) - 3. [Serialization/Deserialization of `Dynamic` with `serde`](rust/serde.md) - 4. [Script Optimization](engine/optimize/index.md) + 1. [Object-Oriented Programming (OOP)](language/oop.md) + 2. [Serialization/Deserialization of `Dynamic` with `serde`](rust/serde.md) + 3. [Script Optimization](engine/optimize/index.md) 1. [Optimization Levels](engine/optimize/optimize-levels.md) 2. [Re-Optimize an AST](engine/optimize/reoptimize.md) 3. [Eager Function Evaluation](engine/optimize/eager.md) 4. [Side-Effect Considerations](engine/optimize/side-effects.md) 5. [Volatility Considerations](engine/optimize/volatility.md) 6. [Subtle Semantic Changes](engine/optimize/semantics.md) - 5. [Eval Statement](language/eval.md) -9. [Appendix](appendix/index.md) - 6. [Keywords](appendix/keywords.md) - 7. [Operators](appendix/operators.md) - 8. [Literals](appendix/literals.md) + 4. [Disable Keywords and/or Operators](engine/disable.md) + 5. [Custom Operators](engine/custom-op.md) + 6. [Eval Statement](language/eval.md) +9. [Appendix](appendix/index.md) + 1. [Keywords](appendix/keywords.md) + 2. [Operators](appendix/operators.md) + 3. [Literals](appendix/literals.md) diff --git a/doc/src/about/features.md b/doc/src/about/features.md index 28f20aef..5a5739cb 100644 --- a/doc/src/about/features.md +++ b/doc/src/about/features.md @@ -66,3 +66,5 @@ Flexible * Supports [most build targets](targets.md) including `no-std` and [WASM]. * Surgically [disable keywords and operators] to restrict the language. + +* [Custom operators]. diff --git a/doc/src/appendix/operators.md b/doc/src/appendix/operators.md index 56782b95..bc0459b8 100644 --- a/doc/src/appendix/operators.md +++ b/doc/src/appendix/operators.md @@ -3,28 +3,28 @@ Operators {{#include ../links.md}} -| Operator | Description | Binary? | -| :---------------: | ------------------------------ | :-----: | -| `+` | Add | Yes | -| `-` | Subtract, Minus | Yes/No | -| `*` | Multiply | Yes | -| `/` | Divide | Yes | -| `%` | Modulo | Yes | -| `~` | Power | Yes | -| `>>` | Right bit-shift | Yes | -| `<<` | Left bit-shift | Yes | -| `&` | Bit-wise _And_, Boolean _And_ | Yes | -| \| | Bit-wise _Or_, Boolean _Or_ | Yes | -| `^` | Bit-wise _Xor_ | Yes | -| `==` | Equals to | Yes | -| `~=` | Not equals to | Yes | -| `>` | Greater than | Yes | -| `>=` | Greater than or equals to | Yes | -| `<` | Less than | Yes | -| `<=` | Less than or equals to | Yes | -| `>=` | Greater than or equals to | Yes | -| `&&` | Boolean _And_ (short-circuits) | Yes | -| \|\| | Boolean _Or_ (short-circuits) | Yes | -| `!` | Boolean _Not_ | No | -| `[` .. `]` | Indexing | Yes | -| `.` | Property access, Method call | Yes | +| Operator | Description | Binary? | Binding direction | +| :---------------: | ------------------------------ | :-----: | :---------------: | +| `+` | Add | Yes | Left | +| `-` | Subtract, Minus | Yes/No | Left | +| `*` | Multiply | Yes | Left | +| `/` | Divide | Yes | Left | +| `%` | Modulo | Yes | Left | +| `~` | Power | Yes | Left | +| `>>` | Right bit-shift | Yes | Left | +| `<<` | Left bit-shift | Yes | Left | +| `&` | Bit-wise _And_, Boolean _And_ | Yes | Left | +| \| | Bit-wise _Or_, Boolean _Or_ | Yes | Left | +| `^` | Bit-wise _Xor_ | Yes | Left | +| `==` | Equals to | Yes | Left | +| `~=` | Not equals to | Yes | Left | +| `>` | Greater than | Yes | Left | +| `>=` | Greater than or equals to | Yes | Left | +| `<` | Less than | Yes | Left | +| `<=` | Less than or equals to | Yes | Left | +| `>=` | Greater than or equals to | Yes | Left | +| `&&` | Boolean _And_ (short-circuits) | Yes | Left | +| \|\| | Boolean _Or_ (short-circuits) | Yes | Left | +| `!` | Boolean _Not_ | No | Left | +| `[` .. `]` | Indexing | Yes | Right | +| `.` | Property access, Method call | Yes | Right | diff --git a/doc/src/engine/custom-op.md b/doc/src/engine/custom-op.md new file mode 100644 index 00000000..1c12263e --- /dev/null +++ b/doc/src/engine/custom-op.md @@ -0,0 +1,105 @@ +Custom Operators +================ + +{{#include ../links.md}} + +For use as a DSL (Domain-Specific Languages), it is sometimes more convenient to augment Rhai with +customized operators performing specific logic. + +`Engine::register_custom_operator` registers a keyword as a custom operator. + + +Example +------- + +```rust +use rhai::{Engine, RegisterFn}; + +let mut engine = Engine::new(); + +// Register a custom operator called 'foo' and give it +// a precedence of 140 (i.e. between +|- and *|/) +engine.register_custom_operator("foo", 140).unwrap(); + +// Register the implementation of the customer operator as a function +engine.register_fn("foo", |x: i64, y: i64| (x * y) - (x + y)); + +// The custom operator can be used in expressions +let result = engine.eval_expression::("1 + 2 * 3 foo 4 - 5 / 6")?; +// ^ custom operator + +// The above is equivalent to: 1 + ((2 * 3) foo 4) - (5 / 6) +result == 15; +``` + + +Alternatives to a Custom Operator +-------------------------------- + +Custom operators are merely _syntactic sugar_. They map directly to registered functions. + +Therefore, the following are equivalent (assuming `foo` has been registered as a custom operator): + +```rust +1 + 2 * 3 foo 4 - 5 / 6 // use custom operator + +1 + foo(2 * 3, 4) - 5 / 6 // use function call +``` + +A script using custom operators can always be pre-processed, via a pre-processor application, +into a syntax that uses the corresponding function calls. + +Using `Engine::register_custom_operator` merely enables a convenient short-cut. + + +Must Follow Variable Naming +-------------------------- + +All custom operators must be _identifiers_ that follow the same naming rules as [variables]. + +```rust +engine.register_custom_operator("foo", 20); // 'foo' is a valid custom operator + +engine.register_custom_operator("=>", 30); // <- error: '=>' is not a valid custom operator +``` + + +Binary Operators Only +--------------------- + +All custom operators must be _binary_ (i.e. they take two operands). +_Unary_ custom operators are not supported. + +```rust +engine.register_custom_operator("foo", 140).unwrap(); + +engine.register_fn("foo", |x: i64| x * x); + +engine.eval::("1 + 2 * 3 foo 4 - 5 / 6")?; // error: function 'foo (i64, i64)' not found +``` + + +Operator Precedence +------------------- + +All operators in Rhai has a _precedence_ indicating how tightly they bind. + +The following _precedence table_ show the built-in precedence of standard Rhai operators: + +| Category | Operators | Precedence (0-255) | +| ------------------- | :-------------------------------------------------------------------------------------: | :----------------: | +| Assignments | `=`, `+=`, `-=`, `*=`, `/=`, `~=`, `%=`,
`<<=`, `>>=`, `&=`, \|=, `^=` | 0 | +| Logic and bit masks | \|\|, \|, `^` | 30 | +| Logic and bit masks | `&`, `&&` | 60 | +| Comparisons | `==`, `!=`, `>`, `>=`, `<`, `<=` | 90 | +| | `in` | 110 | +| Arithmetic | `+`, `-` | 130 | +| Arithmetic | `*`, `/`, `~` | 160 | +| Bit-shifts | `<<`, `>>` | 190 | +| Arithmetic | `%` | 210 | +| Object | `.` _(binds to right)_ | 240 | +| _Others_ | | 0 | + +A higher precedence binds more tightly than a lower precedence, so `*` and `/` binds before `+` and `-` etc. + +When registering a custom operator, the operator's precedence must also be provided. diff --git a/doc/src/links.md b/doc/src/links.md index 7ef02a10..69a01198 100644 --- a/doc/src/links.md +++ b/doc/src/links.md @@ -104,3 +104,5 @@ [`OptimizationLevel::None`]: {{rootUrl}}/engine/optimize/optimize-levels.md [disable keywords and operators]: {{rootUrl}}/engine/disable.md +[custom operator]: {{rootUrl}}/engine/custom-op.md +[custom operators]: {{rootUrl}}/engine/custom-op.md diff --git a/src/api.rs b/src/api.rs index 5b6acdb5..6f49ad73 100644 --- a/src/api.rs +++ b/src/api.rs @@ -553,7 +553,7 @@ impl Engine { scripts: &[&str], optimization_level: OptimizationLevel, ) -> Result { - let stream = lex(scripts, self.max_string_size, self.disable_tokens.as_ref()); + let stream = lex(scripts, self); self.parse(&mut stream.peekable(), scope, optimization_level) } @@ -678,7 +678,7 @@ impl Engine { // Trims the JSON string and add a '#' in front let scripts = ["#", json.trim()]; - let stream = lex(&scripts, self.max_string_size, self.disable_tokens.as_ref()); + let stream = lex(&scripts, self); let ast = self.parse_global_expr(&mut stream.peekable(), &scope, OptimizationLevel::None)?; @@ -759,7 +759,7 @@ impl Engine { script: &str, ) -> Result { let scripts = [script]; - let stream = lex(&scripts, self.max_string_size, self.disable_tokens.as_ref()); + let stream = lex(&scripts, self); { let mut peekable = stream.peekable(); self.parse_global_expr(&mut peekable, scope, self.optimization_level) @@ -914,7 +914,7 @@ impl Engine { script: &str, ) -> Result> { let scripts = [script]; - let stream = lex(&scripts, self.max_string_size, self.disable_tokens.as_ref()); + let stream = lex(&scripts, self); // No need to optimize a lone expression let ast = self.parse_global_expr(&mut stream.peekable(), scope, OptimizationLevel::None)?; @@ -1047,7 +1047,7 @@ impl Engine { script: &str, ) -> Result<(), Box> { let scripts = [script]; - let stream = lex(&scripts, self.max_string_size, self.disable_tokens.as_ref()); + let stream = lex(&scripts, self); let ast = self.parse(&mut stream.peekable(), scope, self.optimization_level)?; self.consume_ast_with_scope(scope, &ast) } diff --git a/src/engine.rs b/src/engine.rs index 9dff48e4..e0a52882 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -268,8 +268,10 @@ pub struct Engine { /// A hashmap mapping type names to pretty-print names. pub(crate) type_names: Option>, - /// A hash-set containing tokens to disable. - pub(crate) disable_tokens: Option>, + /// A hashset containing symbols to disable. + pub(crate) disabled_symbols: Option>, + /// A hashset containing custom keywords and precedence to recognize. + pub(crate) custom_keywords: Option>, /// Callback closure for implementing the `print` command. pub(crate) print: Callback, @@ -317,7 +319,8 @@ impl Default for Engine { module_resolver: None, type_names: None, - disable_tokens: None, + disabled_symbols: None, + custom_keywords: None, // default print/debug implementations print: Box::new(default_print), @@ -497,7 +500,8 @@ impl Engine { module_resolver: None, type_names: None, - disable_tokens: None, + disabled_symbols: None, + custom_keywords: None, print: Box::new(|_| {}), debug: Box::new(|_| {}), diff --git a/src/parser.rs b/src/parser.rs index 797e28f4..4db75d94 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -332,36 +332,26 @@ pub enum ReturnType { Exception, } -#[derive(Debug, Clone, Eq, PartialEq, Hash, Default)] -struct ParseState { +#[derive(Clone)] +struct ParseState<'e> { + /// Reference to the scripting `Engine`. + engine: &'e Engine, /// Encapsulates a local stack with variable names to simulate an actual runtime scope. - pub stack: Vec<(String, ScopeEntryType)>, + stack: Vec<(String, ScopeEntryType)>, /// Encapsulates a local stack with variable names to simulate an actual runtime scope. - pub modules: Vec, + modules: Vec, /// Maximum levels of expression nesting. - pub max_expr_depth: usize, - /// Maximum length of a string. - pub max_string_size: usize, - /// Maximum length of an array. - pub max_array_size: usize, - /// Maximum number of properties in a map. - pub max_map_size: usize, + max_expr_depth: usize, } -impl ParseState { +impl<'e> ParseState<'e> { /// Create a new `ParseState`. - pub fn new( - max_expr_depth: usize, - max_string_size: usize, - max_array_size: usize, - max_map_size: usize, - ) -> Self { + pub fn new(engine: &'e Engine, max_expr_depth: usize) -> Self { Self { + engine, max_expr_depth, - max_string_size, - max_array_size, - max_map_size, - ..Default::default() + stack: Default::default(), + modules: Default::default(), } } /// Find a variable by name in the `ParseState`, searching in reverse. @@ -1206,10 +1196,10 @@ fn parse_array_literal( let mut arr = StaticVec::new(); while !input.peek().unwrap().0.is_eof() { - if state.max_array_size > 0 && arr.len() >= state.max_array_size { + if state.engine.max_array_size > 0 && arr.len() >= state.engine.max_array_size { return Err(PERR::LiteralTooLarge( "Size of array literal".to_string(), - state.max_array_size, + state.engine.max_array_size, ) .into_err(input.peek().unwrap().1)); } @@ -1306,10 +1296,10 @@ fn parse_map_literal( } }; - if state.max_map_size > 0 && map.len() >= state.max_map_size { + if state.engine.max_map_size > 0 && map.len() >= state.engine.max_map_size { return Err(PERR::LiteralTooLarge( "Number of properties in object map literal".to_string(), - state.max_map_size, + state.engine.max_map_size, ) .into_err(input.peek().unwrap().1)); } @@ -1866,7 +1856,8 @@ fn parse_binary_op( loop { let (current_op, _) = input.peek().unwrap(); - let precedence = current_op.precedence(); + let custom = state.engine.custom_keywords.as_ref(); + let precedence = current_op.precedence(custom); let bind_right = current_op.is_bind_right(); // Bind left to the parent lhs expression if precedence is higher @@ -1879,7 +1870,7 @@ fn parse_binary_op( let rhs = parse_unary(input, state, settings)?; - let next_precedence = input.peek().unwrap().0.precedence(); + let next_precedence = input.peek().unwrap().0.precedence(custom); // Bind to right if the next operator has higher precedence // If same precedence, then check if the operator binds right @@ -1949,6 +1940,19 @@ fn parse_binary_op( make_dot_expr(current_lhs, rhs, pos)? } + Token::Custom(s) + if state + .engine + .custom_keywords + .as_ref() + .map(|c| c.contains_key(&s)) + .unwrap_or(false) => + { + // Accept non-native functions for custom operators + let op = (op.0, false, op.2); + Expr::FnCall(Box::new((op, None, hash, args, None))) + } + op_token => return Err(PERR::UnknownOperator(op_token.into()).into_err(pos)), }; } @@ -2467,7 +2471,7 @@ fn parse_fn( settings.ensure_level_within_max_limit(state.max_expr_depth)?; let name = match input.next().unwrap() { - (Token::Identifier(s), _) => s, + (Token::Identifier(s), _) | (Token::Custom(s), _) => s, (_, pos) => return Err(PERR::FnMissingName.into_err(pos)), }; @@ -2555,12 +2559,7 @@ impl Engine { scope: &Scope, optimization_level: OptimizationLevel, ) -> Result { - let mut state = ParseState::new( - self.max_expr_depth, - self.max_string_size, - self.max_array_size, - self.max_map_size, - ); + let mut state = ParseState::new(self, self.max_expr_depth); let settings = ParseSettings { allow_if_expr: false, allow_stmt_expr: false, @@ -2596,12 +2595,7 @@ impl Engine { ) -> Result<(Vec, Vec), ParseError> { let mut statements = Vec::::new(); let mut functions = HashMap::::with_hasher(StraightHasherBuilder); - let mut state = ParseState::new( - self.max_expr_depth, - self.max_string_size, - self.max_array_size, - self.max_map_size, - ); + let mut state = ParseState::new(self, self.max_expr_depth); while !input.peek().unwrap().0.is_eof() { // Collect all the function definitions @@ -2615,12 +2609,7 @@ impl Engine { match input.peek().unwrap() { (Token::Fn, pos) => { - let mut state = ParseState::new( - self.max_function_expr_depth, - self.max_string_size, - self.max_array_size, - self.max_map_size, - ); + let mut state = ParseState::new(self, self.max_function_expr_depth); let settings = ParseSettings { allow_if_expr: true, allow_stmt_expr: true, diff --git a/src/settings.rs b/src/settings.rs index a4379d79..daa714bd 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -2,6 +2,7 @@ use crate::engine::Engine; use crate::module::ModuleResolver; use crate::optimize::OptimizationLevel; use crate::packages::PackageLibrary; +use crate::token::is_valid_identifier; impl Engine { /// Load a new package into the `Engine`. @@ -194,10 +195,60 @@ impl Engine { /// # } /// ``` pub fn disable_symbol(&mut self, symbol: &str) { - if self.disable_tokens.is_none() { - self.disable_tokens = Some(Default::default()); + if self.disabled_symbols.is_none() { + self.disabled_symbols = Some(Default::default()); } - self.disable_tokens.as_mut().unwrap().insert(symbol.into()); + self.disabled_symbols + .as_mut() + .unwrap() + .insert(symbol.into()); + } + + /// Register a custom operator into the language. + /// + /// The operator must be a valid identifier (i.e. it cannot be a symbol). + /// + /// # Examples + /// + /// ```rust + /// # fn main() -> Result<(), Box> { + /// use rhai::{Engine, RegisterFn}; + /// + /// let mut engine = Engine::new(); + /// + /// // Register a custom operator called 'foo' and give it + /// // a precedence of 140 (i.e. between +|- and *|/). + /// engine.register_custom_operator("foo", 140).unwrap(); + /// + /// // Register a binary function named 'foo' + /// engine.register_fn("foo", |x: i64, y: i64| (x * y) - (x + y)); + /// + /// assert_eq!( + /// engine.eval_expression::("1 + 2 * 3 foo 4 - 5 / 6")?, + /// 15 + /// ); + /// # Ok(()) + /// # } + /// ``` + pub fn register_custom_operator( + &mut self, + keyword: &str, + precedence: u8, + ) -> Result<(), String> { + if !is_valid_identifier(keyword.chars()) { + return Err(format!("not a valid identifier: '{}'", keyword).into()); + } + + if self.custom_keywords.is_none() { + self.custom_keywords = Some(Default::default()); + } + + self.custom_keywords + .as_mut() + .unwrap() + .insert(keyword.into(), precedence); + + Ok(()) } } diff --git a/src/token.rs b/src/token.rs index fc527917..056f9714 100644 --- a/src/token.rs +++ b/src/token.rs @@ -1,5 +1,6 @@ //! Main module defining the lexer and parser. +use crate::engine::Engine; use crate::error::LexError; use crate::parser::INT; use crate::utils::StaticVec; @@ -11,7 +12,7 @@ use crate::stdlib::{ borrow::Cow, boxed::Box, char, - collections::HashSet, + collections::{HashMap, HashSet}, fmt, iter::Peekable, str::{Chars, FromStr}, @@ -212,6 +213,7 @@ pub enum Token { As, LexError(Box), Comment(String), + Custom(String), EOF, } @@ -224,12 +226,13 @@ impl Token { IntegerConstant(i) => i.to_string().into(), #[cfg(not(feature = "no_float"))] FloatConstant(f) => f.to_string().into(), - Identifier(s) => s.clone().into(), + StringConstant(_) => "string".into(), CharConstant(c) => c.to_string().into(), + Identifier(s) => s.clone().into(), + Custom(s) => s.clone().into(), LexError(err) => err.to_string().into(), token => match token { - StringConstant(_) => "string", LeftBrace => "{", RightBrace => "}", LeftParen => "(", @@ -324,9 +327,9 @@ impl Token { match self { LexError(_) | - LeftBrace | // (+expr) - is unary + LeftBrace | // {+expr} - is unary // RightBrace | {expr} - expr not unary & is closing - LeftParen | // {-expr} - is unary + LeftParen | // (-expr) - is unary // RightParen | (expr) - expr not unary & is closing LeftBracket | // [-expr] - is unary // RightBracket | [expr] - expr not unary & is closing @@ -371,14 +374,14 @@ impl Token { Throw | PowerOf | In | - PowerOfAssign => true, + PowerOfAssign => true, _ => false, } } /// Get the precedence number of the token. - pub fn precedence(&self) -> u8 { + pub fn precedence(&self, custom: Option<&HashMap>) -> u8 { use Token::*; match self { @@ -387,24 +390,27 @@ impl Token { | RightShiftAssign | AndAssign | OrAssign | XOrAssign | ModuloAssign | PowerOfAssign => 0, - Or | XOr | Pipe => 40, + Or | XOr | Pipe => 30, - And | Ampersand => 50, + And | Ampersand => 60, LessThan | LessThanEqualsTo | GreaterThan | GreaterThanEqualsTo | EqualsTo - | NotEqualsTo => 60, + | NotEqualsTo => 90, - In => 70, + In => 110, - Plus | Minus => 80, + Plus | Minus => 130, - Divide | Multiply | PowerOf => 90, + Divide | Multiply | PowerOf => 160, - LeftShift | RightShift => 100, + LeftShift | RightShift => 190, - Modulo => 110, + Modulo => 210, - Period => 120, + Period => 240, + + // Custom operators + Custom(s) => custom.map_or(0, |c| *c.get(s).unwrap()), _ => 0, } @@ -1211,9 +1217,9 @@ impl InputStream for MultiInputsStream<'_> { } /// An iterator on a `Token` stream. -pub struct TokenIterator<'a, 't> { - /// Disable certain tokens. - pub disable_tokens: Option<&'t HashSet>, +pub struct TokenIterator<'a, 'e> { + /// Reference to the scripting `Engine`. + engine: &'e Engine, /// Current state. state: TokenizeState, /// Current position. @@ -1226,15 +1232,15 @@ impl<'a> Iterator for TokenIterator<'a, '_> { type Item = (Token, Position); fn next(&mut self) -> Option { - match get_next_token(&mut self.stream, &mut self.state, &mut self.pos) { - None => None, - r @ Some(_) if self.disable_tokens.is_none() => r, - Some((token, pos)) - if token.is_operator() - && self - .disable_tokens - .unwrap() - .contains(token.syntax().as_ref()) => + match ( + get_next_token(&mut self.stream, &mut self.state, &mut self.pos), + self.engine.disabled_symbols.as_ref(), + self.engine.custom_keywords.as_ref(), + ) { + (None, _, _) => None, + (r @ Some(_), None, None) => r, + (Some((token, pos)), Some(disabled), _) + if token.is_operator() && disabled.contains(token.syntax().as_ref()) => { // Convert disallowed operators into lex errors Some(( @@ -1242,31 +1248,27 @@ impl<'a> Iterator for TokenIterator<'a, '_> { pos, )) } - Some((token, pos)) - if token.is_keyword() - && self - .disable_tokens - .unwrap() - .contains(token.syntax().as_ref()) => + (Some((token, pos)), Some(disabled), _) + if token.is_keyword() && disabled.contains(token.syntax().as_ref()) => { // Convert disallowed keywords into identifiers Some((Token::Identifier(token.syntax().into()), pos)) } - r => r, + (Some((Token::Identifier(s), pos)), _, Some(custom)) if custom.contains_key(&s) => { + // Convert custom keywords + Some((Token::Custom(s), pos)) + } + (r, _, _) => r, } } } /// Tokenize an input text stream. -pub fn lex<'a, 't>( - input: &'a [&'a str], - max_string_size: usize, - disable_tokens: Option<&'t HashSet>, -) -> TokenIterator<'a, 't> { +pub fn lex<'a, 'e>(input: &'a [&'a str], engine: &'e Engine) -> TokenIterator<'a, 'e> { TokenIterator { - disable_tokens, + engine, state: TokenizeState { - max_string_size, + max_string_size: engine.max_string_size, non_unary: false, comment_level: 0, end_with_none: false, diff --git a/tests/tokens.rs b/tests/tokens.rs index 5fa0b9b6..b86f6a0e 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -1,4 +1,4 @@ -use rhai::{Engine, ParseErrorType}; +use rhai::{Engine, EvalAltResult, ParseErrorType, RegisterFn, INT}; #[test] fn test_tokens_disabled() { @@ -18,3 +18,32 @@ fn test_tokens_disabled() { ParseErrorType::BadInput(ref s) if s == "Unexpected '+='" )); } + +#[test] +fn test_tokens_custom_operator() -> Result<(), Box> { + let mut engine = Engine::new(); + + // Register a custom operator called `foo` and give it + // a precedence of 140 (i.e. between +|- and *|/). + engine.register_custom_operator("foo", 140).unwrap(); + + // Register a binary function named `foo` + engine.register_fn("foo", |x: INT, y: INT| (x * y) - (x + y)); + + assert_eq!( + engine.eval_expression::("1 + 2 * 3 foo 4 - 5 / 6")?, + 15 + ); + + assert_eq!( + engine.eval::( + r" + fn foo(x, y) { y - x } + 1 + 2 * 3 foo 4 - 5 / 6 + " + )?, + -1 + ); + + Ok(()) +}