diff --git a/Cargo.toml b/Cargo.toml index 6d48bc8d..99da6522 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ num-traits = { version = "0.2.11", default-features = false } [features] #default = ["unchecked", "sync", "no_optimize", "no_float", "only_i32", "no_index", "no_object", "no_function", "no_module"] -default = [] +default = ["internals"] plugins = [] unchecked = [] # unchecked arithmetic sync = [] # restrict to only types that implement Send + Sync diff --git a/README.md b/README.md index e448ba69..fb4acca1 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,8 @@ Features * Rugged - protected against malicious attacks (such as [stack-overflow](https://schungx.github.io/rhai/safety/max-call-stack.html), [over-sized data](https://schungx.github.io/rhai/safety/max-string-size.html), and [runaway scripts](https://schungx.github.io/rhai/safety/max-operations.html) etc.) that may come from untrusted third-party user-land scripts. * Track script evaluation [progress](https://schungx.github.io/rhai/safety/progress.html) and manually terminate a script run. * [Function overloading](https://schungx.github.io/rhai/language/overload.html). -* [Operator overloading](https://schungx.github.io/rhai/rust/operators.html) and [custom operators](https://schungx.github.io/rhai/engine/custom-op.html). +* [Operator overloading](https://schungx.github.io/rhai/rust/operators.html). +* Support for use as a [DSL](https://schungx.github.io/rhai/engine/dsl.html) - [disabling keywords/operators](https://schungx.github.io/rhai/engine/disable.html), [custom operators](https://schungx.github.io/rhai/engine/custom-op.html) and extending the language with [custom syntax](https://schungx.github.io/rhai/engine/custom-syntax.html). * Dynamic dispatch via [function pointers](https://schungx.github.io/rhai/language/fn-ptr.html). * Some support for [object-oriented programming (OOP)](https://schungx.github.io/rhai/language/oop.html). * Organize code base with dynamically-loadable [modules](https://schungx.github.io/rhai/language/modules.html). diff --git a/RELEASES.md b/RELEASES.md index 4afe903b..69c0186d 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -11,6 +11,11 @@ This version adds: * Ability to define custom operators (which must be valid identifiers). * Low-level API to register functions. +Bug fixes +--------- + +* Fixed method calls in the middle of a dot chain. + Breaking changes ---------------- @@ -31,6 +36,7 @@ New features * The boolean `^` (XOR) operator is added. * `FnPtr` is exposed as the function pointer type. * `rhai::module_resolvers::ModuleResolversCollection` added to try a list of module resolvers. +* It is now possible to mutate the first argument of a module-qualified function call when the argument is a simple variable (but not a module constant). Version 0.16.1 diff --git a/doc/src/SUMMARY.md b/doc/src/SUMMARY.md index 438b4590..e344c306 100644 --- a/doc/src/SUMMARY.md +++ b/doc/src/SUMMARY.md @@ -105,9 +105,11 @@ The Rhai Scripting Language 5. [Volatility Considerations](engine/optimize/volatility.md) 6. [Subtle Semantic Changes](engine/optimize/semantics.md) 4. [Low-Level API](rust/register-raw.md) - 5. [Disable Keywords and/or Operators](engine/disable.md) - 6. [Custom Operators](engine/custom-op.md) - 7. [Eval Statement](language/eval.md) + 5. [Use as DSL](engine/dsl.md) + 1. [Disable Keywords and/or Operators](engine/disable.md) + 2. [Custom Operators](engine/custom-op.md) + 3. [Extending with Custom Syntax](engine/custom-syntax.md) + 6. [Eval Statement](language/eval.md) 9. [Appendix](appendix/index.md) 1. [Keywords](appendix/keywords.md) 2. [Operators and Symbols](appendix/operators.md) diff --git a/doc/src/about/features.md b/doc/src/about/features.md index 5a5739cb..13b2e4c1 100644 --- a/doc/src/about/features.md +++ b/doc/src/about/features.md @@ -67,4 +67,5 @@ Flexible * Surgically [disable keywords and operators] to restrict the language. -* [Custom operators]. +* Use as a [DSL] by [disabling keywords/operators][disable keywords and operators], [custom operators] + and extending the language with [custom syntax]. diff --git a/doc/src/appendix/operators.md b/doc/src/appendix/operators.md index 3e895da9..b4303d79 100644 --- a/doc/src/appendix/operators.md +++ b/doc/src/appendix/operators.md @@ -41,6 +41,7 @@ Symbols | ------------ | ------------------------ | | `:` | Property value separator | | `::` | Module path separator | +| `#` | _Reserved_ | | `=>` | _Reserved_ | | `->` | _Reserved_ | | `<-` | _Reserved_ | diff --git a/doc/src/engine/custom-syntax.md b/doc/src/engine/custom-syntax.md new file mode 100644 index 00000000..542efaa5 --- /dev/null +++ b/doc/src/engine/custom-syntax.md @@ -0,0 +1,282 @@ +Extending Rhai with Custom Syntax +================================ + +{{#include ../links.md}} + + +For the ultimate advantageous, there is a built-in facility to _extend_ the Rhai language +with custom-defined _syntax_. + +But before going off to define the next weird statement type, heed this warning: + + +Don't Do It™ +------------ + +Stick with standard language syntax as much as possible. + +Having to learn Rhai is bad enough, no sane user would ever want to learn _yet_ another +obscure language syntax just to do something. + +Try to use [custom operators] first. Defining a custom syntax should be considered a _last resort_. + + +Where This Might Be Useful +------------------------- + +* Where an operation is used a _LOT_ and a custom syntax saves a lot of typing. + +* Where a custom syntax _significantly_ simplifies the code and _significantly_ enhances understanding of the code's intent. + +* Where certain logic cannot be easily encapsulated inside a function. This is usually the case where _closures_ are required, because Rhai does not have closures. + +* Where you just want to confuse your user and make their lives miserable, because you can. + + +Step One - Start With `internals` +-------------------------------- + +Since a custom syntax taps deeply into the `AST` and evaluation process of the `Engine`, +the [`internals`] feature must be on in order to expose these necessary internal data structures. + +Beware that Rhai internal data structures are _volatile_ and may change without warning. + +Caveat emptor. + + +Step Two - Design The Syntax +--------------------------- + +A custom syntax is simply a list of symbols. + +These symbol types can be used: + +* Standard [keywords]({{rootUrl}}/appendix/keywords.md) + +* Standard [operators]({{rootUrl}}/appendix/operators.md#operators). + +* Reserved [symbols]({{rootUrl}}/appendix/operators.md#symbols). + +* Identifiers following the [variable] naming rules. + +* `$expr$` - any valid expression, statement or statement block. + +* `$block$` - any valid statement block (i.e. must be enclosed by `'{'` .. `'}'`). + +* `$ident$` - any [variable] name. + +### The First Symbol Must be a Keyword + +There is no specific limit on the combination and sequencing of each symbol type, +except the _first_ symbol which must be a [custom keyword]. + +It _cannot_ be a [built-in keyword]({{rootUrl}}/appendix/keywords.md). + +However, it _may_ be a built-in keyword that has been [disabled][disable keywords and operators]. + +### The First Symbol Must be Unique + +Rhai uses the _first_ symbol as a clue to parse custom syntax. + +Therefore, at any one time, there can only be _one_ custom syntax starting with each unique symbol. + +Any new custom syntax definition using the same first symbol simply _overwrites_ the previous one. + +### Example + +```rust +exec $ident$ <- $expr$ : $block$ +``` + +The above syntax is made up of a stream of symbols: + +| Position | Input | Symbol | Description | +| :------: | :---: | :-------: | -------------------------------------------------------------------------------------------------------- | +| 1 | | `exec` | custom keyword | +| 2 | 1 | `$ident$` | a variable name | +| 3 | | `<-` | the left-arrow symbol (which is a [reserved symbol]({{rootUrl}}/appendix/operators.md#symbols) in Rhai). | +| 4 | 2 | `$expr$` | an expression, which may be enclosed with `{` .. `}`, or not. | +| 5 | | `:` | the colon symbol | +| 6 | 3 | `$block$` | a statement block, which must be enclosed with `{` .. `}`. | + +This syntax matches the following sample code and generates three inputs (one for each non-keyword): + +```rust +// Assuming the 'exec' custom syntax implementation declares the variable 'hello': +let x = exec hello <- foo(1, 2) : { + hello += bar(hello); + baz(hello); + }; + +print(x); // variable 'x' has a value returned by the custom syntax + +print(hello); // variable declared by a custom syntax persists! +``` + + +Step Three - Implementation +-------------------------- + +Any custom syntax must include an _implementation_ of it. + +### Function Signature + +The function signature of an implementation is: + +```rust +Fn( + engine: &Engine, + scope: &mut Scope, + mods: &mut Imports, + state: &mut State, + lib: &Module, + this_ptr: &mut Option<&mut Dynamic>, + inputs: &[Expr], + level: usize +) -> Result> +``` + +where: + +* `engine : &Engine` - reference to the current [`Engine`]. +* `scope : &mut Scope` - mutable reference to the current [`Scope`]; variables can be added to it. +* `mods : &mut Imports` - mutable reference to the current collection of imported [`Module`]'s; **do not touch**. +* `state : &mut State` - mutable reference to the current evaluation state; **do not touch**. +* `lib : &Module` - reference to the current collection of script-defined functions. +* `this_ptr : &mut Option<&mut Dynamic>` - mutable reference to the current binding of the `this` pointer; **do not touch**. +* `inputs : &[Expr]` - a list of input expression trees. +* `level : usize` - the current function call level. + +There are a lot of parameters, most of which should not be touched or Bad Things Happen™. +They represent the running _content_ of a script evaluation and should simply be passed +straight-through the the [`Engine`]. + +### Access Arguments + +The most important argument is `inputs` where the matched identifiers (`$ident$`), expressions/statements (`$expr$`) +and statement blocks (`$block$) are provided. + +To access a particular argument, use the following patterns: + +| Argument type | Pattern (`n` = slot in `inputs`) | Result type | Description | +| :-----------: | ---------------------------------------- | :---------: | ------------------ | +| `$ident$` | `inputs[n].get_variable_name().unwrap()` | `&str` | name of a variable | +| `$expr$` | `inputs.get(n).unwrap()` | `Expr` | an expression tree | +| `$block$` | `inputs.get(n).unwrap()` | `Expr` | an expression tree | + +### Evaluate an Expression Tree + +Use the `engine::eval_expression_tree` method to evaluate an expression tree. + +```rust +let expr = inputs.get(0).unwrap(); +let result = engine.eval_expression_tree(scope, mods, state, lib, this_ptr, expr, level)?; +``` + +As can be seem above, most arguments are simply passed straight-through to `engine::eval_expression_tree`. + +### Declare Variables + +New variables maybe declared (usually with a variable name that is passed in via `$ident$). + +It can simply be pushed into the [`scope`]. + +However, beware that all new variables must be declared _prior_ to evaluating any expression tree. +In other words, any `scope.push(...)` calls must come _before_ any `engine::eval_expression_tree(...)` calls. + +```rust +let var_name = inputs[0].get_variable_name().unwrap().to_string(); +let expr = inputs.get(1).unwrap(); + +scope.push(var_name, 0 as INT); // do this BEFORE engine.eval_expression_tree! + +let result = engine.eval_expression_tree(scope, mods, state, lib, this_ptr, expr, level)?; +``` + + +Step Four - Register the Custom Syntax +------------------------------------- + +Use `Engine::register_custom_syntax` to register a custom syntax. + +Again, beware that the _first_ symbol must be unique. If there already exists a custom syntax starting +with that symbol, the previous syntax will be overwritten. + +The syntax is passed simply as a slice of `&str`. + +```rust +// Custom syntax implementation +fn implementation_func( + engine: &Engine, + scope: &mut Scope, + mods: &mut Imports, + state: &mut State, + lib: &Module, + this_ptr: &mut Option<&mut Dynamic>, + inputs: &[Expr], + level: usize +) -> Result> { + let var_name = inputs[0].get_variable_name().unwrap().to_string(); + let stmt = inputs.get(1).unwrap(); + let condition = inputs.get(2).unwrap(); + + // Push one new variable into the 'scope' BEFORE 'eval_expression_tree' + scope.push(var_name, 0 as INT); + + loop { + // Evaluate the statement block + engine.eval_expression_tree(scope, mods, state, lib, this_ptr, stmt, level)?; + + // Evaluate the condition expression + let stop = !engine + .eval_expression_tree(scope, mods, state, lib, this_ptr, condition, level)? + .as_bool() + .map_err(|_| EvalAltResult::ErrorBooleanArgMismatch( + "do-while".into(), expr.position() + ))?; + + if stop { + break; + } + } + + Ok(().into()) +} + +// Register the custom syntax (sample): do |x| -> { x += 1 } while x < 0; +engine.register_custom_syntax( + &[ "do", "|", "$ident$", "|", "->", "$block$", "while", "$expr$" ], // the custom syntax + 1, // the number of new variables declared within this custom syntax + implementation_func +)?; +``` + + +Step Five - Disable Unneeded Statement Types +------------------------------------------- + +When a DSL needs a custom syntax, most likely than not it is extremely specialized. +Therefore, many statement types actually may not make sense under the same usage scenario. + +So, while at it, better [disable][disable keywords and operators] those built-in keywords +and operators that should not be used by the user. The would leave only the bare minimum +language surface exposed, together with the custom syntax that is tailor-designed for +the scenario. + +A keyword or operator that is disabled can still be used in a custom syntax. + +In an extreme case, it is possible to disable _every_ keyword in the language, leaving only +custom syntax (plus possibly expressions). But again, Don't Do It™ - unless you are certain +of what you're doing. + + +Step Six - Document +------------------- + +For custom syntax, documentation is crucial. + +Make sure there are _lots_ of examples for users to follow. + + +Step Seven - Profit! +-------------------- diff --git a/doc/src/engine/dsl.md b/doc/src/engine/dsl.md new file mode 100644 index 00000000..5534f243 --- /dev/null +++ b/doc/src/engine/dsl.md @@ -0,0 +1,84 @@ +Use Rhai as a Domain-Specific Language (DSL) +=========================================== + +{{#include ../links.md}} + +Rhai can be successfully used as a domain-specific language (DSL). + + +Expressions Only +---------------- + +In many DSL scenarios, only evaluation of expressions is needed. + +The `Engine::eval_expression_XXX`[`eval_expression`] API can be used to restrict +a script to expressions only. + + +Disable Keywords and/or Operators +-------------------------------- + +In some DSL scenarios, it is necessary to further restrict the language to exclude certain +language features that are not necessary or dangerous to the application. + +For example, a DSL may disable the `while` loop altogether while keeping all other statement +types intact. + +It is possible, in Rhai, to surgically [disable keywords and operators]. + + +Custom Operators +---------------- + +On the other hand, some DSL scenarios require special operators that make sense only for +that specific environment. In such cases, it is possible to define [custom operators] in Rhai. + +For example: + +```rust +let animal = "rabbit"; +let food = "carrot"; + +animal eats food // custom operator - 'eats' + +eats(animal, food) // <- the above really de-sugars to this +``` + +Although a [custom operator] always de-sugars to a simple function call, +nevertheless it makes the DSL syntax much simpler and expressive. + + +Custom Syntax +------------- + +For advanced DSL scenarios, it is possible to define entire expression [_syntax_][custom syntax] - +essentially custom statement types. + +The [`internals`] feature is needed to be able to define [custom syntax] in Rhai. + +For example, the following is a SQL like syntax for some obscure DSL operation: + +```rust +let table = [..., ..., ..., ...]; + +// Syntax = "calculate" $ident$ $ident$ "from" $expr$ "->" $ident$ ":" $expr$ +let total = calculate sum price from table -> row : row.weight > 50; + +// Note: There is nothing special about the use of symbols; to make it look exactly like SQL: +// Syntax = "SELECT" $ident$ "(" $ident$ ")" "FROM" $expr$ "AS" $ident$ "WHERE" $expr$ +let total = SELECT sum(price) FROM table AS row WHERE row.weight > 50; +``` + +After registering this custom syntax with Rhai, it can be used anywhere inside a script as +a normal expression. + +For its evaluation, the callback function will receive the following list of parameters: + +`exprs[0] = "sum"` - math operator +`exprs[1] = "price"` - field name +`exprs[2] = Expr(table)` - data source +`exprs[3] = "row"` - loop variable name +`exprs[4] = Expr(row.wright > 50)` - expression + +The other identified, such as `"select"`, `"from"`, as as as symbols `->` and `:` are +parsed in the order defined within the custom syntax. diff --git a/doc/src/language/loop.md b/doc/src/language/loop.md index e625082a..3fd9b5fb 100644 --- a/doc/src/language/loop.md +++ b/doc/src/language/loop.md @@ -3,9 +3,9 @@ Infinite `loop` {{#include ../links.md}} -Infinite loops follow C syntax. +Infinite loops follow Rust syntax. -Like C, `continue` can be used to skip to the next iteration, by-passing all following statements; +Like Rust, `continue` can be used to skip to the next iteration, by-passing all following statements; `break` can be used to break out of the loop unconditionally. ```rust diff --git a/doc/src/links.md b/doc/src/links.md index 85561acc..82ceab3b 100644 --- a/doc/src/links.md +++ b/doc/src/links.md @@ -89,6 +89,7 @@ [`eval`]: {{rootUrl}}/language/eval.md [OOP]: {{rootUrl}}/language/oop.md +[DSL]: {{rootUrl}}/engine/dsl.md [maximum statement depth]: {{rootUrl}}/safety/max-stmt-depth.md [maximum call stack depth]: {{rootUrl}}/safety/max-call-stack.md @@ -107,3 +108,4 @@ [disable keywords and operators]: {{rootUrl}}/engine/disable.md [custom operator]: {{rootUrl}}/engine/custom-op.md [custom operators]: {{rootUrl}}/engine/custom-op.md +[custom syntax]: {{rootUrl}}/engine/custom-syntax.md diff --git a/doc/src/start/features.md b/doc/src/start/features.md index 77a17e10..023834fd 100644 --- a/doc/src/start/features.md +++ b/doc/src/start/features.md @@ -25,7 +25,7 @@ more control over what a script can (or cannot) do. | `no_module` | Disable loading external [modules]. | | `no_std` | Build for `no-std`. Notice that additional dependencies will be pulled in to replace `std` features. | | `serde` | Enable serialization/deserialization via [`serde`]. Notice that the [`serde`](https://crates.io/crates/serde) crate will be pulled in together with its dependencies. | -| `internals` | Expose internal data structures (e.g. [`AST`] nodes). Beware that Rhai internals are volatile and may change from version to version. | +| `internals` | Expose internal data structures (e.g. [`AST`] nodes) and enable defining [custom syntax]. Beware that Rhai internals are volatile and may change from version to version. | Example diff --git a/src/engine.rs b/src/engine.rs index 54890e29..423b45b0 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -17,6 +17,9 @@ use crate::utils::StaticVec; #[cfg(not(feature = "no_float"))] use crate::parser::FLOAT; +#[cfg(feature = "internals")] +use crate::syntax::CustomSyntax; + use crate::stdlib::{ any::{type_name, TypeId}, borrow::Cow, @@ -82,8 +85,15 @@ pub const KEYWORD_THIS: &str = "this"; pub const FN_TO_STRING: &str = "to_string"; pub const FN_GET: &str = "get$"; pub const FN_SET: &str = "set$"; -pub const FN_IDX_GET: &str = "$index$get$"; -pub const FN_IDX_SET: &str = "$index$set$"; +pub const FN_IDX_GET: &str = "index$get$"; +pub const FN_IDX_SET: &str = "index$set$"; + +#[cfg(feature = "internals")] +pub const MARKER_EXPR: &str = "$expr$"; +#[cfg(feature = "internals")] +pub const MARKER_BLOCK: &str = "$block$"; +#[cfg(feature = "internals")] +pub const MARKER_IDENT: &str = "$ident$"; /// A type specifying the method of chaining. #[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] @@ -273,6 +283,9 @@ pub struct Engine { pub(crate) disabled_symbols: Option>, /// A hashset containing custom keywords and precedence to recognize. pub(crate) custom_keywords: Option>, + /// Custom syntax. + #[cfg(feature = "internals")] + pub(crate) custom_syntax: Option>, /// Callback closure for implementing the `print` command. pub(crate) print: Callback, @@ -323,6 +336,9 @@ impl Default for Engine { disabled_symbols: None, custom_keywords: None, + #[cfg(feature = "internals")] + custom_syntax: None, + // default print/debug implementations print: Box::new(default_print), debug: Box::new(default_print), @@ -555,6 +571,9 @@ impl Engine { disabled_symbols: None, custom_keywords: None, + #[cfg(feature = "internals")] + custom_syntax: None, + print: Box::new(|_| {}), debug: Box::new(|_| {}), progress: None, @@ -1003,6 +1022,81 @@ impl Engine { return Ok(result); } + /// Call a dot method. + fn call_method( + &self, + state: &mut State, + lib: &Module, + target: &mut Target, + expr: &Expr, + mut idx_val: Dynamic, + level: usize, + ) -> Result<(Dynamic, bool), Box> { + let ((name, native, pos), _, hash, _, def_val) = match expr { + Expr::FnCall(x) => x.as_ref(), + _ => unreachable!(), + }; + + let is_ref = target.is_ref(); + let is_value = target.is_value(); + let def_val = def_val.as_ref(); + + // Get a reference to the mutation target Dynamic + let obj = target.as_mut(); + let idx = idx_val.downcast_mut::>().unwrap(); + let mut fn_name = name.as_ref(); + + // Check if it is a FnPtr call + let (result, updated) = if fn_name == KEYWORD_FN_PTR_CALL && obj.is::() { + // Redirect function name + fn_name = obj.as_str().unwrap(); + // Recalculate hash + let hash = calc_fn_hash(empty(), fn_name, idx.len(), empty()); + // Arguments are passed as-is + let mut arg_values = idx.iter_mut().collect::>(); + let args = arg_values.as_mut(); + + // Map it to name(args) in function-call style + self.exec_fn_call( + state, lib, fn_name, *native, hash, args, false, false, def_val, level, + ) + } else { + let redirected: Option; + let mut hash = *hash; + + // Check if it is a map method call in OOP style + if let Some(map) = obj.downcast_ref::() { + if let Some(val) = map.get(fn_name) { + if let Some(f) = val.downcast_ref::() { + // Remap the function name + redirected = Some(f.get_fn_name().clone()); + fn_name = redirected.as_ref().unwrap(); + + // Recalculate the hash based on the new function name + hash = calc_fn_hash(empty(), fn_name, idx.len(), empty()); + } + } + }; + + // Attached object pointer in front of the arguments + let mut arg_values = once(obj).chain(idx.iter_mut()).collect::>(); + let args = arg_values.as_mut(); + + self.exec_fn_call( + state, lib, fn_name, *native, hash, args, is_ref, true, def_val, level, + ) + } + .map_err(|err| err.new_position(*pos))?; + + // Feed the changed temp value back + if updated && !is_ref && !is_value { + let new_val = target.as_mut().clone(); + target.set_value(new_val)?; + } + + Ok((result, updated)) + } + /// Chain-evaluate a dot/index chain. /// Position in `EvalAltResult` is None and must be set afterwards. fn eval_dot_index_chain_helper( @@ -1022,7 +1116,6 @@ impl Engine { } let is_ref = target.is_ref(); - let is_value = target.is_value(); let next_chain = match rhs { Expr::Index(_) => ChainType::Index, @@ -1031,7 +1124,7 @@ impl Engine { }; // Pop the last index value - let mut idx_val = idx_values.pop(); + let idx_val = idx_values.pop(); match chain_type { #[cfg(not(feature = "no_index"))] @@ -1115,69 +1208,7 @@ impl Engine { match rhs { // xxx.fn_name(arg_expr_list) Expr::FnCall(x) if x.1.is_none() => { - let ((name, native, pos), _, hash, _, def_val) = x.as_ref(); - let def_val = def_val.as_ref(); - - // Get a reference to the mutation target Dynamic - let (result, updated) = { - let obj = target.as_mut(); - let idx = idx_val.downcast_mut::>().unwrap(); - let mut fn_name = name.as_ref(); - - // Check if it is a FnPtr call - if fn_name == KEYWORD_FN_PTR_CALL && obj.is::() { - // Redirect function name - fn_name = obj.as_str().unwrap(); - // Recalculate hash - let hash = calc_fn_hash(empty(), fn_name, idx.len(), empty()); - // Arguments are passed as-is - let mut arg_values = idx.iter_mut().collect::>(); - let args = arg_values.as_mut(); - - // Map it to name(args) in function-call style - self.exec_fn_call( - state, lib, fn_name, *native, hash, args, false, false, - def_val, level, - ) - } else { - let redirected: Option; - let mut hash = *hash; - - // Check if it is a map method call in OOP style - if let Some(map) = obj.downcast_ref::() { - if let Some(val) = map.get(fn_name) { - if let Some(f) = val.downcast_ref::() { - // Remap the function name - redirected = Some(f.get_fn_name().clone()); - fn_name = redirected.as_ref().unwrap(); - - // Recalculate the hash based on the new function name - hash = - calc_fn_hash(empty(), fn_name, idx.len(), empty()); - } - } - }; - - // Attached object pointer in front of the arguments - let mut arg_values = - once(obj).chain(idx.iter_mut()).collect::>(); - let args = arg_values.as_mut(); - - self.exec_fn_call( - state, lib, fn_name, *native, hash, args, is_ref, true, - def_val, level, - ) - } - .map_err(|err| err.new_position(*pos))? - }; - - // Feed the changed temp value back - if updated && !is_ref && !is_value { - let new_val = target.as_mut().clone(); - target.set_value(new_val)?; - } - - Ok((result, updated)) + self.call_method(state, lib, target, rhs, idx_val, level) } // xxx.module::fn_name(...) - syntax error Expr::FnCall(_) => unreachable!(), @@ -1221,16 +1252,26 @@ impl Engine { .map(|(v, _)| (v, false)) .map_err(|err| err.new_position(*pos)) } - // {xxx:map}.prop[expr] | {xxx:map}.prop.expr + // {xxx:map}.sub_lhs[expr] | {xxx:map}.sub_lhs.expr Expr::Index(x) | Expr::Dot(x) if target.is::() => { - let (prop, expr, pos) = x.as_ref(); + let (sub_lhs, expr, pos) = x.as_ref(); - let mut val = if let Expr::Property(p) = prop { - let ((prop, _, _), _) = p.as_ref(); - let index = prop.clone().into(); - self.get_indexed_mut(state, lib, target, index, *pos, false, level)? - } else { - unreachable!(); + let mut val = match sub_lhs { + Expr::Property(p) => { + let ((prop, _, _), _) = p.as_ref(); + let index = prop.clone().into(); + self.get_indexed_mut(state, lib, target, index, *pos, false, level)? + } + // {xxx:map}.fn_name(arg_expr_list)[expr] | {xxx:map}.fn_name(arg_expr_list).expr + Expr::FnCall(x) if x.1.is_none() => { + let (val, _) = + self.call_method(state, lib, target, sub_lhs, idx_val, level)?; + val.into() + } + // {xxx:map}.module::fn_name(...) - syntax error + Expr::FnCall(_) => unreachable!(), + // Others - syntax error + _ => unreachable!(), }; self.eval_dot_index_chain_helper( @@ -1239,49 +1280,72 @@ impl Engine { ) .map_err(|err| err.new_position(*pos)) } - // xxx.prop[expr] | xxx.prop.expr + // xxx.sub_lhs[expr] | xxx.sub_lhs.expr Expr::Index(x) | Expr::Dot(x) => { - let (prop, expr, pos) = x.as_ref(); - let args = &mut [target.as_mut(), &mut Default::default()]; + let (sub_lhs, expr, pos) = x.as_ref(); - let (mut val, updated) = if let Expr::Property(p) = prop { - let ((_, getter, _), _) = p.as_ref(); - let args = &mut args[..1]; - self.exec_fn_call( - state, lib, getter, true, 0, args, is_ref, true, None, level, - ) - .map_err(|err| err.new_position(*pos))? - } else { - unreachable!(); - }; - let val = &mut val; - let target = &mut val.into(); + match sub_lhs { + // xxx.prop[expr] | xxx.prop.expr + Expr::Property(p) => { + let ((_, getter, setter), _) = p.as_ref(); + let arg_values = &mut [target.as_mut(), &mut Default::default()]; + let args = &mut arg_values[..1]; + let (mut val, updated) = self + .exec_fn_call( + state, lib, getter, true, 0, args, is_ref, true, None, + level, + ) + .map_err(|err| err.new_position(*pos))?; - let (result, may_be_changed) = self - .eval_dot_index_chain_helper( - state, lib, this_ptr, target, expr, idx_values, next_chain, level, - new_val, - ) - .map_err(|err| err.new_position(*pos))?; + let val = &mut val; + let target = &mut val.into(); - // Feed the value back via a setter just in case it has been updated - if updated || may_be_changed { - if let Expr::Property(p) = prop { - let ((_, _, setter), _) = p.as_ref(); - // Re-use args because the first &mut parameter will not be consumed - args[1] = val; - self.exec_fn_call( - state, lib, setter, true, 0, args, is_ref, true, None, level, - ) - .or_else(|err| match *err { - // If there is no setter, no need to feed it back because the property is read-only - EvalAltResult::ErrorDotExpr(_, _) => Ok(Default::default()), - _ => Err(err.new_position(*pos)), - })?; + let (result, may_be_changed) = self + .eval_dot_index_chain_helper( + state, lib, this_ptr, target, expr, idx_values, next_chain, + level, new_val, + ) + .map_err(|err| err.new_position(*pos))?; + + // Feed the value back via a setter just in case it has been updated + if updated || may_be_changed { + // Re-use args because the first &mut parameter will not be consumed + arg_values[1] = val; + self.exec_fn_call( + state, lib, setter, true, 0, arg_values, is_ref, true, + None, level, + ) + .or_else( + |err| match *err { + // If there is no setter, no need to feed it back because the property is read-only + EvalAltResult::ErrorDotExpr(_, _) => { + Ok(Default::default()) + } + _ => Err(err.new_position(*pos)), + }, + )?; + } + + Ok((result, may_be_changed)) } - } + // xxx.fn_name(arg_expr_list)[expr] | xxx.fn_name(arg_expr_list).expr + Expr::FnCall(x) if x.1.is_none() => { + let (mut val, _) = + self.call_method(state, lib, target, sub_lhs, idx_val, level)?; + let val = &mut val; + let target = &mut val.into(); - Ok((result, may_be_changed)) + self.eval_dot_index_chain_helper( + state, lib, this_ptr, target, expr, idx_values, next_chain, + level, new_val, + ) + .map_err(|err| err.new_position(*pos)) + } + // xxx.module::fn_name(...) - syntax error + Expr::FnCall(_) => unreachable!(), + // Others - syntax error + _ => unreachable!(), + } } // Syntax error _ => Err(Box::new(EvalAltResult::ErrorDotExpr( @@ -1316,7 +1380,7 @@ impl Engine { let idx_values = &mut StaticVec::new(); self.eval_indexed_chain( - scope, mods, state, lib, this_ptr, dot_rhs, idx_values, 0, level, + scope, mods, state, lib, this_ptr, dot_rhs, chain_type, idx_values, 0, level, )?; match dot_lhs { @@ -1380,6 +1444,7 @@ impl Engine { lib: &Module, this_ptr: &mut Option<&mut Dynamic>, expr: &Expr, + chain_type: ChainType, idx_values: &mut StaticVec, size: usize, level: usize, @@ -1406,12 +1471,29 @@ impl Engine { // Evaluate in left-to-right order let lhs_val = match lhs { Expr::Property(_) => Default::default(), // Store a placeholder in case of a property + Expr::FnCall(x) if chain_type == ChainType::Dot && x.1.is_none() => { + let arg_values = x + .3 + .iter() + .map(|arg_expr| { + self.eval_expr(scope, mods, state, lib, this_ptr, arg_expr, level) + }) + .collect::, _>>()?; + + Dynamic::from(arg_values) + } + Expr::FnCall(_) => unreachable!(), _ => self.eval_expr(scope, mods, state, lib, this_ptr, lhs, level)?, }; // Push in reverse order + let chain_type = match expr { + Expr::Index(_) => ChainType::Index, + Expr::Dot(_) => ChainType::Dot, + _ => unreachable!(), + }; self.eval_indexed_chain( - scope, mods, state, lib, this_ptr, rhs, idx_values, size, level, + scope, mods, state, lib, this_ptr, rhs, chain_type, idx_values, size, level, )?; idx_values.push(lhs_val); @@ -1595,6 +1677,26 @@ impl Engine { } } + /// Evaluate an expression tree. + /// + /// ## WARNING - Low Level API + /// + /// This function is very low level. It evaluates an expression from an AST. + #[cfg(feature = "internals")] + #[deprecated(note = "this method is volatile and may change")] + pub fn eval_expression_tree( + &self, + scope: &mut Scope, + mods: &mut Imports, + state: &mut State, + lib: &Module, + this_ptr: &mut Option<&mut Dynamic>, + expr: &Expr, + level: usize, + ) -> Result> { + self.eval_expr(scope, mods, state, lib, this_ptr, expr, level) + } + /// Evaluate an expression fn eval_expr( &self, @@ -2027,6 +2129,13 @@ impl Engine { Expr::False(_) => Ok(false.into()), Expr::Unit(_) => Ok(().into()), + #[cfg(feature = "internals")] + Expr::Custom(x) => { + let func = (x.0).1.as_ref(); + let exprs = (x.0).0.as_ref(); + func(self, scope, mods, state, lib, this_ptr, exprs, level) + } + _ => unreachable!(), }; diff --git a/src/fn_native.rs b/src/fn_native.rs index 8de5537e..c7b2a4c9 100644 --- a/src/fn_native.rs +++ b/src/fn_native.rs @@ -1,3 +1,4 @@ +//! Module containing interfaces with native-Rust functions. use crate::any::Dynamic; use crate::engine::Engine; use crate::module::Module; diff --git a/src/lib.rs b/src/lib.rs index 064e0c89..ee409594 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -95,6 +95,8 @@ mod scope; mod serde; mod settings; mod stdlib; +#[cfg(feature = "internals")] +mod syntax; mod token; mod r#unsafe; mod utils; @@ -157,13 +159,21 @@ pub use optimize::OptimizationLevel; // Expose internal data structures. +#[cfg(feature = "internals")] +#[deprecated(note = "this type is volatile and may change")] +pub use error::LexError; + #[cfg(feature = "internals")] #[deprecated(note = "this type is volatile and may change")] pub use token::{get_next_token, parse_string_literal, InputStream, Token, TokenizeState}; #[cfg(feature = "internals")] #[deprecated(note = "this type is volatile and may change")] -pub use parser::{Expr, ReturnType, ScriptFnDef, Stmt}; +pub use parser::{CustomExpr, Expr, ReturnType, ScriptFnDef, Stmt}; + +#[cfg(feature = "internals")] +#[deprecated(note = "this type is volatile and may change")] +pub use engine::{Imports, State as EvalState}; #[cfg(feature = "internals")] #[deprecated(note = "this type is volatile and may change")] diff --git a/src/optimize.rs b/src/optimize.rs index f3f09919..ed07d1fb 100644 --- a/src/optimize.rs +++ b/src/optimize.rs @@ -6,6 +6,9 @@ use crate::parser::{map_dynamic_to_expr, Expr, ReturnType, ScriptFnDef, Stmt, AS use crate::scope::{Entry as ScopeEntry, EntryType as ScopeEntryType, Scope}; use crate::utils::StaticVec; +#[cfg(feature = "internals")] +use crate::parser::CustomExpr; + use crate::stdlib::{ boxed::Box, iter::empty, @@ -598,6 +601,15 @@ fn optimize_expr(expr: Expr, state: &mut State) -> Expr { state.find_constant(&name).expect("should find constant in scope!").clone().set_position(pos) } + // Custom syntax + #[cfg(feature = "internals")] + Expr::Custom(x) => Expr::Custom(Box::new(( + CustomExpr( + (x.0).0.into_iter().map(|expr| optimize_expr(expr, state)).collect(), + (x.0).1), + x.1 + ))), + // All other expressions - skip expr => expr, } diff --git a/src/parser.rs b/src/parser.rs index b9320132..4df5b799 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -10,6 +10,15 @@ use crate::scope::{EntryType as ScopeEntryType, Scope}; use crate::token::{Position, Token, TokenStream}; use crate::utils::{StaticVec, StraightHasherBuilder}; +#[cfg(feature = "internals")] +use crate::engine::{MARKER_BLOCK, MARKER_EXPR, MARKER_IDENT}; + +#[cfg(feature = "internals")] +use crate::fn_native::Shared; + +#[cfg(feature = "internals")] +use crate::syntax::FnCustomSyntaxEval; + use crate::stdlib::{ borrow::Cow, boxed::Box, @@ -568,6 +577,17 @@ impl Stmt { } } +#[derive(Clone)] +#[cfg(feature = "internals")] +pub struct CustomExpr(pub StaticVec, pub Shared); + +#[cfg(feature = "internals")] +impl fmt::Debug for CustomExpr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.0, f) + } +} + /// An expression. /// /// Each variant is at most one pointer in size (for speed), @@ -632,6 +652,9 @@ pub enum Expr { False(Position), /// () Unit(Position), + /// Custom syntax + #[cfg(feature = "internals")] + Custom(Box<(CustomExpr, Position)>), } impl Default for Expr { @@ -726,6 +749,9 @@ impl Expr { Self::True(pos) | Self::False(pos) | Self::Unit(pos) => *pos, Self::Dot(x) | Self::Index(x) => x.0.position(), + + #[cfg(feature = "internals")] + Self::Custom(x) => x.1, } } @@ -758,6 +784,9 @@ impl Expr { Self::Assignment(x) => x.3 = new_pos, Self::Dot(x) => x.2 = new_pos, Self::Index(x) => x.2 = new_pos, + + #[cfg(feature = "internals")] + Self::Custom(x) => x.1 = new_pos, } self @@ -861,6 +890,9 @@ impl Expr { Token::LeftParen => true, _ => false, }, + + #[cfg(feature = "internals")] + Self::Custom(_) => false, } } @@ -876,6 +908,14 @@ impl Expr { _ => self, } } + + #[cfg(feature = "internals")] + pub fn get_variable_name(&self) -> Option<&str> { + match self { + Self::Variable(x) => Some((x.0).0.as_str()), + _ => None, + } + } } /// Consume a particular token, checking that it is the expected one. @@ -2024,6 +2064,80 @@ fn parse_expr( settings.pos = input.peek().unwrap().1; settings.ensure_level_within_max_limit(state.max_expr_depth)?; + // Check if it is a custom syntax. + #[cfg(feature = "internals")] + if let Some(ref custom) = state.engine.custom_syntax { + let (token, pos) = input.peek().unwrap(); + let token_pos = *pos; + + match token { + Token::Custom(key) if custom.contains_key(key) => { + let custom = custom.get_key_value(key).unwrap(); + let (key, syntax) = custom; + + input.next().unwrap(); + + let mut exprs: StaticVec = Default::default(); + + // Adjust the variables stack + match syntax.scope_delta { + delta if delta > 0 => { + state.stack.push(("".to_string(), ScopeEntryType::Normal)) + } + delta if delta < 0 && state.stack.len() <= delta.abs() as usize => { + state.stack.clear() + } + delta if delta < 0 => state + .stack + .truncate(state.stack.len() - delta.abs() as usize), + _ => (), + } + + for segment in syntax.segments.iter() { + settings.pos = input.peek().unwrap().1; + let settings = settings.level_up(); + + match segment.as_str() { + MARKER_IDENT => match input.next().unwrap() { + (Token::Identifier(s), pos) => { + exprs.push(Expr::Variable(Box::new(((s, pos), None, 0, None)))); + } + (_, pos) => return Err(PERR::VariableExpected.into_err(pos)), + }, + MARKER_EXPR => exprs.push(parse_expr(input, state, lib, settings)?), + MARKER_BLOCK => { + let stmt = parse_block(input, state, lib, settings)?; + let pos = stmt.position(); + exprs.push(Expr::Stmt(Box::new((stmt, pos)))) + } + s => match input.peek().unwrap() { + (Token::Custom(custom), _) if custom == s => { + input.next().unwrap(); + } + (t, _) if t.syntax().as_ref() == s => { + input.next().unwrap(); + } + (_, pos) => { + return Err(PERR::MissingToken( + s.to_string(), + format!("for '{}' expression", key), + ) + .into_err(*pos)) + } + }, + } + } + + return Ok(Expr::Custom(Box::new(( + CustomExpr(exprs, syntax.func.clone()), + token_pos, + )))); + } + _ => (), + } + } + + // Parse expression normally. let lhs = parse_unary(input, state, lib, settings.level_up())?; parse_binary_op(input, state, lib, 1, lhs, settings.level_up()) } @@ -2297,7 +2411,7 @@ fn parse_import( fn parse_export( input: &mut TokenStream, state: &mut ParseState, - lib: &mut FunctionsLib, + _lib: &mut FunctionsLib, mut settings: ParseSettings, ) -> Result { settings.pos = eat_token(input, Token::Export); diff --git a/src/syntax.rs b/src/syntax.rs new file mode 100644 index 00000000..2fea999e --- /dev/null +++ b/src/syntax.rs @@ -0,0 +1,151 @@ +//! Module containing implementation for custom syntax. +#![cfg(feature = "internals")] + +use crate::any::Dynamic; +use crate::engine::{Engine, Imports, State, MARKER_BLOCK, MARKER_EXPR, MARKER_IDENT}; +use crate::error::LexError; +use crate::fn_native::{SendSync, Shared}; +use crate::module::Module; +use crate::parser::Expr; +use crate::result::EvalAltResult; +use crate::scope::Scope; +use crate::token::{is_valid_identifier, Token}; +use crate::utils::StaticVec; + +use crate::stdlib::{ + fmt, + rc::Rc, + string::{String, ToString}, + sync::Arc, +}; + +/// A general function trail object. +#[cfg(not(feature = "sync"))] +pub type FnCustomSyntaxEval = dyn Fn( + &Engine, + &mut Scope, + &mut Imports, + &mut State, + &Module, + &mut Option<&mut Dynamic>, + &[Expr], + usize, +) -> Result>; +/// A general function trail object. +#[cfg(feature = "sync")] +pub type FnCustomSyntaxEval = dyn Fn( + &Engine, + &mut Scope, + &mut Imports, + &mut State, + &Module, + &mut Option<&mut Dynamic>, + &[Expr], + usize, + ) -> Result> + + Send + + Sync; + +#[derive(Clone)] +pub struct CustomSyntax { + pub segments: StaticVec, + pub func: Shared, + pub scope_delta: isize, +} + +impl fmt::Debug for CustomSyntax { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.segments, f) + } +} + +impl Engine { + pub fn register_custom_syntax + ToString>( + &mut self, + value: &[S], + scope_delta: isize, + func: impl Fn( + &Engine, + &mut Scope, + &mut Imports, + &mut State, + &Module, + &mut Option<&mut Dynamic>, + &[Expr], + usize, + ) -> Result> + + SendSync + + 'static, + ) -> Result<(), Box> { + if value.is_empty() { + return Err(Box::new(LexError::ImproperSymbol("".to_string()))); + } + + let mut segments: StaticVec<_> = Default::default(); + + for s in value { + let seg = match s.as_ref() { + // Markers not in first position + MARKER_EXPR | MARKER_BLOCK | MARKER_IDENT if !segments.is_empty() => s.to_string(), + // Standard symbols not in first position + s if !segments.is_empty() && Token::lookup_from_syntax(s).is_some() => { + if self + .disabled_symbols + .as_ref() + .map(|d| d.contains(s)) + .unwrap_or(false) + { + // If symbol is disabled, make it a custom keyword + if self.custom_keywords.is_none() { + self.custom_keywords = Some(Default::default()); + } + + if !self.custom_keywords.as_ref().unwrap().contains_key(s) { + self.custom_keywords.as_mut().unwrap().insert(s.into(), 0); + } + } + + s.into() + } + // Identifier + s if is_valid_identifier(s.chars()) => { + if self.custom_keywords.is_none() { + self.custom_keywords = Some(Default::default()); + } + + if !self.custom_keywords.as_ref().unwrap().contains_key(s) { + self.custom_keywords.as_mut().unwrap().insert(s.into(), 0); + } + + s.into() + } + // Anything else is an error + _ => return Err(Box::new(LexError::ImproperSymbol(s.to_string()))), + }; + + segments.push(seg); + } + + let key = segments.remove(0); + + let syntax = CustomSyntax { + segments, + #[cfg(not(feature = "sync"))] + func: Rc::new(func), + #[cfg(feature = "sync")] + func: Arc::new(func), + scope_delta, + }; + + if self.custom_syntax.is_none() { + self.custom_syntax = Some(Default::default()); + } + + self.custom_syntax + .as_mut() + .unwrap() + .insert(key, syntax.into()); + + Ok(()) + } +} diff --git a/src/token.rs b/src/token.rs index fb02f14c..460ddcb5 100644 --- a/src/token.rs +++ b/src/token.rs @@ -312,6 +312,87 @@ impl Token { } } + /// Reverse lookup a token from a piece of syntax. + pub fn lookup_from_syntax(syntax: &str) -> Option { + use Token::*; + + Some(match syntax { + "{" => LeftBrace, + "}" => RightBrace, + "(" => LeftParen, + ")" => RightParen, + "[" => LeftBracket, + "]" => RightBracket, + "+" => Plus, + "-" => Minus, + "*" => Multiply, + "/" => Divide, + ";" => SemiColon, + ":" => Colon, + "::" => DoubleColon, + "," => Comma, + "." => Period, + "#{" => MapStart, + "=" => Equals, + "true" => True, + "false" => False, + "let" => Let, + "const" => Const, + "if" => If, + "else" => Else, + "while" => While, + "loop" => Loop, + "for" => For, + "in" => In, + "<" => LessThan, + ">" => GreaterThan, + "!" => Bang, + "<=" => LessThanEqualsTo, + ">=" => GreaterThanEqualsTo, + "==" => EqualsTo, + "!=" => NotEqualsTo, + "|" => Pipe, + "||" => Or, + "&" => Ampersand, + "&&" => And, + #[cfg(not(feature = "no_function"))] + "fn" => Fn, + "continue" => Continue, + "break" => Break, + "return" => Return, + "throw" => Throw, + "+=" => PlusAssign, + "-=" => MinusAssign, + "*=" => MultiplyAssign, + "/=" => DivideAssign, + "<<=" => LeftShiftAssign, + ">>=" => RightShiftAssign, + "&=" => AndAssign, + "|=" => OrAssign, + "^=" => XOrAssign, + "<<" => LeftShift, + ">>" => RightShift, + "^" => XOr, + "%" => Modulo, + "%=" => ModuloAssign, + "~" => PowerOf, + "~=" => PowerOfAssign, + #[cfg(not(feature = "no_function"))] + "private" => Private, + #[cfg(not(feature = "no_module"))] + "import" => Import, + #[cfg(not(feature = "no_module"))] + "export" => Export, + #[cfg(not(feature = "no_module"))] + "as" => As, + "===" | "!==" | "->" | "<-" | "=>" | ":=" | "::<" | "(*" | "*)" | "#" => { + Reserved(syntax.into()) + } + + _ => return None, + }) + } + // Is this token EOF? pub fn is_eof(&self) -> bool { use Token::*; @@ -450,7 +531,7 @@ impl Token { } } - /// Is this token a keyword? + /// Is this token a standard keyword? pub fn is_keyword(&self) -> bool { use Token::*; @@ -467,6 +548,22 @@ impl Token { _ => false, } } + + /// Is this token a reserved keyword? + pub fn is_reserved(&self) -> bool { + match self { + Self::Reserved(_) => true, + _ => false, + } + } + + /// Is this token a custom keyword? + pub fn is_custom(&self) -> bool { + match self { + Self::Custom(_) => true, + _ => false, + } + } } impl From for String { @@ -628,9 +725,9 @@ pub fn parse_string_literal( } /// Consume the next character. -fn eat_next(stream: &mut impl InputStream, pos: &mut Position) { - stream.get_next(); +fn eat_next(stream: &mut impl InputStream, pos: &mut Position) -> Option { pos.advance(); + stream.get_next() } /// Scan for a block comment until the end. @@ -858,35 +955,8 @@ fn get_next_token_inner( } return Some(( - match identifier.as_str() { - "true" => Token::True, - "false" => Token::False, - "let" => Token::Let, - "const" => Token::Const, - "if" => Token::If, - "else" => Token::Else, - "while" => Token::While, - "loop" => Token::Loop, - "continue" => Token::Continue, - "break" => Token::Break, - "return" => Token::Return, - "throw" => Token::Throw, - "for" => Token::For, - "in" => Token::In, - #[cfg(not(feature = "no_function"))] - "private" => Token::Private, - #[cfg(not(feature = "no_module"))] - "import" => Token::Import, - #[cfg(not(feature = "no_module"))] - "export" => Token::Export, - #[cfg(not(feature = "no_module"))] - "as" => Token::As, - - #[cfg(not(feature = "no_function"))] - "fn" => Token::Fn, - - _ => Token::Identifier(identifier), - }, + Token::lookup_from_syntax(&identifier) + .unwrap_or_else(|| Token::Identifier(identifier)), start_pos, )); } @@ -933,7 +1003,10 @@ fn get_next_token_inner( ('}', _) => return Some((Token::RightBrace, start_pos)), // Parentheses - ('(', '*') => return Some((Token::Reserved("(*".into()), start_pos)), + ('(', '*') => { + eat_next(stream, pos); + return Some((Token::Reserved("(*".into()), start_pos)); + } ('(', _) => return Some((Token::LeftParen, start_pos)), (')', _) => return Some((Token::RightParen, start_pos)), @@ -947,6 +1020,7 @@ fn get_next_token_inner( eat_next(stream, pos); return Some((Token::MapStart, start_pos)); } + ('#', _) => return Some((Token::Reserved("#".into()), start_pos)), // Operators ('+', '=') => { @@ -962,11 +1036,17 @@ fn get_next_token_inner( eat_next(stream, pos); return Some((Token::MinusAssign, start_pos)); } - ('-', '>') => return Some((Token::Reserved("->".into()), start_pos)), + ('-', '>') => { + eat_next(stream, pos); + return Some((Token::Reserved("->".into()), start_pos)); + } ('-', _) if !state.non_unary => return Some((Token::UnaryMinus, start_pos)), ('-', _) => return Some((Token::Minus, start_pos)), - ('*', ')') => return Some((Token::Reserved("*)".into()), start_pos)), + ('*', ')') => { + eat_next(stream, pos); + return Some((Token::Reserved("*)".into()), start_pos)); + } ('*', '=') => { eat_next(stream, pos); return Some((Token::MultiplyAssign, start_pos)); @@ -1031,31 +1111,42 @@ fn get_next_token_inner( // Warn against `===` if stream.peek_next() == Some('=') { + eat_next(stream, pos); return Some((Token::Reserved("===".into()), start_pos)); } return Some((Token::EqualsTo, start_pos)); } - ('=', '>') => return Some((Token::Reserved("=>".into()), start_pos)), + ('=', '>') => { + eat_next(stream, pos); + return Some((Token::Reserved("=>".into()), start_pos)); + } ('=', _) => return Some((Token::Equals, start_pos)), (':', ':') => { eat_next(stream, pos); if stream.peek_next() == Some('<') { + eat_next(stream, pos); return Some((Token::Reserved("::<".into()), start_pos)); } return Some((Token::DoubleColon, start_pos)); } - (':', '=') => return Some((Token::Reserved(":=".into()), start_pos)), + (':', '=') => { + eat_next(stream, pos); + return Some((Token::Reserved(":=".into()), start_pos)); + } (':', _) => return Some((Token::Colon, start_pos)), ('<', '=') => { eat_next(stream, pos); return Some((Token::LessThanEqualsTo, start_pos)); } - ('<', '-') => return Some((Token::Reserved("<-".into()), start_pos)), + ('<', '-') => { + eat_next(stream, pos); + return Some((Token::Reserved("<-".into()), start_pos)); + } ('<', '<') => { eat_next(stream, pos); @@ -1094,6 +1185,7 @@ fn get_next_token_inner( eat_next(stream, pos); if stream.peek_next() == Some('=') { + eat_next(stream, pos); return Some((Token::Reserved("!==".into()), start_pos)); } @@ -1163,40 +1255,42 @@ fn get_next_token_inner( } /// A type that implements the `InputStream` trait. -/// Multiple charaacter streams are jointed together to form one single stream. +/// Multiple character streams are jointed together to form one single stream. pub struct MultiInputsStream<'a> { /// The input character streams. streams: StaticVec>>, + /// The current stream index. + index: usize, } impl InputStream for MultiInputsStream<'_> { /// Get the next character fn get_next(&mut self) -> Option { loop { - if self.streams.is_empty() { + if self.index >= self.streams.len() { // No more streams return None; - } else if let Some(ch) = self.streams[0].next() { + } else if let Some(ch) = self.streams[self.index].next() { // Next character in current stream return Some(ch); } else { // Jump to the next stream - let _ = self.streams.remove(0); + self.index += 1; } } } /// Peek the next character fn peek_next(&mut self) -> Option { loop { - if self.streams.is_empty() { + if self.index >= self.streams.len() { // No more streams return None; - } else if let Some(ch) = self.streams[0].peek() { + } else if let Some(&ch) = self.streams[self.index].peek() { // Next character in current stream - return Some(*ch); + return Some(ch); } else { // Jump to the next stream - let _ = self.streams.remove(0); + self.index += 1; } } } @@ -1252,7 +1346,11 @@ impl<'a> Iterator for TokenIterator<'a, '_> { .to_string(), ))), "(*" | "*)" => Token::LexError(Box::new(LERR::ImproperSymbol( - "'(* .. *)' is not a valid comment style. This is not Pascal! Should it be '/* .. */'?" + "'(* .. *)' is not a valid comment format. This is not Pascal! Should it be '/* .. */'?" + .to_string(), + ))), + "#" => Token::LexError(Box::new(LERR::ImproperSymbol( + "'#' is not a valid symbol. Should it be '#{'?" .to_string(), ))), token => Token::LexError(Box::new(LERR::ImproperSymbol( @@ -1260,6 +1358,17 @@ impl<'a> Iterator for TokenIterator<'a, '_> { ))), }, pos)), (r @ Some(_), None, None) => r, + (Some((Token::Identifier(s), pos)), _, Some(custom)) if custom.contains_key(&s) => { + // Convert custom keywords + Some((Token::Custom(s), pos)) + } + (Some((token, pos)), _, Some(custom)) + if (token.is_keyword() || token.is_operator() || token.is_reserved()) + && custom.contains_key(token.syntax().as_ref()) => + { + // Convert into custom keywords + Some((Token::Custom(token.syntax().into()), pos)) + } (Some((token, pos)), Some(disabled), _) if token.is_operator() && disabled.contains(token.syntax().as_ref()) => { @@ -1275,10 +1384,6 @@ impl<'a> Iterator for TokenIterator<'a, '_> { // Convert disallowed keywords into identifiers Some((Token::Identifier(token.syntax().into()), pos)) } - (Some((Token::Identifier(s), pos)), _, Some(custom)) if custom.contains_key(&s) => { - // Convert custom keywords - Some((Token::Custom(s), pos)) - } (r, _, _) => r, } } @@ -1298,6 +1403,7 @@ pub fn lex<'a, 'e>(input: &'a [&'a str], engine: &'e Engine) -> TokenIterator<'a pos: Position::new(1, 0), stream: MultiInputsStream { streams: input.iter().map(|s| s.chars().peekable()).collect(), + index: 0, }, } } diff --git a/src/utils.rs b/src/utils.rs index 77cef0e7..5dd619ba 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -356,10 +356,13 @@ impl StaticVec { panic!("nothing to pop!"); } - let result = if self.is_fixed_storage() { - self.extract_from_list(self.len - 1) + if self.is_fixed_storage() { + let value = self.extract_from_list(self.len - 1); + self.len -= 1; + value } else { let value = self.more.pop().unwrap(); + self.len -= 1; // Move back to the fixed list if self.more.len() == MAX_STATIC_VEC { @@ -370,11 +373,7 @@ impl StaticVec { } value - }; - - self.len -= 1; - - result + } } /// Remove a value from this `StaticVec` at a particular position. /// @@ -386,18 +385,20 @@ impl StaticVec { panic!("index OOB in StaticVec"); } - let result = if self.is_fixed_storage() { + if self.is_fixed_storage() { let value = self.extract_from_list(index); // Move all items one slot to the left - for x in index..self.len - 1 { - let orig_value = self.extract_from_list(x + 1); - self.set_into_list(x, orig_value, false); + for x in index + 1..self.len - 1 { + let orig_value = self.extract_from_list(x); + self.set_into_list(x - 1, orig_value, false); } + self.len -= 1; value } else { let value = self.more.remove(index); + self.len -= 1; // Move back to the fixed list if self.more.len() == MAX_STATIC_VEC { @@ -408,11 +409,7 @@ impl StaticVec { } value - }; - - self.len -= 1; - - result + } } /// Get the number of items in this `StaticVec`. #[inline(always)] diff --git a/tests/maps.rs b/tests/maps.rs index bc2b24a9..c57deaa6 100644 --- a/tests/maps.rs +++ b/tests/maps.rs @@ -21,7 +21,7 @@ fn test_map_indexing() -> Result<(), Box> { r#" let y = #{d: 1, "e": #{a: 42, b: 88, "": "hello"}, " 123 xyz": 9}; y.e[""][4] - "# + "# )?, 'o' ); @@ -47,7 +47,7 @@ fn test_map_indexing() -> Result<(), Box> { let x = #{a: 1, b: 2, c: 3}; let c = x.remove("c"); x.len() + c - "# + "# )?, 5 ); @@ -58,7 +58,7 @@ fn test_map_indexing() -> Result<(), Box> { let y = #{b: 42, d: 9}; x.mixin(y); x.len() + x.b - " + " )?, 46 ); @@ -68,7 +68,7 @@ fn test_map_indexing() -> Result<(), Box> { let x = #{a: 1, b: 2, c: 3}; x += #{b: 42, d: 9}; x.len() + x.b - " + " )?, 46 ); @@ -79,7 +79,7 @@ fn test_map_indexing() -> Result<(), Box> { let x = #{a: 1, b: 2, c: 3}; let y = #{b: 42, d: 9}; x + y - " + " )? .len(), 4 @@ -94,27 +94,9 @@ fn test_map_assign() -> Result<(), Box> { let x = engine.eval::(r#"let x = #{a: 1, b: true, "c$": "hello"}; x"#)?; - assert_eq!( - x.get("a") - .cloned() - .expect("should have property a") - .cast::(), - 1 - ); - assert_eq!( - x.get("b") - .cloned() - .expect("should have property b") - .cast::(), - true - ); - assert_eq!( - x.get("c$") - .cloned() - .expect("should have property c$") - .cast::(), - "hello" - ); + assert_eq!(x["a"].clone().cast::(), 1); + assert_eq!(x["b"].clone().cast::(), true); + assert_eq!(x["c$"].clone().cast::(), "hello"); Ok(()) } @@ -125,27 +107,9 @@ fn test_map_return() -> Result<(), Box> { let x = engine.eval::(r#"#{a: 1, b: true, "c$": "hello"}"#)?; - assert_eq!( - x.get("a") - .cloned() - .expect("should have property a") - .cast::(), - 1 - ); - assert_eq!( - x.get("b") - .cloned() - .expect("should have property b") - .cast::(), - true - ); - assert_eq!( - x.get("c$") - .cloned() - .expect("should have property c$") - .cast::(), - "hello" - ); + assert_eq!(x["a"].clone().cast::(), 1); + assert_eq!(x["b"].clone().cast::(), true); + assert_eq!(x["c$"].clone().cast::(), "hello"); Ok(()) } @@ -167,7 +131,7 @@ fn test_map_for() -> Result<(), Box> { } s - "# + "# )? .len(), 11 @@ -188,41 +152,11 @@ fn test_map_json() -> Result<(), Box> { assert!(!map.contains_key("x")); - assert_eq!( - map.get("a") - .cloned() - .expect("should have property a") - .cast::(), - 1 - ); - assert_eq!( - map.get("b") - .cloned() - .expect("should have property b") - .cast::(), - true - ); - assert_eq!( - map.get("c") - .cloned() - .expect("should have property a") - .cast::(), - 42 - ); - assert_eq!( - map.get("$d e f!") - .cloned() - .expect("should have property $d e f!") - .cast::(), - "hello" - ); - assert_eq!( - map.get("z") - .cloned() - .expect("should have property z") - .cast::<()>(), - () - ); + assert_eq!(map["a"].clone().cast::(), 1); + assert_eq!(map["b"].clone().cast::(), true); + assert_eq!(map["c"].clone().cast::(), 42); + assert_eq!(map["$d e f!"].clone().cast::(), "hello"); + assert_eq!(map["z"].clone().cast::<()>(), ()); #[cfg(not(feature = "no_index"))] { @@ -241,7 +175,7 @@ fn test_map_json() -> Result<(), Box> { } s - "# + "# )? .len(), 11 @@ -265,7 +199,7 @@ fn test_map_oop() -> Result<(), Box> { obj.action(2); obj.data - "#, + "#, )?, 42 ); diff --git a/tests/syntax.rs b/tests/syntax.rs new file mode 100644 index 00000000..501c90b3 --- /dev/null +++ b/tests/syntax.rs @@ -0,0 +1,77 @@ +#![cfg(feature = "internals")] +use rhai::{ + Dynamic, Engine, EvalAltResult, EvalState, Expr, Imports, LexError, Module, Scope, INT, +}; + +#[test] +fn test_custom_syntax() -> Result<(), Box> { + let mut engine = Engine::new(); + + // Disable 'while' and make sure it still works with custom syntax + engine.disable_symbol("while"); + engine.consume("while false {}").expect_err("should error"); + engine.consume("let while = 0")?; + + engine + .register_custom_syntax( + &[ + "do", "|", "$ident$", "|", "->", "$block$", "while", "$expr$", + ], + 1, + |engine: &Engine, + scope: &mut Scope, + mods: &mut Imports, + state: &mut EvalState, + lib: &Module, + this_ptr: &mut Option<&mut Dynamic>, + inputs: &[Expr], + level: usize| { + let var_name = inputs[0].get_variable_name().unwrap().to_string(); + let stmt = inputs.get(1).unwrap(); + let expr = inputs.get(2).unwrap(); + + scope.push(var_name, 0 as INT); + + loop { + engine.eval_expression_tree(scope, mods, state, lib, this_ptr, stmt, level)?; + + if !engine + .eval_expression_tree(scope, mods, state, lib, this_ptr, expr, level)? + .as_bool() + .map_err(|_| { + EvalAltResult::ErrorBooleanArgMismatch( + "do-while".into(), + expr.position(), + ) + })? + { + break; + } + } + + Ok(().into()) + }, + ) + .unwrap(); + + // 'while' is now a custom keyword so this it can no longer be a variable + engine.consume("let while = 0").expect_err("should error"); + + assert_eq!( + engine.eval::( + r" + do |x| -> { x += 1 } while x < 42; + x + " + )?, + 42 + ); + + // The first symbol must be an identifier + assert!(matches!( + *engine.register_custom_syntax(&["!"], 0, |_, _, _, _, _, _, _, _| Ok(().into())).expect_err("should error"), + LexError::ImproperSymbol(s) if s == "!" + )); + + Ok(()) +}