diff --git a/RELEASES.md b/RELEASES.md index 22aae65d..520a2bf1 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -6,24 +6,32 @@ Version 0.18.0 This version adds: -* Anonymous functions (in closure syntax). Simplifies creation of ad hoc functions. +* Binding the `this` pointer in a function pointer `call`. +* Anonymous functions (in Rust closure syntax). Simplifies creation of single-use ad-hoc functions. * Currying of function pointers. New features ------------ * `call` can now be called function-call style for function pointers - this is to handle builds with `no_object`. -* Disallow many keywords as variables, such as `print`, `eval`, `call`, `this` etc. +* Reserve language keywords, such as `print`, `eval`, `call`, `this` etc. * `x.call(f, ...)` allows binding `x` to `this` for the function referenced by the function pointer `f`. -* Anonymous functions in the syntax of a closure, e.g. `|x, y, z| x + y - z`. +* Anonymous functions are supported in the syntax of a Rust closure, e.g. `|x, y, z| x + y - z`. * Custom syntax now works even without the `internals` feature. -* Currying of function pointers is supported via the `curry` keyword. +* Currying of function pointers is supported via the new `curry` keyword. * `Module::set_indexer_get_set_fn` is added as a shorthand of both `Module::set_indexer_get_fn` and `Module::set_indexer_set_fn`. Breaking changes ---------------- +* Language keywords are now _reserved_ (even when disabled) and they can no longer be used as variable names. * Function signature for defining custom syntax is simplified. +* `Engine::register_raw_fn_XXX` API shortcuts are removed. + +Housekeeping +------------ + +* Most compilation warnings are eliminated via feature gates. Version 0.17.0 @@ -57,7 +65,7 @@ New features * `Engine::disable_symbol` to surgically disable keywords and/or operators. * `Engine::register_custom_operator` to define a custom operator. * `Engine::register_custom_syntax` to define a custom syntax. -* New low-level API `Engine::register_raw_fn` and `Engine::register_raw_fn_XXX`. +* New low-level API `Engine::register_raw_fn`. * New low-level API `Module::set_raw_fn` mirroring `Engine::register_raw_fn`. * `AST::clone_functions_only`, `AST::clone_functions_only_filtered` and `AST::clone_statements_only` to clone only part of an `AST`. * The boolean `^` (XOR) operator is added. diff --git a/doc/src/appendix/keywords.md b/doc/src/appendix/keywords.md index 7c8dbaff..f7c1be73 100644 --- a/doc/src/appendix/keywords.md +++ b/doc/src/appendix/keywords.md @@ -3,32 +3,66 @@ Keywords List {{#include ../links.md}} -| Keyword | Description | Not available under | -| :-------------------: | ---------------------------------------- | :-----------------: | -| `true` | Boolean true literal | | -| `false` | Boolean false literal | | -| `let` | Variable declaration | | -| `const` | Constant declaration | | -| `if` | If statement | | -| `else` | else block of if statement | | -| `while` | While loop | | -| `loop` | Infinite loop | | -| `for` | For loop | | -| `in` | Containment test, part of for loop | | -| `continue` | Continue a loop at the next iteration | | -| `break` | Loop breaking | | -| `return` | Return value | | -| `throw` | Throw exception | | -| `import` | Import module | [`no_module`] | -| `export` | Export variable | [`no_module`] | -| `as` | Alias for variable export | [`no_module`] | -| `private` | Mark function private | [`no_function`] | -| `fn` (lower-case `f`) | Function definition | [`no_function`] | -| `Fn` (capital `F`) | Function to create a [function pointer] | | -| `call` | Call a [function pointer] | | -| `curry` | Curry a [function pointer] | | -| `this` | Reference to base object for method call | [`no_function`] | -| `type_of` | Get type name of value | | -| `print` | Print value | | -| `debug` | Print value in debug format | | -| `eval` | Evaluate script | | +| Keyword | Description | Inactive under | +| :-------------------: | ---------------------------------------- | :-------------: | +| `true` | Boolean true literal | | +| `false` | Boolean false literal | | +| `let` | Variable declaration | | +| `const` | Constant declaration | | +| `if` | If statement | | +| `else` | else block of if statement | | +| `while` | While loop | | +| `loop` | Infinite loop | | +| `for` | For loop | | +| `in` | Containment test, part of for loop | | +| `continue` | Continue a loop at the next iteration | | +| `break` | Loop breaking | | +| `return` | Return value | | +| `throw` | Throw exception | | +| `import` | Import module | [`no_module`] | +| `export` | Export variable | [`no_module`] | +| `as` | Alias for variable export | [`no_module`] | +| `private` | Mark function private | [`no_function`] | +| `fn` (lower-case `f`) | Function definition | [`no_function`] | +| `Fn` (capital `F`) | Function to create a [function pointer] | | +| `call` | Call a [function pointer] | | +| `curry` | Curry a [function pointer] | | +| `this` | Reference to base object for method call | [`no_function`] | +| `type_of` | Get type name of value | | +| `print` | Print value | | +| `debug` | Print value in debug format | | +| `eval` | Evaluate script | | + + +Reserved Keywords +----------------- + +| Keyword | Potential usage | +| --------- | --------------------- | +| `var` | Variable declaration | +| `static` | Variable declaration | +| `do` | Looping | +| `each` | Looping | +| `then` | Control flow | +| `goto` | Control flow | +| `exit` | Control flow | +| `switch` | Matching | +| `match` | Matching | +| `case` | Matching | +| `public` | Function/field access | +| `new` | Constructor | +| `try` | Trap exception | +| `catch` | Catch exception | +| `use` | Import namespace | +| `with` | Scope | +| `module` | Module | +| `package` | Package | +| `spawn` | Threading | +| `go` | Threading | +| `await` | Async | +| `async` | Async | +| `yield` | Async | +| `default` | Special value | +| `void` | Special value | +| `null` | Special value | +| `nil` | Special value | diff --git a/doc/src/engine/custom-syntax.md b/doc/src/engine/custom-syntax.md index 89fdf792..8f308106 100644 --- a/doc/src/engine/custom-syntax.md +++ b/doc/src/engine/custom-syntax.md @@ -114,10 +114,7 @@ Any custom syntax must include an _implementation_ of it. The function signature of an implementation is: -```rust -Fn(engine: &Engine, context: &mut EvalContext, scope: &mut Scope, inputs: &[Expression]) - -> Result> -``` +> `Fn(engine: &Engine, context: &mut EvalContext, scope: &mut Scope, inputs: &[Expression]) -> Result>` where: diff --git a/doc/src/language/keywords.md b/doc/src/language/keywords.md index 836e9b0c..851b7907 100644 --- a/doc/src/language/keywords.md +++ b/doc/src/language/keywords.md @@ -5,18 +5,21 @@ Keywords The following are reserved keywords in Rhai: -| Keywords | Usage | Not available under feature | -| ------------------------------------------------- | --------------------- | :-------------------------: | -| `true`, `false` | Boolean constants | | -| `let`, `const` | Variable declarations | | -| `if`, `else` | Control flow | | -| `while`, `loop`, `for`, `in`, `continue`, `break` | Looping | | -| `fn`, `private` | Functions | [`no_function`] | -| `return` | Return values | | -| `throw` | throw exceptions | | -| `import`, `export`, `as` | Modules | [`no_module`] | -| `Fn`, `call` | Function pointers | | -| `type_of`, `print`, `debug`, `eval` | Special functions | | +| Active keywords | Reserved keywords | Usage | Inactive under feature | +| ------------------------------------------------- | ---------------------------------------- | --------------------- | :--------------------: | +| `true`, `false` | | Boolean constants | | +| `let`, `const` | `var`, `static` | Variable declarations | | +| `if`, `else` | `then`, `goto`, `exit` | Control flow | | +| | `switch`, `match`, `case` | Matching | | +| `while`, `loop`, `for`, `in`, `continue`, `break` | `do`, `each` | Looping | | +| `fn`, `private` | `public`, `new` | Functions | [`no_function`] | +| `return` | | Return values | | +| `throw` | `try`, `catch` | Throw exceptions | | +| `import`, `export`, `as` | `use`, `with`, `module`, `package` | Modules/packages | [`no_module`] | +| `Fn`, `call`, `curry` | | Function pointers | | +| | `spawn`, `go`, `async`, `await`, `yield` | Threading/async | | +| `type_of`, `print`, `debug`, `eval` | | Special functions | | +| | `default`, `void`, `null`, `nil` | Special values | | + +Keywords cannot become the name of a [function] or [variable], even when they are disabled. -Keywords cannot be the name of a [function] or [variable], unless the relevant feature is enabled. -For example, `fn` is a valid variable name under [`no_function`]. diff --git a/doc/src/rust/register-raw.md b/doc/src/rust/register-raw.md index 6d3f22c5..76866249 100644 --- a/doc/src/rust/register-raw.md +++ b/doc/src/rust/register-raw.md @@ -55,27 +55,12 @@ engine.register_fn("increment_by", |x: &mut i64, y: i64| x += y); ``` -Shortcuts ---------- +Function Signature +------------------ -As usual with Rhai, there are shortcuts. For functions of zero to four parameters, which should be -the majority, use one of the `Engine::register_raw_fn_n` (where `n = 0..4`) methods: +The function signature passed to `Engine::register_raw_fn` takes the following form: -```rust -// Specify parameter types as generics -engine.register_raw_fn_2::( - "increment_by", - |engine: &Engine, lib: &Module, args: &mut [&mut Dynamic]| { ... } -); -``` - - -Closure Signature ------------------ - -The closure passed to `Engine::register_raw_fn` takes the following form: - -`Fn(engine: &Engine, lib: &Module, args: &mut [&mut Dynamic]) -> Result> + 'static` +> `Fn(engine: &Engine, lib: &Module, args: &mut [&mut Dynamic]) -> Result> + 'static` where: @@ -113,6 +98,12 @@ there can be no other immutable references to `args`, otherwise the Rust borrow Example - Passing a Function Pointer to a Rust Function ------------------------------------------------------ +The low-level API is useful when there is a need to interact with the scripting [`Engine`] within a function. + +The following example registers a function that takes a [function pointer] as an argument, +then calls it within the same [`Engine`]. This way, a _callback_ function can be provided +to a native Rust function. + ```rust use rhai::{Engine, Module, Dynamic, FnPtr}; @@ -133,11 +124,10 @@ engine.register_raw_fn( let value = args[2].clone(); // 3rd argument - function argument let this_ptr = args.get_mut(0).unwrap(); // 1st argument - this pointer - // Use 'call_fn_dynamic' to call the function name. - // Pass 'lib' as the current global library of functions. - engine.call_fn_dynamic(&mut Scope::new(), lib, fp.fn_name(), Some(this_ptr), [value])?; - - Ok(()) + // Use 'FnPtr::call_dynamic' to call the function pointer. + // Beware, only script-defined functions are supported by 'FnPtr::call_dynamic'. + // If it is a native Rust function, directly call it here in Rust instead! + fp.call_dynamic(engine, lib, Some(this_ptr), [value]) }, ); diff --git a/src/api.rs b/src/api.rs index b7a1bb74..42ff7037 100644 --- a/src/api.rs +++ b/src/api.rs @@ -19,6 +19,7 @@ use crate::engine::{FN_IDX_GET, FN_IDX_SET}; use crate::{ engine::{make_getter, make_setter, Map}, fn_register::RegisterFn, + token::Token, }; #[cfg(not(feature = "no_function"))] @@ -62,147 +63,6 @@ impl Engine { self } - /// Register a function of no parameters with the `Engine`. - /// - /// ## WARNING - Low Level API - /// - /// This function is very low level. - #[deprecated(note = "this function is volatile and may change")] - pub fn register_raw_fn_0( - &mut self, - name: &str, - func: impl Fn(&Engine, &Module, &mut [&mut Dynamic]) -> FuncReturn + SendSync + 'static, - ) -> &mut Self { - self.global_module.set_raw_fn(name, &[], func); - self - } - - /// Register a function of one parameter with the `Engine`. - /// - /// ## WARNING - Low Level API - /// - /// This function is very low level. - /// - /// Arguments are simply passed in as a mutable array of `&mut Dynamic`, - /// which is guaranteed to contain enough arguments of the correct types. - /// - /// To access a primary parameter value (i.e. cloning is cheap), use: `args[n].clone().cast::()` - /// - /// To access a parameter value and avoid cloning, use `std::mem::take(args[n]).cast::()`. - /// Notice that this will _consume_ the argument, replacing it with `()`. - /// - /// To access the first mutable parameter, use `args.get_mut(0).unwrap()` - #[deprecated(note = "this function is volatile and may change")] - pub fn register_raw_fn_1( - &mut self, - name: &str, - func: impl Fn(&Engine, &Module, &mut [&mut Dynamic]) -> FuncReturn + SendSync + 'static, - ) -> &mut Self { - self.global_module - .set_raw_fn(name, &[TypeId::of::()], func); - self - } - - /// Register a function of two parameters with the `Engine`. - /// - /// ## WARNING - Low Level API - /// - /// This function is very low level. - /// - /// Arguments are simply passed in as a mutable array of `&mut Dynamic`, - /// which is guaranteed to contain enough arguments of the correct types. - /// - /// To access a primary parameter value (i.e. cloning is cheap), use: `args[n].clone().cast::()` - /// - /// To access a parameter value and avoid cloning, use `std::mem::take(args[n]).cast::()`. - /// Notice that this will _consume_ the argument, replacing it with `()`. - /// - /// To access the first mutable parameter, use `args.get_mut(0).unwrap()` - #[deprecated(note = "this function is volatile and may change")] - pub fn register_raw_fn_2( - &mut self, - name: &str, - func: impl Fn(&Engine, &Module, &mut [&mut Dynamic]) -> FuncReturn + SendSync + 'static, - ) -> &mut Self { - self.global_module - .set_raw_fn(name, &[TypeId::of::(), TypeId::of::()], func); - self - } - - /// Register a function of three parameters with the `Engine`. - /// - /// ## WARNING - Low Level API - /// - /// This function is very low level. - /// - /// Arguments are simply passed in as a mutable array of `&mut Dynamic`, - /// which is guaranteed to contain enough arguments of the correct types. - /// - /// To access a primary parameter value (i.e. cloning is cheap), use: `args[n].clone().cast::()` - /// - /// To access a parameter value and avoid cloning, use `std::mem::take(args[n]).cast::()`. - /// Notice that this will _consume_ the argument, replacing it with `()`. - /// - /// To access the first mutable parameter, use `args.get_mut(0).unwrap()` - #[deprecated(note = "this function is volatile and may change")] - pub fn register_raw_fn_3< - A: Variant + Clone, - B: Variant + Clone, - C: Variant + Clone, - T: Variant + Clone, - >( - &mut self, - name: &str, - func: impl Fn(&Engine, &Module, &mut [&mut Dynamic]) -> FuncReturn + SendSync + 'static, - ) -> &mut Self { - self.global_module.set_raw_fn( - name, - &[TypeId::of::(), TypeId::of::(), TypeId::of::()], - func, - ); - self - } - - /// Register a function of four parameters with the `Engine`. - /// - /// ## WARNING - Low Level API - /// - /// This function is very low level. - /// - /// Arguments are simply passed in as a mutable array of `&mut Dynamic`, - /// which is guaranteed to contain enough arguments of the correct types. - /// - /// To access a primary parameter value (i.e. cloning is cheap), use: `args[n].clone().cast::()` - /// - /// To access a parameter value and avoid cloning, use `std::mem::take(args[n]).cast::()`. - /// Notice that this will _consume_ the argument, replacing it with `()`. - /// - /// To access the first mutable parameter, use `args.get_mut(0).unwrap()` - #[deprecated(note = "this function is volatile and may change")] - pub fn register_raw_fn_4< - A: Variant + Clone, - B: Variant + Clone, - C: Variant + Clone, - D: Variant + Clone, - T: Variant + Clone, - >( - &mut self, - name: &str, - func: impl Fn(&Engine, &Module, &mut [&mut Dynamic]) -> FuncReturn + SendSync + 'static, - ) -> &mut Self { - self.global_module.set_raw_fn( - name, - &[ - TypeId::of::(), - TypeId::of::(), - TypeId::of::(), - TypeId::of::(), - ], - func, - ); - self - } - /// Register a custom type for use with the `Engine`. /// The type must implement `Clone`. /// @@ -731,7 +591,7 @@ impl Engine { scripts: &[&str], optimization_level: OptimizationLevel, ) -> Result { - let stream = lex(scripts, self); + let stream = lex(scripts, None, self); self.parse(&mut stream.peekable(), scope, optimization_level) } @@ -856,7 +716,19 @@ impl Engine { // Trims the JSON string and add a '#' in front let scripts = ["#", json.trim()]; - let stream = lex(&scripts, self); + let stream = lex( + &scripts, + if has_null { + Some(Box::new(|token| match token { + // If `null` is present, make sure `null` is treated as a variable + Token::Reserved(s) if s == "null" => Token::Identifier(s), + _ => token, + })) + } else { + None + }, + self, + ); let ast = self.parse_global_expr(&mut stream.peekable(), &scope, OptimizationLevel::None)?; @@ -937,7 +809,7 @@ impl Engine { script: &str, ) -> Result { let scripts = [script]; - let stream = lex(&scripts, self); + let stream = lex(&scripts, None, self); { let mut peekable = stream.peekable(); self.parse_global_expr(&mut peekable, scope, self.optimization_level) @@ -1092,7 +964,7 @@ impl Engine { script: &str, ) -> Result> { let scripts = [script]; - let stream = lex(&scripts, self); + let stream = lex(&scripts, None, self); // No need to optimize a lone expression let ast = self.parse_global_expr(&mut stream.peekable(), scope, OptimizationLevel::None)?; @@ -1225,7 +1097,7 @@ impl Engine { script: &str, ) -> Result<(), Box> { let scripts = [script]; - let stream = lex(&scripts, self); + let stream = lex(&scripts, None, self); let ast = self.parse(&mut stream.peekable(), scope, self.optimization_level)?; self.consume_ast_with_scope(scope, &ast) } diff --git a/src/error.rs b/src/error.rs index b6901c68..fc92e5ea 100644 --- a/src/error.rs +++ b/src/error.rs @@ -172,7 +172,7 @@ impl ParseErrorType { Self::VariableExpected => "Expecting name of a variable", Self::Reserved(_) => "Invalid use of reserved keyword", Self::ExprExpected(_) => "Expecting an expression", - Self::FnMissingName => "Expecting name in function declaration", + Self::FnMissingName => "Expecting function name in function declaration", Self::FnMissingParams(_) => "Expecting parameters in function declaration", Self::FnDuplicatedParam(_,_) => "Duplicated parameters in function declaration", Self::FnMissingBody(_) => "Expecting body statement block for function declaration", diff --git a/src/packages/arithmetic.rs b/src/packages/arithmetic.rs index 61350996..fd95e675 100644 --- a/src/packages/arithmetic.rs +++ b/src/packages/arithmetic.rs @@ -18,6 +18,8 @@ use num_traits::{ CheckedShr, CheckedSub, }; +#[cfg(not(feature = "only_i32"))] +#[cfg(not(feature = "only_i64"))] use crate::stdlib::ops::{BitAnd, BitOr, BitXor}; #[cfg(any(feature = "unchecked", not(feature = "no_float")))] @@ -145,12 +147,18 @@ where } } // Bit operators +#[cfg(not(feature = "only_i32"))] +#[cfg(not(feature = "only_i64"))] fn binary_and(x: T, y: T) -> FuncReturn<::Output> { Ok(x & y) } +#[cfg(not(feature = "only_i32"))] +#[cfg(not(feature = "only_i64"))] fn binary_or(x: T, y: T) -> FuncReturn<::Output> { Ok(x | y) } +#[cfg(not(feature = "only_i32"))] +#[cfg(not(feature = "only_i64"))] fn binary_xor(x: T, y: T) -> FuncReturn<::Output> { Ok(x ^ y) } diff --git a/src/parser.rs b/src/parser.rs index baf1be85..51685a6c 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -9,7 +9,7 @@ use crate::module::{Module, ModuleRef}; use crate::optimize::{optimize_into_ast, OptimizationLevel}; use crate::scope::{EntryType as ScopeEntryType, Scope}; use crate::syntax::FnCustomSyntaxEval; -use crate::token::{is_valid_identifier, Position, Token, TokenStream}; +use crate::token::{is_keyword_function, is_valid_identifier, Position, Token, TokenStream}; use crate::utils::{StaticVec, StraightHasherBuilder}; #[cfg(not(feature = "no_function"))] @@ -1044,7 +1044,7 @@ fn parse_paren_expr( } /// Parse a function call. -fn parse_call_expr( +fn parse_fn_call( input: &mut TokenStream, state: &mut ParseState, lib: &mut FunctionsLib, @@ -1553,8 +1553,12 @@ fn parse_primary( Expr::Variable(Box::new(((s, settings.pos), None, 0, index))) } // Function call is allowed to have reserved keyword - Token::Reserved(s) if s != KEYWORD_THIS && input.peek().unwrap().0 == Token::LeftParen => { - Expr::Variable(Box::new(((s, settings.pos), None, 0, None))) + Token::Reserved(s) if input.peek().unwrap().0 == Token::LeftParen => { + if is_keyword_function(&s) { + Expr::Variable(Box::new(((s, settings.pos), None, 0, None))) + } else { + return Err(PERR::Reserved(s).into_err(settings.pos)); + } } // Access to `this` as a variable is OK Token::Reserved(s) if s == KEYWORD_THIS && input.peek().unwrap().0 != Token::LeftParen => { @@ -1601,7 +1605,7 @@ fn parse_primary( (Expr::Variable(x), Token::LeftParen) => { let ((name, pos), modules, _, _) = *x; settings.pos = pos; - parse_call_expr(input, state, lib, name, modules, settings.level_up())? + parse_fn_call(input, state, lib, name, modules, settings.level_up())? } (Expr::Property(_), _) => unreachable!(), // module access @@ -2882,14 +2886,12 @@ fn parse_fn( #[cfg(not(feature = "unchecked"))] settings.ensure_level_within_max_limit(state.max_expr_depth)?; - let name = match input.next().unwrap() { - (Token::Identifier(s), _) | (Token::Custom(s), _) | (Token::Reserved(s), _) - if s != KEYWORD_THIS && is_valid_identifier(s.chars()) => - { - s - } - (_, pos) => return Err(PERR::FnMissingName.into_err(pos)), - }; + let (token, pos) = input.next().unwrap(); + + let name = token.into_function_name().map_err(|t| match t { + Token::Reserved(s) => PERR::Reserved(s).into_err(pos), + _ => PERR::FnMissingName.into_err(pos), + })?; match input.peek().unwrap() { (Token::LeftParen, _) => eat_token(input, Token::LeftParen), diff --git a/src/token.rs b/src/token.rs index 775e78db..0aab8ce5 100644 --- a/src/token.rs +++ b/src/token.rs @@ -492,9 +492,12 @@ impl Token { #[cfg(feature = "no_module")] "import" | "export" | "as" => Reserved(syntax.into()), - "===" | "!==" | "->" | "<-" | "=>" | ":=" | "::<" | "(*" | "*)" | "#" => { - Reserved(syntax.into()) - } + "===" | "!==" | "->" | "<-" | "=>" | ":=" | "::<" | "(*" | "*)" | "#" | "public" + | "new" | "use" | "module" | "package" | "var" | "static" | "with" | "do" | "each" + | "then" | "goto" | "exit" | "switch" | "match" | "case" | "try" | "catch" + | "default" | "void" | "null" | "nil" | "spawn" | "go" | "async" | "await" + | "yield" => Reserved(syntax.into()), + KEYWORD_PRINT | KEYWORD_DEBUG | KEYWORD_TYPE_OF | KEYWORD_EVAL | KEYWORD_FN_PTR | KEYWORD_FN_PTR_CALL | KEYWORD_FN_PTR_CURRY | KEYWORD_THIS => Reserved(syntax.into()), @@ -666,6 +669,15 @@ impl Token { } } + /// Convert a token into a function name, if possible. + pub fn into_function_name(self) -> Result { + match self { + Self::Reserved(s) if is_keyword_function(&s) => Ok(s), + Self::Custom(s) | Self::Identifier(s) if is_valid_identifier(s.chars()) => Ok(s), + _ => Err(self), + } + } + /// Is this token a custom keyword? pub fn is_custom(&self) -> bool { match self { @@ -714,6 +726,16 @@ pub trait InputStream { fn peek_next(&mut self) -> Option; } +pub fn is_keyword_function(name: &str) -> bool { + name == KEYWORD_PRINT + || name == KEYWORD_DEBUG + || name == KEYWORD_TYPE_OF + || name == KEYWORD_EVAL + || name == KEYWORD_FN_PTR + || name == KEYWORD_FN_PTR_CALL + || name == KEYWORD_FN_PTR_CURRY +} + pub fn is_valid_identifier(name: impl Iterator) -> bool { let mut first_alphabetic = false; @@ -1452,13 +1474,15 @@ pub struct TokenIterator<'a, 'e> { pos: Position, /// Input character stream. stream: MultiInputsStream<'a>, + /// A processor function (if any) that maps a token to another. + map: Option Token>>, } impl<'a> Iterator for TokenIterator<'a, '_> { type Item = (Token, Position); fn next(&mut self) -> Option { - match ( + let token = match ( get_next_token(&mut self.stream, &mut self.state, &mut self.pos), self.engine.disabled_symbols.as_ref(), self.engine.custom_keywords.as_ref(), @@ -1540,12 +1564,27 @@ impl<'a> Iterator for TokenIterator<'a, '_> { Some((Token::Reserved(token.syntax().into()), pos)) } (r, _, _) => r, + }; + + match token { + None => None, + Some((token, pos)) => { + if let Some(ref map) = self.map { + Some((map(token), pos)) + } else { + Some((token, pos)) + } + } } } } /// Tokenize an input text stream. -pub fn lex<'a, 'e>(input: &'a [&'a str], engine: &'e Engine) -> TokenIterator<'a, 'e> { +pub fn lex<'a, 'e>( + input: &'a [&'a str], + map: Option Token>>, + engine: &'e Engine, +) -> TokenIterator<'a, 'e> { TokenIterator { engine, state: TokenizeState { @@ -1563,5 +1602,6 @@ pub fn lex<'a, 'e>(input: &'a [&'a str], engine: &'e Engine) -> TokenIterator<'a streams: input.iter().map(|s| s.chars().peekable()).collect(), index: 0, }, + map, } } diff --git a/tests/call_fn.rs b/tests/call_fn.rs index 9808ff70..3b93d520 100644 --- a/tests/call_fn.rs +++ b/tests/call_fn.rs @@ -131,15 +131,7 @@ fn test_fn_ptr_raw() -> Result<(), Box> { let value = args[2].clone(); let this_ptr = args.get_mut(0).unwrap(); - engine.call_fn_dynamic( - &mut Scope::new(), - lib, - fp.fn_name(), - Some(this_ptr), - [value], - )?; - - Ok(()) + fp.call_dynamic(engine, lib, Some(this_ptr), [value]) }, );