kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Add Engine::on_parse_token.

Browse Source
This commit is contained in:
Stephen Chung 2021-09-24 18:00:48 +08:00
parent cd8af67842
commit d791052d5c
5 changed files with 72 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -18,6 +18,7 @@ Enhancements
* `Engine::consume_XXX` methods are renamed to `Engine::run_XXX` to make meanings clearer. The `consume_XXX` API is deprecated.
* `Engine::register_type_XXX` are now available even under `no_object`.
* Added `Engine::on_parse_token` to allow remapping certain tokens during parsing.
### Custom Syntax

6
src/engine.rs
View File

@ -5,7 +5,8 @@ use crate::custom_syntax::CustomSyntax;
use crate::dynamic::{map_std_type_name, AccessMode, Union, Variant};
use crate::fn_hash::get_hasher;
use crate::fn_native::{
CallableFunction, IteratorFn, OnDebugCallback, OnPrintCallback, OnVarCallback,
CallableFunction, IteratorFn, OnDebugCallback, OnParseTokenCallback, OnPrintCallback,
OnVarCallback,
};
use crate::module::NamespaceRef;
use crate::optimize::OptimizationLevel;
@ -921,6 +922,8 @@ pub struct Engine {
pub(crate) custom_syntax: BTreeMap<Identifier, Box<CustomSyntax>>,
/// Callback closure for resolving variable access.
pub(crate) resolve_var: Option<OnVarCallback>,
/// Callback closure to remap tokens during parsing.
pub(crate) token_mapper: Option<Box<OnParseTokenCallback>>,
/// Callback closure for implementing the `print` command.
pub(crate) print: Option<OnPrintCallback>,
@ -1045,6 +1048,7 @@ impl Engine {
custom_syntax: Default::default(),
resolve_var: None,
token_mapper: None,
print: None,
debug: None,

56
src/engine_api.rs
View File

@ -1181,7 +1181,8 @@ impl Engine {
scripts: &[&str],
optimization_level: OptimizationLevel,
) -> Result<AST, ParseError> {
let (stream, tokenizer_control) = self.lex_raw(scripts, None);
let (stream, tokenizer_control) =
self.lex_raw(scripts, self.token_mapper.as_ref().map(Box::as_ref));
let mut state = ParseState::new(self, tokenizer_control);
self.parse(
&mut stream.peekable(),
@ -1368,13 +1369,13 @@ impl Engine {
let (stream, tokenizer_control) = engine.lex_raw(
&scripts,
Some(if has_null {
|token| match token {
&|token| match token {
// If `null` is present, make sure `null` is treated as a variable
Token::Reserved(s) if s == "null" => Token::Identifier(s),
_ => token,
}
} else {
|t| t
&|t| t
}),
);
let mut state = ParseState::new(engine, tokenizer_control);
@ -1462,7 +1463,8 @@ impl Engine {
script: &str,
) -> Result<AST, ParseError> {
let scripts = [script];
let (stream, tokenizer_control) = self.lex_raw(&scripts, None);
let (stream, tokenizer_control) =
self.lex_raw(&scripts, self.token_mapper.as_ref().map(Box::as_ref));
let mut peekable = stream.peekable();
let mut state = ParseState::new(self, tokenizer_control);
@ -1624,7 +1626,8 @@ impl Engine {
script: &str,
) -> Result<T, Box<EvalAltResult>> {
let scripts = [script];
let (stream, tokenizer_control) = self.lex_raw(&scripts, None);
let (stream, tokenizer_control) =
self.lex_raw(&scripts, self.token_mapper.as_ref().map(Box::as_ref));
let mut state = ParseState::new(self, tokenizer_control);
// No need to optimize a lone expression
@ -1769,7 +1772,8 @@ impl Engine {
script: &str,
) -> Result<(), Box<EvalAltResult>> {
let scripts = [script];
let (stream, tokenizer_control) = self.lex_raw(&scripts, None);
let (stream, tokenizer_control) =
self.lex_raw(&scripts, self.token_mapper.as_ref().map(Box::as_ref));
let mut state = ParseState::new(self, tokenizer_control);
let ast = self.parse(
@ -2114,6 +2118,46 @@ impl Engine {
self.resolve_var = Some(Box::new(callback));
self
}
/// _(internals)_ Provide a callback that will be invoked during parsing to remap certain tokens.
/// Exported under the `internals` feature only.
///
/// # Example
///
/// ```
/// # fn main() -> Result<(), Box<rhai::EvalAltResult>> {
/// use rhai::{Engine, Token};
///
/// let mut engine = Engine::new();
///
/// // Register a token mapper.
/// engine.on_parse_token(|token| {
/// match token {
/// // Convert all integer literals to strings
/// Token::IntegerConstant(n) => Token::StringConstant(n.to_string()),
/// // Convert 'begin' .. 'end' to '{' .. '}'
/// Token::Identifier(s) if &s == "begin" => Token::LeftBrace,
/// Token::Identifier(s) if &s == "end" => Token::RightBrace,
/// // Pass through all other tokens unchanged
/// _ => token
/// }
/// });
///
/// assert_eq!(engine.eval::<String>("42")?, "42");
/// assert_eq!(engine.eval::<bool>("true")?, true);
/// assert_eq!(engine.eval::<String>("let x = 42; begin let x = 0; end; x")?, "42");
///
/// # Ok(())
/// # }
/// ```
#[cfg(feature = "internals")]
#[inline(always)]
pub fn on_parse_token(
&mut self,
callback: impl Fn(crate::token::Token) -> crate::token::Token + SendSync + 'static,
) -> &mut Self {
self.token_mapper = Some(Box::new(callback));
self
}
/// Register a callback for script evaluation progress.
///
/// Not available under `unchecked`.

8
src/fn_native.rs
View File

@ -4,6 +4,7 @@ use crate::ast::{FnAccess, FnCallHashes};
use crate::engine::Imports;
use crate::fn_call::FnCallArgs;
use crate::plugin::PluginFunction;
use crate::token::Token;
use crate::{
calc_fn_hash, Dynamic, Engine, EvalAltResult, EvalContext, Module, Position, RhaiResult,
};
@ -307,6 +308,13 @@ pub type OnDebugCallback = Box<dyn Fn(&str, Option<&str>, Position) + 'static>;
#[cfg(feature = "sync")]
pub type OnDebugCallback = Box<dyn Fn(&str, Option<&str>, Position) + Send + Sync + 'static>;
/// A standard callback function for mapping tokens during parsing.
#[cfg(not(feature = "sync"))]
pub type OnParseTokenCallback = dyn Fn(Token) -> Token;
/// A standard callback function for mapping tokens during parsing.
#[cfg(feature = "sync")]
pub type OnParseTokenCallback = dyn Fn(Token) -> Token + Send + Sync + 'static;
/// A standard callback function for variable access.
#[cfg(not(feature = "sync"))]
pub type OnVarCallback =

15
src/token.rs
View File

@ -4,6 +4,7 @@ use crate::engine::{
Precedence, KEYWORD_DEBUG, KEYWORD_EVAL, KEYWORD_FN_PTR, KEYWORD_FN_PTR_CALL,
KEYWORD_FN_PTR_CURRY, KEYWORD_IS_DEF_VAR, KEYWORD_PRINT, KEYWORD_THIS, KEYWORD_TYPE_OF,
};
use crate::fn_native::OnParseTokenCallback;
use crate::{Engine, LexError, StaticVec, INT};
#[cfg(feature = "no_std")]
use std::prelude::v1::*;
@ -2123,12 +2124,12 @@ pub struct TokenIterator<'a> {
pub state: TokenizeState,
/// Current position.
pub pos: Position,
/// External buffer containing the next character to read, if any.
/// Shared object to allow controlling the tokenizer externally.
pub tokenizer_control: TokenizerControl,
/// Input character stream.
pub stream: MultiInputsStream<'a>,
/// A processor function that maps a token to another.
pub map: Option<fn(Token) -> Token>,
pub token_mapper: Option<&'a OnParseTokenCallback>,
}
impl<'a> Iterator for TokenIterator<'a> {
@ -2222,7 +2223,7 @@ impl<'a> Iterator for TokenIterator<'a> {
};
// Run the mapper, if any
let token = if let Some(map_func) = self.map {
let token = if let Some(map_func) = self.token_mapper {
map_func(token)
} else {
token
@ -2254,9 +2255,9 @@ impl Engine {
pub fn lex_with_map<'a>(
&'a self,
input: impl IntoIterator<Item = &'a &'a str>,
map: fn(Token) -> Token,
token_mapper: &'a OnParseTokenCallback,
) -> (TokenIterator<'a>, TokenizerControl) {
self.lex_raw(input, Some(map))
self.lex_raw(input, Some(token_mapper))
}
/// Tokenize an input text stream with an optional mapping function.
#[inline]
@ -2264,7 +2265,7 @@ impl Engine {
pub(crate) fn lex_raw<'a>(
&'a self,
input: impl IntoIterator<Item = &'a &'a str>,
map: Option<fn(Token) -> Token>,
token_mapper: Option<&'a OnParseTokenCallback>,
) -> (TokenIterator<'a>, TokenizerControl) {
let buffer: TokenizerControl = Default::default();
let buffer2 = buffer.clone();
@ -2289,7 +2290,7 @@ impl Engine {
streams: input.into_iter().map(|s| s.chars().peekable()).collect(),
index: 0,
},
map,
token_mapper,
},
buffer2,
)