From 0715262c44841cebf44f7dc28f1d93b3e79d1ca7 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Fri, 24 Sep 2021 22:44:39 +0800 Subject: [PATCH] Enhance signature of on_parse_token callback. --- src/engine_api.rs | 26 ++++++++++++++++---------- src/fn_native.rs | 7 ++++--- src/token.rs | 26 +++++++++++++++++++------- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/src/engine_api.rs b/src/engine_api.rs index ece66787..0abcaa89 100644 --- a/src/engine_api.rs +++ b/src/engine_api.rs @@ -1368,15 +1368,17 @@ impl Engine { }; let (stream, tokenizer_control) = engine.lex_raw( &scripts, - Some(if has_null { - &|token| match token { - // If `null` is present, make sure `null` is treated as a variable - Token::Reserved(s) if s == "null" => Token::Identifier(s), - _ => token, - } + if has_null { + Some(&|token, _, _| { + match token { + // If `null` is present, make sure `null` is treated as a variable + Token::Reserved(s) if s == "null" => Token::Identifier(s), + _ => token, + } + }) } else { - &|t| t - }), + None + }, ); let mut state = ParseState::new(engine, tokenizer_control); let ast = engine.parse_global_expr( @@ -2130,7 +2132,7 @@ impl Engine { /// let mut engine = Engine::new(); /// /// // Register a token mapper. - /// engine.on_parse_token(|token| { + /// engine.on_parse_token(|token, _, _| { /// match token { /// // Convert all integer literals to strings /// Token::IntegerConstant(n) => Token::StringConstant(n.to_string()), @@ -2153,8 +2155,12 @@ impl Engine { #[inline(always)] pub fn on_parse_token( &mut self, - callback: impl Fn(crate::token::Token) -> crate::token::Token + SendSync + 'static, + callback: impl Fn(crate::token::Token, Position, &crate::token::TokenizeState) -> crate::token::Token + + SendSync + + 'static, ) -> &mut Self { + use std::string::ParseError; + self.token_mapper = Some(Box::new(callback)); self } diff --git a/src/fn_native.rs b/src/fn_native.rs index 60f46274..9fa9d008 100644 --- a/src/fn_native.rs +++ b/src/fn_native.rs @@ -4,7 +4,7 @@ use crate::ast::{FnAccess, FnCallHashes}; use crate::engine::Imports; use crate::fn_call::FnCallArgs; use crate::plugin::PluginFunction; -use crate::token::Token; +use crate::token::{Token, TokenizeState}; use crate::{ calc_fn_hash, Dynamic, Engine, EvalAltResult, EvalContext, Module, Position, RhaiResult, }; @@ -310,10 +310,11 @@ pub type OnDebugCallback = Box, Position) + Send + Syn /// A standard callback function for mapping tokens during parsing. #[cfg(not(feature = "sync"))] -pub type OnParseTokenCallback = dyn Fn(Token) -> Token; +pub type OnParseTokenCallback = dyn Fn(Token, Position, &TokenizeState) -> Token; /// A standard callback function for mapping tokens during parsing. #[cfg(feature = "sync")] -pub type OnParseTokenCallback = dyn Fn(Token) -> Token + Send + Sync + 'static; +pub type OnParseTokenCallback = + dyn Fn(Token, Position, &TokenizeState) -> Token + Send + Sync + 'static; /// A standard callback function for variable access. #[cfg(not(feature = "sync"))] diff --git a/src/token.rs b/src/token.rs index 9baaf22a..1ffef163 100644 --- a/src/token.rs +++ b/src/token.rs @@ -29,6 +29,10 @@ use rust_decimal::Decimal; use crate::engine::KEYWORD_IS_DEF_FN; /// _(internals)_ A type containing commands to control the tokenizer. +/// +/// # Volatile Data Structure +/// +/// This type is volatile and may change. #[derive(Debug, Clone, Eq, PartialEq, Hash, Copy, Default)] pub struct TokenizerControlBlock { /// Is the current tokenizer position within an interpolated text string? @@ -992,7 +996,7 @@ pub struct TokenizeState { /// Maximum length of a string. pub max_string_size: Option, /// Can the next token be a unary operator? - pub non_unary: bool, + pub next_token_cannot_be_unary: bool, /// Is the tokenizer currently inside a block comment? pub comment_level: usize, /// Include comments? @@ -1327,7 +1331,7 @@ pub fn get_next_token( // Save the last token's state if let Some((ref token, _)) = result { - state.non_unary = !token.is_next_unary(); + state.next_token_cannot_be_unary = !token.is_next_unary(); } result @@ -1678,10 +1682,12 @@ fn get_next_token_inner( eat_next(stream, pos); return Some((Token::Reserved("++".into()), start_pos)); } - ('+', _) if !state.non_unary => return Some((Token::UnaryPlus, start_pos)), + ('+', _) if !state.next_token_cannot_be_unary => { + return Some((Token::UnaryPlus, start_pos)) + } ('+', _) => return Some((Token::Plus, start_pos)), - ('-', '0'..='9') if !state.non_unary => negated = Some(start_pos), + ('-', '0'..='9') if !state.next_token_cannot_be_unary => negated = Some(start_pos), ('-', '0'..='9') => return Some((Token::Minus, start_pos)), ('-', '=') => { eat_next(stream, pos); @@ -1695,7 +1701,9 @@ fn get_next_token_inner( eat_next(stream, pos); return Some((Token::Reserved("--".into()), start_pos)); } - ('-', _) if !state.non_unary => return Some((Token::UnaryMinus, start_pos)), + ('-', _) if !state.next_token_cannot_be_unary => { + return Some((Token::UnaryMinus, start_pos)) + } ('-', _) => return Some((Token::Minus, start_pos)), ('*', ')') => { @@ -2117,6 +2125,10 @@ impl InputStream for MultiInputsStream<'_> { /// _(internals)_ An iterator on a [`Token`] stream. /// Exported under the `internals` feature only. +/// +/// # Volatile Data Structure +/// +/// This type is volatile and may change. pub struct TokenIterator<'a> { /// Reference to the scripting `Engine`. pub engine: &'a Engine, @@ -2224,7 +2236,7 @@ impl<'a> Iterator for TokenIterator<'a> { // Run the mapper, if any let token = if let Some(map_func) = self.token_mapper { - map_func(token) + map_func(token, pos, &self.state) } else { token }; @@ -2278,7 +2290,7 @@ impl Engine { max_string_size: self.limits.max_string_size, #[cfg(feature = "unchecked")] max_string_size: None, - non_unary: false, + next_token_cannot_be_unary: false, comment_level: 0, include_comments: false, is_within_text_terminated_by: None,