kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Merge branch 'master' into plugins

Browse Source
This commit is contained in:
Stephen Chung
2020-07-10 22:06:02 +08:00
parent 72c034ccd1 ebffbf0f98
commit d95761d064
21 changed files with 1167 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files

337
src/engine.rs
View File

@@ -17,6 +17,9 @@ use crate::utils::StaticVec;
#[cfg(not(feature = "no_float"))]
use crate::parser::FLOAT;
#[cfg(feature = "internals")]
use crate::syntax::CustomSyntax;
use crate::stdlib::{
any::{type_name, TypeId},
borrow::Cow,
@@ -82,8 +85,15 @@ pub const KEYWORD_THIS: &str = "this";
pub const FN_TO_STRING: &str = "to_string";
pub const FN_GET: &str = "get$";
pub const FN_SET: &str = "set$";
pub const FN_IDX_GET: &str = "$index$get$";
pub const FN_IDX_SET: &str = "$index$set$";
pub const FN_IDX_GET: &str = "index$get$";
pub const FN_IDX_SET: &str = "index$set$";
#[cfg(feature = "internals")]
pub const MARKER_EXPR: &str = "$expr$";
#[cfg(feature = "internals")]
pub const MARKER_BLOCK: &str = "$block$";
#[cfg(feature = "internals")]
pub const MARKER_IDENT: &str = "$ident$";
/// A type specifying the method of chaining.
#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
@@ -273,6 +283,9 @@ pub struct Engine {
pub(crate) disabled_symbols: Option<HashSet<String>>,
/// A hashset containing custom keywords and precedence to recognize.
pub(crate) custom_keywords: Option<HashMap<String, u8>>,
/// Custom syntax.
#[cfg(feature = "internals")]
pub(crate) custom_syntax: Option<HashMap<String, CustomSyntax>>,
/// Callback closure for implementing the `print` command.
pub(crate) print: Callback<str, ()>,
@@ -323,6 +336,9 @@ impl Default for Engine {
disabled_symbols: None,
custom_keywords: None,
#[cfg(feature = "internals")]
custom_syntax: None,
// default print/debug implementations
print: Box::new(default_print),
debug: Box::new(default_print),
@@ -555,6 +571,9 @@ impl Engine {
disabled_symbols: None,
custom_keywords: None,
#[cfg(feature = "internals")]
custom_syntax: None,
print: Box::new(|_| {}),
debug: Box::new(|_| {}),
progress: None,
@@ -1003,6 +1022,81 @@ impl Engine {
return Ok(result);
}
/// Call a dot method.
fn call_method(
&self,
state: &mut State,
lib: &Module,
target: &mut Target,
expr: &Expr,
mut idx_val: Dynamic,
level: usize,
) -> Result<(Dynamic, bool), Box<EvalAltResult>> {
let ((name, native, pos), _, hash, _, def_val) = match expr {
Expr::FnCall(x) => x.as_ref(),
_ => unreachable!(),
};
let is_ref = target.is_ref();
let is_value = target.is_value();
let def_val = def_val.as_ref();
// Get a reference to the mutation target Dynamic
let obj = target.as_mut();
let idx = idx_val.downcast_mut::<StaticVec<Dynamic>>().unwrap();
let mut fn_name = name.as_ref();
// Check if it is a FnPtr call
let (result, updated) = if fn_name == KEYWORD_FN_PTR_CALL && obj.is::<FnPtr>() {
// Redirect function name
fn_name = obj.as_str().unwrap();
// Recalculate hash
let hash = calc_fn_hash(empty(), fn_name, idx.len(), empty());
// Arguments are passed as-is
let mut arg_values = idx.iter_mut().collect::<StaticVec<_>>();
let args = arg_values.as_mut();
// Map it to name(args) in function-call style
self.exec_fn_call(
state, lib, fn_name, *native, hash, args, false, false, def_val, level,
)
} else {
let redirected: Option<ImmutableString>;
let mut hash = *hash;
// Check if it is a map method call in OOP style
if let Some(map) = obj.downcast_ref::<Map>() {
if let Some(val) = map.get(fn_name) {
if let Some(f) = val.downcast_ref::<FnPtr>() {
// Remap the function name
redirected = Some(f.get_fn_name().clone());
fn_name = redirected.as_ref().unwrap();
// Recalculate the hash based on the new function name
hash = calc_fn_hash(empty(), fn_name, idx.len(), empty());
}
}
};
// Attached object pointer in front of the arguments
let mut arg_values = once(obj).chain(idx.iter_mut()).collect::<StaticVec<_>>();
let args = arg_values.as_mut();
self.exec_fn_call(
state, lib, fn_name, *native, hash, args, is_ref, true, def_val, level,
)
}
.map_err(|err| err.new_position(*pos))?;
// Feed the changed temp value back
if updated && !is_ref && !is_value {
let new_val = target.as_mut().clone();
target.set_value(new_val)?;
}
Ok((result, updated))
}
/// Chain-evaluate a dot/index chain.
/// Position in `EvalAltResult` is None and must be set afterwards.
fn eval_dot_index_chain_helper(
@@ -1022,7 +1116,6 @@ impl Engine {
}
let is_ref = target.is_ref();
let is_value = target.is_value();
let next_chain = match rhs {
Expr::Index(_) => ChainType::Index,
@@ -1031,7 +1124,7 @@ impl Engine {
};
// Pop the last index value
let mut idx_val = idx_values.pop();
let idx_val = idx_values.pop();
match chain_type {
#[cfg(not(feature = "no_index"))]
@@ -1115,69 +1208,7 @@ impl Engine {
match rhs {
// xxx.fn_name(arg_expr_list)
Expr::FnCall(x) if x.1.is_none() => {
let ((name, native, pos), _, hash, _, def_val) = x.as_ref();
let def_val = def_val.as_ref();
// Get a reference to the mutation target Dynamic
let (result, updated) = {
let obj = target.as_mut();
let idx = idx_val.downcast_mut::<StaticVec<Dynamic>>().unwrap();
let mut fn_name = name.as_ref();
// Check if it is a FnPtr call
if fn_name == KEYWORD_FN_PTR_CALL && obj.is::<FnPtr>() {
// Redirect function name
fn_name = obj.as_str().unwrap();
// Recalculate hash
let hash = calc_fn_hash(empty(), fn_name, idx.len(), empty());
// Arguments are passed as-is
let mut arg_values = idx.iter_mut().collect::<StaticVec<_>>();
let args = arg_values.as_mut();
// Map it to name(args) in function-call style
self.exec_fn_call(
state, lib, fn_name, *native, hash, args, false, false,
def_val, level,
)
} else {
let redirected: Option<ImmutableString>;
let mut hash = *hash;
// Check if it is a map method call in OOP style
if let Some(map) = obj.downcast_ref::<Map>() {
if let Some(val) = map.get(fn_name) {
if let Some(f) = val.downcast_ref::<FnPtr>() {
// Remap the function name
redirected = Some(f.get_fn_name().clone());
fn_name = redirected.as_ref().unwrap();
// Recalculate the hash based on the new function name
hash =
calc_fn_hash(empty(), fn_name, idx.len(), empty());
}
}
};
// Attached object pointer in front of the arguments
let mut arg_values =
once(obj).chain(idx.iter_mut()).collect::<StaticVec<_>>();
let args = arg_values.as_mut();
self.exec_fn_call(
state, lib, fn_name, *native, hash, args, is_ref, true,
def_val, level,
)
}
.map_err(|err| err.new_position(*pos))?
};
// Feed the changed temp value back
if updated && !is_ref && !is_value {
let new_val = target.as_mut().clone();
target.set_value(new_val)?;
}
Ok((result, updated))
self.call_method(state, lib, target, rhs, idx_val, level)
}
// xxx.module::fn_name(...) - syntax error
Expr::FnCall(_) => unreachable!(),
@@ -1221,16 +1252,26 @@ impl Engine {
.map(|(v, _)| (v, false))
.map_err(|err| err.new_position(*pos))
}
// {xxx:map}.prop[expr] | {xxx:map}.prop.expr
// {xxx:map}.sub_lhs[expr] | {xxx:map}.sub_lhs.expr
Expr::Index(x) | Expr::Dot(x) if target.is::<Map>() => {
let (prop, expr, pos) = x.as_ref();
let (sub_lhs, expr, pos) = x.as_ref();
let mut val = if let Expr::Property(p) = prop {
let ((prop, _, _), _) = p.as_ref();
let index = prop.clone().into();
self.get_indexed_mut(state, lib, target, index, *pos, false, level)?
} else {
unreachable!();
let mut val = match sub_lhs {
Expr::Property(p) => {
let ((prop, _, _), _) = p.as_ref();
let index = prop.clone().into();
self.get_indexed_mut(state, lib, target, index, *pos, false, level)?
}
// {xxx:map}.fn_name(arg_expr_list)[expr] | {xxx:map}.fn_name(arg_expr_list).expr
Expr::FnCall(x) if x.1.is_none() => {
let (val, _) =
self.call_method(state, lib, target, sub_lhs, idx_val, level)?;
val.into()
}
// {xxx:map}.module::fn_name(...) - syntax error
Expr::FnCall(_) => unreachable!(),
// Others - syntax error
_ => unreachable!(),
};
self.eval_dot_index_chain_helper(
@@ -1239,49 +1280,72 @@ impl Engine {
)
.map_err(|err| err.new_position(*pos))
}
// xxx.prop[expr] | xxx.prop.expr
// xxx.sub_lhs[expr] | xxx.sub_lhs.expr
Expr::Index(x) | Expr::Dot(x) => {
let (prop, expr, pos) = x.as_ref();
let args = &mut [target.as_mut(), &mut Default::default()];
let (sub_lhs, expr, pos) = x.as_ref();
let (mut val, updated) = if let Expr::Property(p) = prop {
let ((_, getter, _), _) = p.as_ref();
let args = &mut args[..1];
self.exec_fn_call(
state, lib, getter, true, 0, args, is_ref, true, None, level,
)
.map_err(|err| err.new_position(*pos))?
} else {
unreachable!();
};
let val = &mut val;
let target = &mut val.into();
match sub_lhs {
// xxx.prop[expr] | xxx.prop.expr
Expr::Property(p) => {
let ((_, getter, setter), _) = p.as_ref();
let arg_values = &mut [target.as_mut(), &mut Default::default()];
let args = &mut arg_values[..1];
let (mut val, updated) = self
.exec_fn_call(
state, lib, getter, true, 0, args, is_ref, true, None,
level,
)
.map_err(|err| err.new_position(*pos))?;
let (result, may_be_changed) = self
.eval_dot_index_chain_helper(
state, lib, this_ptr, target, expr, idx_values, next_chain, level,
new_val,
)
.map_err(|err| err.new_position(*pos))?;
let val = &mut val;
let target = &mut val.into();
// Feed the value back via a setter just in case it has been updated
if updated || may_be_changed {
if let Expr::Property(p) = prop {
let ((_, _, setter), _) = p.as_ref();
// Re-use args because the first &mut parameter will not be consumed
args[1] = val;
self.exec_fn_call(
state, lib, setter, true, 0, args, is_ref, true, None, level,
)
.or_else(|err| match *err {
// If there is no setter, no need to feed it back because the property is read-only
EvalAltResult::ErrorDotExpr(_, _) => Ok(Default::default()),
_ => Err(err.new_position(*pos)),
})?;
let (result, may_be_changed) = self
.eval_dot_index_chain_helper(
state, lib, this_ptr, target, expr, idx_values, next_chain,
level, new_val,
)
.map_err(|err| err.new_position(*pos))?;
// Feed the value back via a setter just in case it has been updated
if updated || may_be_changed {
// Re-use args because the first &mut parameter will not be consumed
arg_values[1] = val;
self.exec_fn_call(
state, lib, setter, true, 0, arg_values, is_ref, true,
None, level,
)
.or_else(
|err| match *err {
// If there is no setter, no need to feed it back because the property is read-only
EvalAltResult::ErrorDotExpr(_, _) => {
Ok(Default::default())
}
_ => Err(err.new_position(*pos)),
},
)?;
}
Ok((result, may_be_changed))
}
}
// xxx.fn_name(arg_expr_list)[expr] | xxx.fn_name(arg_expr_list).expr
Expr::FnCall(x) if x.1.is_none() => {
let (mut val, _) =
self.call_method(state, lib, target, sub_lhs, idx_val, level)?;
let val = &mut val;
let target = &mut val.into();
Ok((result, may_be_changed))
self.eval_dot_index_chain_helper(
state, lib, this_ptr, target, expr, idx_values, next_chain,
level, new_val,
)
.map_err(|err| err.new_position(*pos))
}
// xxx.module::fn_name(...) - syntax error
Expr::FnCall(_) => unreachable!(),
// Others - syntax error
_ => unreachable!(),
}
}
// Syntax error
_ => Err(Box::new(EvalAltResult::ErrorDotExpr(
@@ -1316,7 +1380,7 @@ impl Engine {
let idx_values = &mut StaticVec::new();
self.eval_indexed_chain(
scope, mods, state, lib, this_ptr, dot_rhs, idx_values, 0, level,
scope, mods, state, lib, this_ptr, dot_rhs, chain_type, idx_values, 0, level,
)?;
match dot_lhs {
@@ -1380,6 +1444,7 @@ impl Engine {
lib: &Module,
this_ptr: &mut Option<&mut Dynamic>,
expr: &Expr,
chain_type: ChainType,
idx_values: &mut StaticVec<Dynamic>,
size: usize,
level: usize,
@@ -1406,12 +1471,29 @@ impl Engine {
// Evaluate in left-to-right order
let lhs_val = match lhs {
Expr::Property(_) => Default::default(), // Store a placeholder in case of a property
Expr::FnCall(x) if chain_type == ChainType::Dot && x.1.is_none() => {
let arg_values = x
.3
.iter()
.map(|arg_expr| {
self.eval_expr(scope, mods, state, lib, this_ptr, arg_expr, level)
})
.collect::<Result<StaticVec<Dynamic>, _>>()?;
Dynamic::from(arg_values)
}
Expr::FnCall(_) => unreachable!(),
_ => self.eval_expr(scope, mods, state, lib, this_ptr, lhs, level)?,
};
// Push in reverse order
let chain_type = match expr {
Expr::Index(_) => ChainType::Index,
Expr::Dot(_) => ChainType::Dot,
_ => unreachable!(),
};
self.eval_indexed_chain(
scope, mods, state, lib, this_ptr, rhs, idx_values, size, level,
scope, mods, state, lib, this_ptr, rhs, chain_type, idx_values, size, level,
)?;
idx_values.push(lhs_val);
@@ -1595,6 +1677,26 @@ impl Engine {
}
}
/// Evaluate an expression tree.
///
/// ## WARNING - Low Level API
///
/// This function is very low level. It evaluates an expression from an AST.
#[cfg(feature = "internals")]
#[deprecated(note = "this method is volatile and may change")]
pub fn eval_expression_tree(
&self,
scope: &mut Scope,
mods: &mut Imports,
state: &mut State,
lib: &Module,
this_ptr: &mut Option<&mut Dynamic>,
expr: &Expr,
level: usize,
) -> Result<Dynamic, Box<EvalAltResult>> {
self.eval_expr(scope, mods, state, lib, this_ptr, expr, level)
}
/// Evaluate an expression
fn eval_expr(
&self,
@@ -2027,6 +2129,13 @@ impl Engine {
Expr::False(_) => Ok(false.into()),
Expr::Unit(_) => Ok(().into()),
#[cfg(feature = "internals")]
Expr::Custom(x) => {
let func = (x.0).1.as_ref();
let exprs = (x.0).0.as_ref();
func(self, scope, mods, state, lib, this_ptr, exprs, level)
}
_ => unreachable!(),
};

1
src/fn_native.rs
View File

@@ -1,3 +1,4 @@
//! Module containing interfaces with native-Rust functions.
use crate::any::Dynamic;
use crate::engine::Engine;
use crate::module::Module;

12
src/lib.rs
View File

@@ -95,6 +95,8 @@ mod scope;
mod serde;
mod settings;
mod stdlib;
#[cfg(feature = "internals")]
mod syntax;
mod token;
mod r#unsafe;
mod utils;
@@ -157,13 +159,21 @@ pub use optimize::OptimizationLevel;
// Expose internal data structures.
#[cfg(feature = "internals")]
#[deprecated(note = "this type is volatile and may change")]
pub use error::LexError;
#[cfg(feature = "internals")]
#[deprecated(note = "this type is volatile and may change")]
pub use token::{get_next_token, parse_string_literal, InputStream, Token, TokenizeState};
#[cfg(feature = "internals")]
#[deprecated(note = "this type is volatile and may change")]
pub use parser::{Expr, ReturnType, ScriptFnDef, Stmt};
pub use parser::{CustomExpr, Expr, ReturnType, ScriptFnDef, Stmt};
#[cfg(feature = "internals")]
#[deprecated(note = "this type is volatile and may change")]
pub use engine::{Imports, State as EvalState};
#[cfg(feature = "internals")]
#[deprecated(note = "this type is volatile and may change")]

12
src/optimize.rs
View File

@@ -6,6 +6,9 @@ use crate::parser::{map_dynamic_to_expr, Expr, ReturnType, ScriptFnDef, Stmt, AS
use crate::scope::{Entry as ScopeEntry, EntryType as ScopeEntryType, Scope};
use crate::utils::StaticVec;
#[cfg(feature = "internals")]
use crate::parser::CustomExpr;
use crate::stdlib::{
boxed::Box,
iter::empty,
@@ -598,6 +601,15 @@ fn optimize_expr(expr: Expr, state: &mut State) -> Expr {
state.find_constant(&name).expect("should find constant in scope!").clone().set_position(pos)
}
// Custom syntax
#[cfg(feature = "internals")]
Expr::Custom(x) => Expr::Custom(Box::new((
CustomExpr(
(x.0).0.into_iter().map(|expr| optimize_expr(expr, state)).collect(),
(x.0).1),
x.1
))),
// All other expressions - skip
expr => expr,
}

116
src/parser.rs
View File

@@ -10,6 +10,15 @@ use crate::scope::{EntryType as ScopeEntryType, Scope};
use crate::token::{Position, Token, TokenStream};
use crate::utils::{StaticVec, StraightHasherBuilder};
#[cfg(feature = "internals")]
use crate::engine::{MARKER_BLOCK, MARKER_EXPR, MARKER_IDENT};
#[cfg(feature = "internals")]
use crate::fn_native::Shared;
#[cfg(feature = "internals")]
use crate::syntax::FnCustomSyntaxEval;
use crate::stdlib::{
borrow::Cow,
boxed::Box,
@@ -568,6 +577,17 @@ impl Stmt {
}
}
#[derive(Clone)]
#[cfg(feature = "internals")]
pub struct CustomExpr(pub StaticVec<Expr>, pub Shared<FnCustomSyntaxEval>);
#[cfg(feature = "internals")]
impl fmt::Debug for CustomExpr {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Debug::fmt(&self.0, f)
}
}
/// An expression.
///
/// Each variant is at most one pointer in size (for speed),
@@ -632,6 +652,9 @@ pub enum Expr {
False(Position),
/// ()
Unit(Position),
/// Custom syntax
#[cfg(feature = "internals")]
Custom(Box<(CustomExpr, Position)>),
}
impl Default for Expr {
@@ -726,6 +749,9 @@ impl Expr {
Self::True(pos) | Self::False(pos) | Self::Unit(pos) => *pos,
Self::Dot(x) | Self::Index(x) => x.0.position(),
#[cfg(feature = "internals")]
Self::Custom(x) => x.1,
}
}
@@ -758,6 +784,9 @@ impl Expr {
Self::Assignment(x) => x.3 = new_pos,
Self::Dot(x) => x.2 = new_pos,
Self::Index(x) => x.2 = new_pos,
#[cfg(feature = "internals")]
Self::Custom(x) => x.1 = new_pos,
}
self
@@ -861,6 +890,9 @@ impl Expr {
Token::LeftParen => true,
_ => false,
},
#[cfg(feature = "internals")]
Self::Custom(_) => false,
}
}
@@ -876,6 +908,14 @@ impl Expr {
_ => self,
}
}
#[cfg(feature = "internals")]
pub fn get_variable_name(&self) -> Option<&str> {
match self {
Self::Variable(x) => Some((x.0).0.as_str()),
_ => None,
}
}
}
/// Consume a particular token, checking that it is the expected one.
@@ -2024,6 +2064,80 @@ fn parse_expr(
settings.pos = input.peek().unwrap().1;
settings.ensure_level_within_max_limit(state.max_expr_depth)?;
// Check if it is a custom syntax.
#[cfg(feature = "internals")]
if let Some(ref custom) = state.engine.custom_syntax {
let (token, pos) = input.peek().unwrap();
let token_pos = *pos;
match token {
Token::Custom(key) if custom.contains_key(key) => {
let custom = custom.get_key_value(key).unwrap();
let (key, syntax) = custom;
input.next().unwrap();
let mut exprs: StaticVec<Expr> = Default::default();
// Adjust the variables stack
match syntax.scope_delta {
delta if delta > 0 => {
state.stack.push(("".to_string(), ScopeEntryType::Normal))
}
delta if delta < 0 && state.stack.len() <= delta.abs() as usize => {
state.stack.clear()
}
delta if delta < 0 => state
.stack
.truncate(state.stack.len() - delta.abs() as usize),
_ => (),
}
for segment in syntax.segments.iter() {
settings.pos = input.peek().unwrap().1;
let settings = settings.level_up();
match segment.as_str() {
MARKER_IDENT => match input.next().unwrap() {
(Token::Identifier(s), pos) => {
exprs.push(Expr::Variable(Box::new(((s, pos), None, 0, None))));
}
(_, pos) => return Err(PERR::VariableExpected.into_err(pos)),
},
MARKER_EXPR => exprs.push(parse_expr(input, state, lib, settings)?),
MARKER_BLOCK => {
let stmt = parse_block(input, state, lib, settings)?;
let pos = stmt.position();
exprs.push(Expr::Stmt(Box::new((stmt, pos))))
}
s => match input.peek().unwrap() {
(Token::Custom(custom), _) if custom == s => {
input.next().unwrap();
}
(t, _) if t.syntax().as_ref() == s => {
input.next().unwrap();
}
(_, pos) => {
return Err(PERR::MissingToken(
s.to_string(),
format!("for '{}' expression", key),
)
.into_err(*pos))
}
},
}
}
return Ok(Expr::Custom(Box::new((
CustomExpr(exprs, syntax.func.clone()),
token_pos,
))));
}
_ => (),
}
}
// Parse expression normally.
let lhs = parse_unary(input, state, lib, settings.level_up())?;
parse_binary_op(input, state, lib, 1, lhs, settings.level_up())
}
@@ -2297,7 +2411,7 @@ fn parse_import(
fn parse_export(
input: &mut TokenStream,
state: &mut ParseState,
lib: &mut FunctionsLib,
_lib: &mut FunctionsLib,
mut settings: ParseSettings,
) -> Result<Stmt, ParseError> {
settings.pos = eat_token(input, Token::Export);

151
src/syntax.rs Normal file
View File

@@ -0,0 +1,151 @@
//! Module containing implementation for custom syntax.
#![cfg(feature = "internals")]
use crate::any::Dynamic;
use crate::engine::{Engine, Imports, State, MARKER_BLOCK, MARKER_EXPR, MARKER_IDENT};
use crate::error::LexError;
use crate::fn_native::{SendSync, Shared};
use crate::module::Module;
use crate::parser::Expr;
use crate::result::EvalAltResult;
use crate::scope::Scope;
use crate::token::{is_valid_identifier, Token};
use crate::utils::StaticVec;
use crate::stdlib::{
fmt,
rc::Rc,
string::{String, ToString},
sync::Arc,
};
/// A general function trail object.
#[cfg(not(feature = "sync"))]
pub type FnCustomSyntaxEval = dyn Fn(
&Engine,
&mut Scope,
&mut Imports,
&mut State,
&Module,
&mut Option<&mut Dynamic>,
&[Expr],
usize,
) -> Result<Dynamic, Box<EvalAltResult>>;
/// A general function trail object.
#[cfg(feature = "sync")]
pub type FnCustomSyntaxEval = dyn Fn(
&Engine,
&mut Scope,
&mut Imports,
&mut State,
&Module,
&mut Option<&mut Dynamic>,
&[Expr],
usize,
) -> Result<Dynamic, Box<EvalAltResult>>
+ Send
+ Sync;
#[derive(Clone)]
pub struct CustomSyntax {
pub segments: StaticVec<String>,
pub func: Shared<FnCustomSyntaxEval>,
pub scope_delta: isize,
}
impl fmt::Debug for CustomSyntax {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Debug::fmt(&self.segments, f)
}
}
impl Engine {
pub fn register_custom_syntax<S: AsRef<str> + ToString>(
&mut self,
value: &[S],
scope_delta: isize,
func: impl Fn(
&Engine,
&mut Scope,
&mut Imports,
&mut State,
&Module,
&mut Option<&mut Dynamic>,
&[Expr],
usize,
) -> Result<Dynamic, Box<EvalAltResult>>
+ SendSync
+ 'static,
) -> Result<(), Box<LexError>> {
if value.is_empty() {
return Err(Box::new(LexError::ImproperSymbol("".to_string())));
}
let mut segments: StaticVec<_> = Default::default();
for s in value {
let seg = match s.as_ref() {
// Markers not in first position
MARKER_EXPR | MARKER_BLOCK | MARKER_IDENT if !segments.is_empty() => s.to_string(),
// Standard symbols not in first position
s if !segments.is_empty() && Token::lookup_from_syntax(s).is_some() => {
if self
.disabled_symbols
.as_ref()
.map(|d| d.contains(s))
.unwrap_or(false)
{
// If symbol is disabled, make it a custom keyword
if self.custom_keywords.is_none() {
self.custom_keywords = Some(Default::default());
}
if !self.custom_keywords.as_ref().unwrap().contains_key(s) {
self.custom_keywords.as_mut().unwrap().insert(s.into(), 0);
}
}
s.into()
}
// Identifier
s if is_valid_identifier(s.chars()) => {
if self.custom_keywords.is_none() {
self.custom_keywords = Some(Default::default());
}
if !self.custom_keywords.as_ref().unwrap().contains_key(s) {
self.custom_keywords.as_mut().unwrap().insert(s.into(), 0);
}
s.into()
}
// Anything else is an error
_ => return Err(Box::new(LexError::ImproperSymbol(s.to_string()))),
};
segments.push(seg);
}
let key = segments.remove(0);
let syntax = CustomSyntax {
segments,
#[cfg(not(feature = "sync"))]
func: Rc::new(func),
#[cfg(feature = "sync")]
func: Arc::new(func),
scope_delta,
};
if self.custom_syntax.is_none() {
self.custom_syntax = Some(Default::default());
}
self.custom_syntax
.as_mut()
.unwrap()
.insert(key, syntax.into());
Ok(())
}
}

208
src/token.rs
View File

@@ -312,6 +312,87 @@ impl Token {
}
}
/// Reverse lookup a token from a piece of syntax.
pub fn lookup_from_syntax(syntax: &str) -> Option<Self> {
use Token::*;
Some(match syntax {
"{" => LeftBrace,
"}" => RightBrace,
"(" => LeftParen,
")" => RightParen,
"[" => LeftBracket,
"]" => RightBracket,
"+" => Plus,
"-" => Minus,
"*" => Multiply,
"/" => Divide,
";" => SemiColon,
":" => Colon,
"::" => DoubleColon,
"," => Comma,
"." => Period,
"#{" => MapStart,
"=" => Equals,
"true" => True,
"false" => False,
"let" => Let,
"const" => Const,
"if" => If,
"else" => Else,
"while" => While,
"loop" => Loop,
"for" => For,
"in" => In,
"<" => LessThan,
">" => GreaterThan,
"!" => Bang,
"<=" => LessThanEqualsTo,
">=" => GreaterThanEqualsTo,
"==" => EqualsTo,
"!=" => NotEqualsTo,
"|" => Pipe,
"||" => Or,
"&" => Ampersand,
"&&" => And,
#[cfg(not(feature = "no_function"))]
"fn" => Fn,
"continue" => Continue,
"break" => Break,
"return" => Return,
"throw" => Throw,
"+=" => PlusAssign,
"-=" => MinusAssign,
"*=" => MultiplyAssign,
"/=" => DivideAssign,
"<<=" => LeftShiftAssign,
">>=" => RightShiftAssign,
"&=" => AndAssign,
"|=" => OrAssign,
"^=" => XOrAssign,
"<<" => LeftShift,
">>" => RightShift,
"^" => XOr,
"%" => Modulo,
"%=" => ModuloAssign,
"~" => PowerOf,
"~=" => PowerOfAssign,
#[cfg(not(feature = "no_function"))]
"private" => Private,
#[cfg(not(feature = "no_module"))]
"import" => Import,
#[cfg(not(feature = "no_module"))]
"export" => Export,
#[cfg(not(feature = "no_module"))]
"as" => As,
"===" | "!==" | "->" | "<-" | "=>" | ":=" | "::<" | "(*" | "*)" | "#" => {
Reserved(syntax.into())
}
_ => return None,
})
}
// Is this token EOF?
pub fn is_eof(&self) -> bool {
use Token::*;
@@ -450,7 +531,7 @@ impl Token {
}
}
/// Is this token a keyword?
/// Is this token a standard keyword?
pub fn is_keyword(&self) -> bool {
use Token::*;
@@ -467,6 +548,22 @@ impl Token {
_ => false,
}
}
/// Is this token a reserved keyword?
pub fn is_reserved(&self) -> bool {
match self {
Self::Reserved(_) => true,
_ => false,
}
}
/// Is this token a custom keyword?
pub fn is_custom(&self) -> bool {
match self {
Self::Custom(_) => true,
_ => false,
}
}
}
impl From<Token> for String {
@@ -628,9 +725,9 @@ pub fn parse_string_literal(
}
/// Consume the next character.
fn eat_next(stream: &mut impl InputStream, pos: &mut Position) {
stream.get_next();
fn eat_next(stream: &mut impl InputStream, pos: &mut Position) -> Option<char> {
pos.advance();
stream.get_next()
}
/// Scan for a block comment until the end.
@@ -858,35 +955,8 @@ fn get_next_token_inner(
}
return Some((
match identifier.as_str() {
"true" => Token::True,
"false" => Token::False,
"let" => Token::Let,
"const" => Token::Const,
"if" => Token::If,
"else" => Token::Else,
"while" => Token::While,
"loop" => Token::Loop,
"continue" => Token::Continue,
"break" => Token::Break,
"return" => Token::Return,
"throw" => Token::Throw,
"for" => Token::For,
"in" => Token::In,
#[cfg(not(feature = "no_function"))]
"private" => Token::Private,
#[cfg(not(feature = "no_module"))]
"import" => Token::Import,
#[cfg(not(feature = "no_module"))]
"export" => Token::Export,
#[cfg(not(feature = "no_module"))]
"as" => Token::As,
#[cfg(not(feature = "no_function"))]
"fn" => Token::Fn,
_ => Token::Identifier(identifier),
},
Token::lookup_from_syntax(&identifier)
.unwrap_or_else(|| Token::Identifier(identifier)),
start_pos,
));
}
@@ -933,7 +1003,10 @@ fn get_next_token_inner(
('}', _) => return Some((Token::RightBrace, start_pos)),
// Parentheses
('(', '*') => return Some((Token::Reserved("(*".into()), start_pos)),
('(', '*') => {
eat_next(stream, pos);
return Some((Token::Reserved("(*".into()), start_pos));
}
('(', _) => return Some((Token::LeftParen, start_pos)),
(')', _) => return Some((Token::RightParen, start_pos)),
@@ -947,6 +1020,7 @@ fn get_next_token_inner(
eat_next(stream, pos);
return Some((Token::MapStart, start_pos));
}
('#', _) => return Some((Token::Reserved("#".into()), start_pos)),
// Operators
('+', '=') => {
@@ -962,11 +1036,17 @@ fn get_next_token_inner(
eat_next(stream, pos);
return Some((Token::MinusAssign, start_pos));
}
('-', '>') => return Some((Token::Reserved("->".into()), start_pos)),
('-', '>') => {
eat_next(stream, pos);
return Some((Token::Reserved("->".into()), start_pos));
}
('-', _) if !state.non_unary => return Some((Token::UnaryMinus, start_pos)),
('-', _) => return Some((Token::Minus, start_pos)),
('*', ')') => return Some((Token::Reserved("*)".into()), start_pos)),
('*', ')') => {
eat_next(stream, pos);
return Some((Token::Reserved("*)".into()), start_pos));
}
('*', '=') => {
eat_next(stream, pos);
return Some((Token::MultiplyAssign, start_pos));
@@ -1031,31 +1111,42 @@ fn get_next_token_inner(
// Warn against `===`
if stream.peek_next() == Some('=') {
eat_next(stream, pos);
return Some((Token::Reserved("===".into()), start_pos));
}
return Some((Token::EqualsTo, start_pos));
}
('=', '>') => return Some((Token::Reserved("=>".into()), start_pos)),
('=', '>') => {
eat_next(stream, pos);
return Some((Token::Reserved("=>".into()), start_pos));
}
('=', _) => return Some((Token::Equals, start_pos)),
(':', ':') => {
eat_next(stream, pos);
if stream.peek_next() == Some('<') {
eat_next(stream, pos);
return Some((Token::Reserved("::<".into()), start_pos));
}
return Some((Token::DoubleColon, start_pos));
}
(':', '=') => return Some((Token::Reserved(":=".into()), start_pos)),
(':', '=') => {
eat_next(stream, pos);
return Some((Token::Reserved(":=".into()), start_pos));
}
(':', _) => return Some((Token::Colon, start_pos)),
('<', '=') => {
eat_next(stream, pos);
return Some((Token::LessThanEqualsTo, start_pos));
}
('<', '-') => return Some((Token::Reserved("<-".into()), start_pos)),
('<', '-') => {
eat_next(stream, pos);
return Some((Token::Reserved("<-".into()), start_pos));
}
('<', '<') => {
eat_next(stream, pos);
@@ -1094,6 +1185,7 @@ fn get_next_token_inner(
eat_next(stream, pos);
if stream.peek_next() == Some('=') {
eat_next(stream, pos);
return Some((Token::Reserved("!==".into()), start_pos));
}
@@ -1163,40 +1255,42 @@ fn get_next_token_inner(
}
/// A type that implements the `InputStream` trait.
/// Multiple charaacter streams are jointed together to form one single stream.
/// Multiple character streams are jointed together to form one single stream.
pub struct MultiInputsStream<'a> {
/// The input character streams.
streams: StaticVec<Peekable<Chars<'a>>>,
/// The current stream index.
index: usize,
}
impl InputStream for MultiInputsStream<'_> {
/// Get the next character
fn get_next(&mut self) -> Option<char> {
loop {
if self.streams.is_empty() {
if self.index >= self.streams.len() {
// No more streams
return None;
} else if let Some(ch) = self.streams[0].next() {
} else if let Some(ch) = self.streams[self.index].next() {
// Next character in current stream
return Some(ch);
} else {
// Jump to the next stream
let _ = self.streams.remove(0);
self.index += 1;
}
}
}
/// Peek the next character
fn peek_next(&mut self) -> Option<char> {
loop {
if self.streams.is_empty() {
if self.index >= self.streams.len() {
// No more streams
return None;
} else if let Some(ch) = self.streams[0].peek() {
} else if let Some(&ch) = self.streams[self.index].peek() {
// Next character in current stream
return Some(*ch);
return Some(ch);
} else {
// Jump to the next stream
let _ = self.streams.remove(0);
self.index += 1;
}
}
}
@@ -1252,7 +1346,11 @@ impl<'a> Iterator for TokenIterator<'a, '_> {
.to_string(),
))),
"(*" | "*)" => Token::LexError(Box::new(LERR::ImproperSymbol(
"'(* .. *)' is not a valid comment style. This is not Pascal! Should it be '/* .. */'?"
"'(* .. *)' is not a valid comment format. This is not Pascal! Should it be '/* .. */'?"
.to_string(),
))),
"#" => Token::LexError(Box::new(LERR::ImproperSymbol(
"'#' is not a valid symbol. Should it be '#{'?"
.to_string(),
))),
token => Token::LexError(Box::new(LERR::ImproperSymbol(
@@ -1260,6 +1358,17 @@ impl<'a> Iterator for TokenIterator<'a, '_> {
))),
}, pos)),
(r @ Some(_), None, None) => r,
(Some((Token::Identifier(s), pos)), _, Some(custom)) if custom.contains_key(&s) => {
// Convert custom keywords
Some((Token::Custom(s), pos))
}
(Some((token, pos)), _, Some(custom))
if (token.is_keyword() || token.is_operator() || token.is_reserved())
&& custom.contains_key(token.syntax().as_ref()) =>
{
// Convert into custom keywords
Some((Token::Custom(token.syntax().into()), pos))
}
(Some((token, pos)), Some(disabled), _)
if token.is_operator() && disabled.contains(token.syntax().as_ref()) =>
{
@@ -1275,10 +1384,6 @@ impl<'a> Iterator for TokenIterator<'a, '_> {
// Convert disallowed keywords into identifiers
Some((Token::Identifier(token.syntax().into()), pos))
}
(Some((Token::Identifier(s), pos)), _, Some(custom)) if custom.contains_key(&s) => {
// Convert custom keywords
Some((Token::Custom(s), pos))
}
(r, _, _) => r,
}
}
@@ -1298,6 +1403,7 @@ pub fn lex<'a, 'e>(input: &'a [&'a str], engine: &'e Engine) -> TokenIterator<'a
pos: Position::new(1, 0),
stream: MultiInputsStream {
streams: input.iter().map(|s| s.chars().peekable()).collect(),
index: 0,
},
}
}

29
src/utils.rs
View File

@@ -356,10 +356,13 @@ impl<T> StaticVec<T> {
panic!("nothing to pop!");
}
let result = if self.is_fixed_storage() {
self.extract_from_list(self.len - 1)
if self.is_fixed_storage() {
let value = self.extract_from_list(self.len - 1);
self.len -= 1;
value
} else {
let value = self.more.pop().unwrap();
self.len -= 1;
// Move back to the fixed list
if self.more.len() == MAX_STATIC_VEC {
@@ -370,11 +373,7 @@ impl<T> StaticVec<T> {
}
value
};
self.len -= 1;
result
}
}
/// Remove a value from this `StaticVec` at a particular position.
///
@@ -386,18 +385,20 @@ impl<T> StaticVec<T> {
panic!("index OOB in StaticVec");
}
let result = if self.is_fixed_storage() {
if self.is_fixed_storage() {
let value = self.extract_from_list(index);
// Move all items one slot to the left
for x in index..self.len - 1 {
let orig_value = self.extract_from_list(x + 1);
self.set_into_list(x, orig_value, false);
for x in index + 1..self.len - 1 {
let orig_value = self.extract_from_list(x);
self.set_into_list(x - 1, orig_value, false);
}
self.len -= 1;
value
} else {
let value = self.more.remove(index);
self.len -= 1;
// Move back to the fixed list
if self.more.len() == MAX_STATIC_VEC {
@@ -408,11 +409,7 @@ impl<T> StaticVec<T> {
}
value
};
self.len -= 1;
result
}
}
/// Get the number of items in this `StaticVec`.
#[inline(always)]