kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Store short index in variable access.

Browse Source
This commit is contained in:
Stephen Chung 2021-04-05 23:06:48 +08:00
parent a3ee0f4245
commit 94fc5af285
6 changed files with 106 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/ast.rs
View File

@ -9,7 +9,7 @@ use crate::stdlib::{
fmt, fmt,
hash::Hash, hash::Hash,
iter::empty, iter::empty,
num::NonZeroUsize, num::{NonZeroU8, NonZeroUsize},
ops::{Add, AddAssign}, ops::{Add, AddAssign},
string::String, string::String,
vec, vec,
@ -1574,8 +1574,15 @@ pub enum Expr {
), ),
/// () /// ()
Unit(Position), Unit(Position),
/// Variable access - (optional index, optional (hash, modules), variable name) /// Variable access - optional short index, (optional index, optional (hash, modules), variable name)
Variable(Box<(Option<NonZeroUsize>, Option<(u64, NamespaceRef)>, Ident)>), ///
/// The short index is [`u8`] which is used when the index is <= 255, which should be the vast
/// majority of cases (unless there are more than 255 variables defined!).
/// This is to avoid reading a pointer redirection during each variable access.
Variable(
Option<NonZeroU8>,
Box<(Option<NonZeroUsize>, Option<(u64, NamespaceRef)>, Ident)>,
),
/// Property access - ((getter, hash), (setter, hash), prop) /// Property access - ((getter, hash), (setter, hash), prop)
Property(Box<((Identifier, u64), (Identifier, u64), Ident)>), Property(Box<((Identifier, u64), (Identifier, u64), Ident)>),
/// { [statement][Stmt] ... } /// { [statement][Stmt] ... }
@ -1644,7 +1651,7 @@ impl Expr {
#[inline(always)] #[inline(always)]
pub(crate) fn get_variable_access(&self, non_qualified: bool) -> Option<&str> { pub(crate) fn get_variable_access(&self, non_qualified: bool) -> Option<&str> {
match self { match self {
Self::Variable(x) if !non_qualified || x.1.is_none() => Some((x.2).name.as_str()), Self::Variable(_, x) if !non_qualified || x.1.is_none() => Some((x.2).name.as_str()),
_ => None, _ => None,
} }
} }
@ -1666,7 +1673,7 @@ impl Expr {
Self::Map(_, pos) => *pos, Self::Map(_, pos) => *pos,
Self::Property(x) => (x.2).pos, Self::Property(x) => (x.2).pos,
Self::Stmt(x) => x.pos, Self::Stmt(x) => x.pos,
Self::Variable(x) => (x.2).pos, Self::Variable(_, x) => (x.2).pos,
Self::FnCall(_, pos) => *pos, Self::FnCall(_, pos) => *pos,
Self::And(x, _) | Self::Or(x, _) => x.lhs.position(), Self::And(x, _) | Self::Or(x, _) => x.lhs.position(),
@ -1696,7 +1703,7 @@ impl Expr {
Self::FnPointer(_, pos) => *pos = new_pos, Self::FnPointer(_, pos) => *pos = new_pos,
Self::Array(_, pos) => *pos = new_pos, Self::Array(_, pos) => *pos = new_pos,
Self::Map(_, pos) => *pos = new_pos, Self::Map(_, pos) => *pos = new_pos,
Self::Variable(x) => (x.2).pos = new_pos, Self::Variable(_, x) => (x.2).pos = new_pos,
Self::Property(x) => (x.2).pos = new_pos, Self::Property(x) => (x.2).pos = new_pos,
Self::Stmt(x) => x.pos = new_pos, Self::Stmt(x) => x.pos = new_pos,
Self::FnCall(_, pos) => *pos = new_pos, Self::FnCall(_, pos) => *pos = new_pos,
@ -1724,7 +1731,7 @@ impl Expr {
Self::Stmt(x) => x.statements.iter().all(Stmt::is_pure), Self::Stmt(x) => x.statements.iter().all(Stmt::is_pure),
Self::Variable(_) => true, Self::Variable(_, _) => true,
_ => self.is_constant(), _ => self.is_constant(),
} }
@ -1794,7 +1801,7 @@ impl Expr {
_ => false, _ => false,
}, },
Self::Variable(_) => match token { Self::Variable(_, _) => match token {
#[cfg(not(feature = "no_index"))] #[cfg(not(feature = "no_index"))]
Token::LeftBracket => true, Token::LeftBracket => true,
Token::LeftParen => true, Token::LeftParen => true,

35
src/engine.rs
View File

@ -959,7 +959,12 @@ impl Engine {
expr: &Expr, expr: &Expr,
) -> Result<(Target<'s>, Position), Box<EvalAltResult>> { ) -> Result<(Target<'s>, Position), Box<EvalAltResult>> {
match expr { match expr {
Expr::Variable(v) => match v.as_ref() { Expr::Variable(Some(_), _) => {
self.search_scope_only(scope, mods, state, lib, this_ptr, expr)
}
Expr::Variable(None, v) => match v.as_ref() {
// Normal variable access
(_, None, _) => self.search_scope_only(scope, mods, state, lib, this_ptr, expr),
// Qualified variable // Qualified variable
(_, Some((hash_var, modules)), Ident { name, pos, .. }) => { (_, Some((hash_var, modules)), Ident { name, pos, .. }) => {
let module = self.search_imports(mods, state, modules).ok_or_else(|| { let module = self.search_imports(mods, state, modules).ok_or_else(|| {
@ -983,8 +988,6 @@ impl Engine {
target.set_access_mode(AccessMode::ReadOnly); target.set_access_mode(AccessMode::ReadOnly);
Ok((target.into(), *pos)) Ok((target.into(), *pos))
} }
// Normal variable access
_ => self.search_scope_only(scope, mods, state, lib, this_ptr, expr),
}, },
_ => unreachable!("Expr::Variable expected, but gets {:?}", expr), _ => unreachable!("Expr::Variable expected, but gets {:?}", expr),
} }
@ -1000,8 +1003,8 @@ impl Engine {
this_ptr: &'s mut Option<&mut Dynamic>, this_ptr: &'s mut Option<&mut Dynamic>,
expr: &Expr, expr: &Expr,
) -> Result<(Target<'s>, Position), Box<EvalAltResult>> { ) -> Result<(Target<'s>, Position), Box<EvalAltResult>> {
let (index, _, Ident { name, pos, .. }) = match expr { let (short_index, (index, _, Ident { name, pos, .. })) = match expr {
Expr::Variable(v) => v.as_ref(), Expr::Variable(i, v) => (i, v.as_ref()),
_ => unreachable!("Expr::Variable expected, but gets {:?}", expr), _ => unreachable!("Expr::Variable expected, but gets {:?}", expr),
}; };
@ -1015,11 +1018,17 @@ impl Engine {
} }
// Check if it is directly indexed // Check if it is directly indexed
let index = if state.always_search { &None } else { index }; let index = if state.always_search {
0
} else {
short_index.map_or_else(
|| index.map(NonZeroUsize::get).unwrap_or(0),
|x| x.get() as usize,
)
};
// Check the variable resolver, if any // Check the variable resolver, if any
if let Some(ref resolve_var) = self.resolve_var { if let Some(ref resolve_var) = self.resolve_var {
let index = index.map(NonZeroUsize::get).unwrap_or(0);
let context = EvalContext { let context = EvalContext {
engine: self, engine: self,
scope, scope,
@ -1037,8 +1046,8 @@ impl Engine {
} }
} }
let index = if let Some(index) = index { let index = if index > 0 {
scope.len() - index.get() scope.len() - index
} else { } else {
// Find the variable in the scope // Find the variable in the scope
scope scope
@ -1401,7 +1410,7 @@ impl Engine {
match lhs { match lhs {
// id.??? or id[???] // id.??? or id[???]
Expr::Variable(x) => { Expr::Variable(_, x) => {
let Ident { let Ident {
name: var_name, name: var_name,
pos: var_pos, pos: var_pos,
@ -1679,11 +1688,11 @@ impl Engine {
Expr::CharConstant(x, _) => Ok((*x).into()), Expr::CharConstant(x, _) => Ok((*x).into()),
Expr::FnPointer(x, _) => Ok(FnPtr::new_unchecked(x.clone(), Default::default()).into()), Expr::FnPointer(x, _) => Ok(FnPtr::new_unchecked(x.clone(), Default::default()).into()),
Expr::Variable(x) if (x.2).name == KEYWORD_THIS => this_ptr Expr::Variable(None, x) if x.0.is_none() && (x.2).name == KEYWORD_THIS => this_ptr
.as_deref() .as_deref()
.cloned() .cloned()
.ok_or_else(|| EvalAltResult::ErrorUnboundThis((x.2).pos).into()), .ok_or_else(|| EvalAltResult::ErrorUnboundThis((x.2).pos).into()),
Expr::Variable(_) => self Expr::Variable(_, _) => self
.search_namespace(scope, mods, state, lib, this_ptr, expr) .search_namespace(scope, mods, state, lib, this_ptr, expr)
.map(|(val, _)| val.take_or_clone()), .map(|(val, _)| val.take_or_clone()),
@ -2061,7 +2070,7 @@ impl Engine {
// Must be either `var[index] op= val` or `var.prop op= val` // Must be either `var[index] op= val` or `var.prop op= val`
match lhs_expr { match lhs_expr {
// name op= rhs (handled above) // name op= rhs (handled above)
Expr::Variable(_) => { Expr::Variable(_, _) => {
unreachable!("Expr::Variable case should already been handled") unreachable!("Expr::Variable case should already been handled")
} }
// idx_lhs[idx_expr] op= rhs // idx_lhs[idx_expr] op= rhs

4
src/fn_call.rs
View File

@ -750,7 +750,7 @@ impl Engine {
// Method call of script function - map first argument to `this` // Method call of script function - map first argument to `this`
let (first, rest) = args.split_first_mut().unwrap(); let (first, rest) = args.split_first_mut().unwrap();
let orig_source = mem::take(&mut state.source); let orig_source = state.source.take();
state.source = source; state.source = source;
let level = _level + 1; let level = _level + 1;
@ -780,7 +780,7 @@ impl Engine {
backup.as_mut().unwrap().change_first_arg_to_copy(args); backup.as_mut().unwrap().change_first_arg_to_copy(args);
} }
let orig_source = mem::take(&mut state.source); let orig_source = state.source.take();
state.source = source; state.source = source;
let level = _level + 1; let level = _level + 1;

8
src/optimize.rs
View File

@ -385,7 +385,7 @@ fn optimize_stmt(stmt: &mut Stmt, state: &mut State, preserve_result: bool) {
match stmt { match stmt {
// expr op= expr // expr op= expr
Stmt::Assignment(x, _) => match x.0 { Stmt::Assignment(x, _) => match x.0 {
Expr::Variable(_) => optimize_expr(&mut x.2, state), Expr::Variable(_, _) => optimize_expr(&mut x.2, state),
_ => { _ => {
optimize_expr(&mut x.0, state); optimize_expr(&mut x.0, state);
optimize_expr(&mut x.2, state); optimize_expr(&mut x.2, state);
@ -635,7 +635,7 @@ fn optimize_expr(expr: &mut Expr, state: &mut State) {
.unwrap_or_else(|| Expr::Unit(*pos)); .unwrap_or_else(|| Expr::Unit(*pos));
} }
// var.rhs // var.rhs
(Expr::Variable(_), rhs) => optimize_expr(rhs, state), (Expr::Variable(_, _), rhs) => optimize_expr(rhs, state),
// lhs.rhs // lhs.rhs
(lhs, rhs) => { optimize_expr(lhs, state); optimize_expr(rhs, state); } (lhs, rhs) => { optimize_expr(lhs, state); optimize_expr(rhs, state); }
} }
@ -670,7 +670,7 @@ fn optimize_expr(expr: &mut Expr, state: &mut State) {
*expr = Expr::CharConstant(s.chars().nth(*i as usize).unwrap(), *pos); *expr = Expr::CharConstant(s.chars().nth(*i as usize).unwrap(), *pos);
} }
// var[rhs] // var[rhs]
(Expr::Variable(_), rhs) => optimize_expr(rhs, state), (Expr::Variable(_, _), rhs) => optimize_expr(rhs, state),
// lhs[rhs] // lhs[rhs]
(lhs, rhs) => { optimize_expr(lhs, state); optimize_expr(rhs, state); } (lhs, rhs) => { optimize_expr(lhs, state); optimize_expr(rhs, state); }
}, },
@ -901,7 +901,7 @@ fn optimize_expr(expr: &mut Expr, state: &mut State) {
} }
// constant-name // constant-name
Expr::Variable(x) if x.1.is_none() && state.find_constant(&x.2.name).is_some() => { Expr::Variable(_, x) if x.1.is_none() && state.find_constant(&x.2.name).is_some() => {
state.set_dirty(); state.set_dirty();
// Replace constant with value // Replace constant with value

64
src/parser.rs
View File

@ -15,7 +15,7 @@ use crate::stdlib::{
format, format,
hash::{Hash, Hasher}, hash::{Hash, Hasher},
iter::empty, iter::empty,
num::NonZeroUsize, num::{NonZeroU8, NonZeroUsize},
string::{String, ToString}, string::{String, ToString},
vec, vec,
vec::Vec, vec::Vec,
@ -225,7 +225,7 @@ impl Expr {
#[inline(always)] #[inline(always)]
fn into_property(self, state: &mut ParseState) -> Self { fn into_property(self, state: &mut ParseState) -> Self {
match self { match self {
Self::Variable(x) if x.1.is_none() => { Self::Variable(_, x) if x.1.is_none() => {
let ident = x.2; let ident = x.2;
let getter = state.get_identifier(crate::engine::make_getter(&ident.name)); let getter = state.get_identifier(crate::engine::make_getter(&ident.name));
let hash_get = calc_fn_hash(empty(), &getter, 1); let hash_get = calc_fn_hash(empty(), &getter, 1);
@ -1081,7 +1081,7 @@ fn parse_primary(
name: state.get_identifier(s), name: state.get_identifier(s),
pos: settings.pos, pos: settings.pos,
}; };
Expr::Variable(Box::new((None, None, var_name_def))) Expr::Variable(None, Box::new((None, None, var_name_def)))
} }
// Namespace qualification // Namespace qualification
#[cfg(not(feature = "no_module"))] #[cfg(not(feature = "no_module"))]
@ -1095,7 +1095,7 @@ fn parse_primary(
name: state.get_identifier(s), name: state.get_identifier(s),
pos: settings.pos, pos: settings.pos,
}; };
Expr::Variable(Box::new((None, None, var_name_def))) Expr::Variable(None, Box::new((None, None, var_name_def)))
} }
// Normal variable access // Normal variable access
_ => { _ => {
@ -1104,7 +1104,14 @@ fn parse_primary(
name: state.get_identifier(s), name: state.get_identifier(s),
pos: settings.pos, pos: settings.pos,
}; };
Expr::Variable(Box::new((index, None, var_name_def))) let short_index = index.and_then(|x| {
if x.get() <= u8::MAX as usize {
NonZeroU8::new(x.get() as u8)
} else {
None
}
});
Expr::Variable(short_index, Box::new((index, None, var_name_def)))
} }
} }
} }
@ -1123,7 +1130,7 @@ fn parse_primary(
name: state.get_identifier(s), name: state.get_identifier(s),
pos: settings.pos, pos: settings.pos,
}; };
Expr::Variable(Box::new((None, None, var_name_def))) Expr::Variable(None, Box::new((None, None, var_name_def)))
} }
// Access to `this` as a variable is OK within a function scope // Access to `this` as a variable is OK within a function scope
_ if s == KEYWORD_THIS && settings.is_function_scope => { _ if s == KEYWORD_THIS && settings.is_function_scope => {
@ -1131,7 +1138,7 @@ fn parse_primary(
name: state.get_identifier(s), name: state.get_identifier(s),
pos: settings.pos, pos: settings.pos,
}; };
Expr::Variable(Box::new((None, None, var_name_def))) Expr::Variable(None, Box::new((None, None, var_name_def)))
} }
// Cannot access to `this` as a variable not in a function scope // Cannot access to `this` as a variable not in a function scope
_ if s == KEYWORD_THIS => { _ if s == KEYWORD_THIS => {
@ -1168,7 +1175,7 @@ fn parse_primary(
root_expr = match (root_expr, tail_token) { root_expr = match (root_expr, tail_token) {
// Qualified function call with ! // Qualified function call with !
(Expr::Variable(x), Token::Bang) if x.1.is_some() => { (Expr::Variable(_, x), Token::Bang) if x.1.is_some() => {
return Err(if !match_token(input, Token::LeftParen).0 { return Err(if !match_token(input, Token::LeftParen).0 {
LexError::UnexpectedInput(Token::Bang.syntax().to_string()).into_err(tail_pos) LexError::UnexpectedInput(Token::Bang.syntax().to_string()).into_err(tail_pos)
} else { } else {
@ -1180,7 +1187,7 @@ fn parse_primary(
}); });
} }
// Function call with ! // Function call with !
(Expr::Variable(x), Token::Bang) => { (Expr::Variable(_, x), Token::Bang) => {
let (matched, pos) = match_token(input, Token::LeftParen); let (matched, pos) = match_token(input, Token::LeftParen);
if !matched { if !matched {
return Err(PERR::MissingToken( return Err(PERR::MissingToken(
@ -1196,31 +1203,30 @@ fn parse_primary(
parse_fn_call(input, state, lib, name, true, ns, settings.level_up())? parse_fn_call(input, state, lib, name, true, ns, settings.level_up())?
} }
// Function call // Function call
(Expr::Variable(x), Token::LeftParen) => { (Expr::Variable(_, x), Token::LeftParen) => {
let (_, namespace, Ident { name, pos, .. }) = *x; let (_, namespace, Ident { name, pos, .. }) = *x;
settings.pos = pos; settings.pos = pos;
let ns = namespace.map(|(_, ns)| ns); let ns = namespace.map(|(_, ns)| ns);
parse_fn_call(input, state, lib, name, false, ns, settings.level_up())? parse_fn_call(input, state, lib, name, false, ns, settings.level_up())?
} }
// module access // module access
(Expr::Variable(x), Token::DoubleColon) => match input.next().unwrap() { (Expr::Variable(_, x), Token::DoubleColon) => match input.next().unwrap() {
(Token::Identifier(id2), pos2) => { (Token::Identifier(id2), pos2) => {
let (index, mut namespace, var_name_def) = *x; let (_, mut namespace, var_name_def) = *x;
if let Some((_, ref mut namespace)) = namespace { if let Some((_, ref mut namespace)) = namespace {
namespace.push(var_name_def); namespace.push(var_name_def);
} else { } else {
let mut ns: NamespaceRef = Default::default(); let mut ns: NamespaceRef = Default::default();
ns.push(var_name_def); ns.push(var_name_def);
let index = 42; // Dummy namespace = Some((42, ns));
namespace = Some((index, ns));
} }
let var_name_def = Ident { let var_name_def = Ident {
name: state.get_identifier(id2), name: state.get_identifier(id2),
pos: pos2, pos: pos2,
}; };
Expr::Variable(Box::new((index, namespace, var_name_def))) Expr::Variable(None, Box::new((None, namespace, var_name_def)))
} }
(Token::Reserved(id2), pos2) if is_valid_identifier(id2.chars()) => { (Token::Reserved(id2), pos2) if is_valid_identifier(id2.chars()) => {
return Err(PERR::Reserved(id2).into_err(pos2)); return Err(PERR::Reserved(id2).into_err(pos2));
@ -1263,9 +1269,9 @@ fn parse_primary(
// Cache the hash key for namespace-qualified variables // Cache the hash key for namespace-qualified variables
match &mut root_expr { match &mut root_expr {
Expr::Variable(x) if x.1.is_some() => Some(x), Expr::Variable(_, x) if x.1.is_some() => Some(x),
Expr::Index(x, _) | Expr::Dot(x, _) => match &mut x.lhs { Expr::Index(x, _) | Expr::Dot(x, _) => match &mut x.lhs {
Expr::Variable(x) if x.1.is_some() => Some(x), Expr::Variable(_, x) if x.1.is_some() => Some(x),
_ => None, _ => None,
}, },
_ => None, _ => None,
@ -1423,13 +1429,14 @@ fn make_assignment_stmt<'a>(
Err(PERR::AssignmentToConstant("".into()).into_err(lhs.position())) Err(PERR::AssignmentToConstant("".into()).into_err(lhs.position()))
} }
// var (non-indexed) = rhs // var (non-indexed) = rhs
Expr::Variable(x) if x.0.is_none() => { Expr::Variable(None, x) if x.0.is_none() => {
Ok(Stmt::Assignment(Box::new((lhs, op_info, rhs)), op_pos)) Ok(Stmt::Assignment(Box::new((lhs, op_info, rhs)), op_pos))
} }
// var (indexed) = rhs // var (indexed) = rhs
Expr::Variable(x) => { Expr::Variable(i, x) => {
let (index, _, Ident { name, pos, .. }) = x.as_ref(); let (index, _, Ident { name, pos, .. }) = x.as_ref();
match state.stack[(state.stack.len() - index.unwrap().get())].1 { let index = i.map_or_else(|| index.unwrap().get(), |n| n.get() as usize);
match state.stack[state.stack.len() - index].1 {
AccessMode::ReadWrite => { AccessMode::ReadWrite => {
Ok(Stmt::Assignment(Box::new((lhs, op_info, rhs)), op_pos)) Ok(Stmt::Assignment(Box::new((lhs, op_info, rhs)), op_pos))
} }
@ -1444,13 +1451,14 @@ fn make_assignment_stmt<'a>(
match check_lvalue(&x.rhs, matches!(lhs, Expr::Dot(_, _))) { match check_lvalue(&x.rhs, matches!(lhs, Expr::Dot(_, _))) {
Position::NONE => match &x.lhs { Position::NONE => match &x.lhs {
// var[???] (non-indexed) = rhs, var.??? (non-indexed) = rhs // var[???] (non-indexed) = rhs, var.??? (non-indexed) = rhs
Expr::Variable(x) if x.0.is_none() => { Expr::Variable(None, x) if x.0.is_none() => {
Ok(Stmt::Assignment(Box::new((lhs, op_info, rhs)), op_pos)) Ok(Stmt::Assignment(Box::new((lhs, op_info, rhs)), op_pos))
} }
// var[???] (indexed) = rhs, var.??? (indexed) = rhs // var[???] (indexed) = rhs, var.??? (indexed) = rhs
Expr::Variable(x) => { Expr::Variable(i, x) => {
let (index, _, Ident { name, pos, .. }) = x.as_ref(); let (index, _, Ident { name, pos, .. }) = x.as_ref();
match state.stack[(state.stack.len() - index.unwrap().get())].1 { let index = i.map_or_else(|| index.unwrap().get(), |n| n.get() as usize);
match state.stack[state.stack.len() - index].1 {
AccessMode::ReadWrite => { AccessMode::ReadWrite => {
Ok(Stmt::Assignment(Box::new((lhs, op_info, rhs)), op_pos)) Ok(Stmt::Assignment(Box::new((lhs, op_info, rhs)), op_pos))
} }
@ -1532,7 +1540,7 @@ fn make_dot_expr(
Expr::Index(x, pos) Expr::Index(x, pos)
} }
// lhs.id // lhs.id
(lhs, Expr::Variable(x)) if x.1.is_none() => { (lhs, Expr::Variable(_, x)) if x.1.is_none() => {
let ident = x.2; let ident = x.2;
let getter = state.get_identifier(crate::engine::make_getter(&ident.name)); let getter = state.get_identifier(crate::engine::make_getter(&ident.name));
let hash_get = calc_fn_hash(empty(), &getter, 1); let hash_get = calc_fn_hash(empty(), &getter, 1);
@ -1544,7 +1552,7 @@ fn make_dot_expr(
Expr::Dot(Box::new(BinaryExpr { lhs, rhs }), op_pos) Expr::Dot(Box::new(BinaryExpr { lhs, rhs }), op_pos)
} }
// lhs.module::id - syntax error // lhs.module::id - syntax error
(_, Expr::Variable(x)) if x.1.is_some() => { (_, Expr::Variable(_, x)) if x.1.is_some() => {
return Err(PERR::PropertyExpected.into_err(x.1.unwrap().1[0].pos)) return Err(PERR::PropertyExpected.into_err(x.1.unwrap().1[0].pos))
} }
// lhs.prop // lhs.prop
@ -1553,7 +1561,7 @@ fn make_dot_expr(
} }
// lhs.dot_lhs.dot_rhs // lhs.dot_lhs.dot_rhs
(lhs, Expr::Dot(x, pos)) => match x.lhs { (lhs, Expr::Dot(x, pos)) => match x.lhs {
Expr::Variable(_) | Expr::Property(_) => { Expr::Variable(_, _) | Expr::Property(_) => {
let rhs = Expr::Dot( let rhs = Expr::Dot(
Box::new(BinaryExpr { Box::new(BinaryExpr {
lhs: x.lhs.into_property(state), lhs: x.lhs.into_property(state),
@ -1869,7 +1877,7 @@ fn parse_custom_syntax(
segments.push(name.clone().into()); segments.push(name.clone().into());
tokens.push(state.get_identifier(MARKER_IDENT)); tokens.push(state.get_identifier(MARKER_IDENT));
let var_name_def = Ident { name, pos }; let var_name_def = Ident { name, pos };
keywords.push(Expr::Variable(Box::new((None, None, var_name_def)))); keywords.push(Expr::Variable(None, Box::new((None, None, var_name_def))));
} }
(Token::Reserved(s), pos) if is_valid_identifier(s.chars()) => { (Token::Reserved(s), pos) if is_valid_identifier(s.chars()) => {
return Err(PERR::Reserved(s).into_err(pos)); return Err(PERR::Reserved(s).into_err(pos));
@ -2825,7 +2833,7 @@ fn make_curry_from_externals(
name: x.clone(), name: x.clone(),
pos: Position::NONE, pos: Position::NONE,
}; };
args.push(Expr::Variable(Box::new((None, None, var_def)))); args.push(Expr::Variable(None, Box::new((None, None, var_def))));
}); });
let expr = Expr::FnCall( let expr = Expr::FnCall(

29
src/token.rs
View File

@ -9,7 +9,6 @@ use crate::stdlib::{
cell::Cell, cell::Cell,
char, fmt, format, char, fmt, format,
iter::{FusedIterator, Peekable}, iter::{FusedIterator, Peekable},
mem,
num::NonZeroUsize, num::NonZeroUsize,
ops::{Add, AddAssign}, ops::{Add, AddAssign},
rc::Rc, rc::Rc,
@ -828,7 +827,7 @@ impl From<Token> for String {
/// This type is volatile and may change. /// This type is volatile and may change.
#[derive(Debug, Clone, Eq, PartialEq, Default)] #[derive(Debug, Clone, Eq, PartialEq, Default)]
pub struct TokenizeState { pub struct TokenizeState {
/// Maximum length of a string (0 = unlimited). /// Maximum length of a string.
pub max_string_size: Option<NonZeroUsize>, pub max_string_size: Option<NonZeroUsize>,
/// Can the next token be a unary operator? /// Can the next token be a unary operator?
pub non_unary: bool, pub non_unary: bool,
@ -1187,7 +1186,7 @@ fn get_next_token_inner(
} }
// Within text? // Within text?
if let Some(ch) = mem::take(&mut state.is_within_text_terminated_by) { if let Some(ch) = state.is_within_text_terminated_by.take() {
let start_pos = *pos; let start_pos = *pos;
return parse_string_literal(stream, state, pos, ch, false, true, true, true).map_or_else( return parse_string_literal(stream, state, pos, ch, false, true, true, true).map_or_else(
@ -1316,17 +1315,16 @@ fn get_next_token_inner(
} }
// Parse number // Parse number
if let Some(radix) = radix_base {
let out: String = result.iter().skip(2).filter(|&&c| c != NUM_SEP).collect();
return Some(( return Some((
if let Some(radix) = radix_base {
let out: String =
result.iter().skip(2).filter(|&&c| c != NUM_SEP).collect();
INT::from_str_radix(&out, radix) INT::from_str_radix(&out, radix)
.map(Token::IntegerConstant) .map(Token::IntegerConstant)
.unwrap_or_else(|_| { .unwrap_or_else(|_| {
Token::LexError(LERR::MalformedNumber(result.into_iter().collect())) Token::LexError(LERR::MalformedNumber(result.into_iter().collect()))
}), })
start_pos,
));
} else { } else {
let out: String = result.iter().filter(|&&c| c != NUM_SEP).collect(); let out: String = result.iter().filter(|&&c| c != NUM_SEP).collect();
let num = INT::from_str(&out).map(Token::IntegerConstant); let num = INT::from_str(&out).map(Token::IntegerConstant);
@ -1338,21 +1336,22 @@ fn get_next_token_inner(
// Then try decimal // Then try decimal
#[cfg(feature = "decimal")] #[cfg(feature = "decimal")]
let num = num.or_else(|_| Decimal::from_str(&out).map(Token::DecimalConstant)); let num =
num.or_else(|_| Decimal::from_str(&out).map(Token::DecimalConstant));
// Then try decimal in scientific notation // Then try decimal in scientific notation
#[cfg(feature = "decimal")] #[cfg(feature = "decimal")]
let num = let num = num.or_else(|_| {
num.or_else(|_| Decimal::from_scientific(&out).map(Token::DecimalConstant)); Decimal::from_scientific(&out).map(Token::DecimalConstant)
});
return Some((
num.unwrap_or_else(|_| { num.unwrap_or_else(|_| {
Token::LexError(LERR::MalformedNumber(result.into_iter().collect())) Token::LexError(LERR::MalformedNumber(result.into_iter().collect()))
}), })
},
start_pos, start_pos,
)); ));
} }
}
// letter or underscore ... // letter or underscore ...
#[cfg(not(feature = "unicode-xid-ident"))] #[cfg(not(feature = "unicode-xid-ident"))]