kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Code cleanup.

Browse Source
This commit is contained in:
Stephen Chung 2022-10-30 22:16:09 +08:00
parent dc5e80c821
commit 32493524ed
9 changed files with 144 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/api/custom_syntax.rs
View File

@ -228,7 +228,13 @@ impl Engine {
continue; continue;
} }
let token = Token::lookup_from_syntax(s); let token = Token::lookup_symbol_from_syntax(s).or_else(|| {
if Token::is_reserved_keyword(s) {
Some(Token::Reserved(Box::new(s.into())))
} else {
None
}
});
let seg = match s { let seg = match s {
// Markers not in first position // Markers not in first position

2
src/api/mod.rs
View File

@ -154,7 +154,7 @@ impl Engine {
let keyword = keyword.as_ref(); let keyword = keyword.as_ref();
match Token::lookup_from_syntax(keyword) { match Token::lookup_symbol_from_syntax(keyword) {
// Standard identifiers and reserved keywords are OK // Standard identifiers and reserved keywords are OK
None | Some(Token::Reserved(..)) => (), None | Some(Token::Reserved(..)) => (),
// custom keywords are OK // custom keywords are OK

7
src/ast/stmt.rs
View File

@ -61,7 +61,10 @@ impl OpAssignment {
#[must_use] #[must_use]
#[inline(always)] #[inline(always)]
pub fn new_op_assignment(name: &str, pos: Position) -> Self { pub fn new_op_assignment(name: &str, pos: Position) -> Self {
Self::new_op_assignment_from_token(&Token::lookup_from_syntax(name).expect("operator"), pos) Self::new_op_assignment_from_token(
&Token::lookup_symbol_from_syntax(name).expect("operator"),
pos,
)
} }
/// Create a new [`OpAssignment`] from a [`Token`]. /// Create a new [`OpAssignment`] from a [`Token`].
/// ///
@ -90,7 +93,7 @@ impl OpAssignment {
#[inline(always)] #[inline(always)]
pub fn new_op_assignment_from_base(name: &str, pos: Position) -> Self { pub fn new_op_assignment_from_base(name: &str, pos: Position) -> Self {
Self::new_op_assignment_from_base_token( Self::new_op_assignment_from_base_token(
&Token::lookup_from_syntax(name).expect("operator"), &Token::lookup_symbol_from_syntax(name).expect("operator"),
pos, pos,
) )
} }

20
src/eval/chaining.rs
View File

@ -270,8 +270,8 @@ impl Engine {
let args = &mut [target.as_mut()]; let args = &mut [target.as_mut()];
let (mut orig_val, ..) = self let (mut orig_val, ..) = self
.exec_native_fn_call( .exec_native_fn_call(
global, caches, lib, getter, *hash_get, args, is_ref_mut, global, caches, lib, getter, None, *hash_get, args, is_ref_mut,
false, *pos, level, *pos, level,
) )
.or_else(|err| match *err { .or_else(|err| match *err {
// Try an indexer if property does not exist // Try an indexer if property does not exist
@ -304,7 +304,7 @@ impl Engine {
let args = &mut [target.as_mut(), &mut new_val]; let args = &mut [target.as_mut(), &mut new_val];
self.exec_native_fn_call( self.exec_native_fn_call(
global, caches, lib, setter, *hash_set, args, is_ref_mut, false, *pos, global, caches, lib, setter, None, *hash_set, args, is_ref_mut, *pos,
level, level,
) )
.or_else(|err| match *err { .or_else(|err| match *err {
@ -331,7 +331,7 @@ impl Engine {
let ((getter, hash_get), _, name) = &**x; let ((getter, hash_get), _, name) = &**x;
let args = &mut [target.as_mut()]; let args = &mut [target.as_mut()];
self.exec_native_fn_call( self.exec_native_fn_call(
global, caches, lib, getter, *hash_get, args, is_ref_mut, false, *pos, global, caches, lib, getter, None, *hash_get, args, is_ref_mut, *pos,
level, level,
) )
.map_or_else( .map_or_else(
@ -430,8 +430,8 @@ impl Engine {
// Assume getters are always pure // Assume getters are always pure
let (mut val, ..) = self let (mut val, ..) = self
.exec_native_fn_call( .exec_native_fn_call(
global, caches, lib, getter, *hash_get, args, is_ref_mut, global, caches, lib, getter, None, *hash_get, args,
false, pos, level, is_ref_mut, pos, level,
) )
.or_else(|err| match *err { .or_else(|err| match *err {
// Try an indexer if property does not exist // Try an indexer if property does not exist
@ -466,8 +466,8 @@ impl Engine {
let mut arg_values = [target.as_mut(), val.as_mut()]; let mut arg_values = [target.as_mut(), val.as_mut()];
let args = &mut arg_values; let args = &mut arg_values;
self.exec_native_fn_call( self.exec_native_fn_call(
global, caches, lib, setter, *hash_set, args, is_ref_mut, global, caches, lib, setter, None, *hash_set, args,
false, pos, level, is_ref_mut, pos, level,
) )
.or_else( .or_else(
|err| match *err { |err| match *err {
@ -765,7 +765,7 @@ impl Engine {
let level = level + 1; let level = level + 1;
self.exec_native_fn_call( self.exec_native_fn_call(
global, caches, lib, fn_name, hash, args, true, false, pos, level, global, caches, lib, fn_name, None, hash, args, true, pos, level,
) )
.map(|(r, ..)| r) .map(|(r, ..)| r)
} }
@ -790,7 +790,7 @@ impl Engine {
let level = level + 1; let level = level + 1;
self.exec_native_fn_call( self.exec_native_fn_call(
global, caches, lib, fn_name, hash, args, is_ref_mut, false, pos, level, global, caches, lib, fn_name, None, hash, args, is_ref_mut, pos, level,
) )
} }

20
src/eval/stmt.rs
View File

@ -130,8 +130,8 @@ impl Engine {
let OpAssignment { let OpAssignment {
hash_op_assign, hash_op_assign,
hash_op, hash_op,
op_assign, op_assign: op_assign_token,
op, op: op_token,
pos: op_pos, pos: op_pos,
} = op_info; } = op_info;
@ -142,27 +142,31 @@ impl Engine {
let level = level + 1; let level = level + 1;
if self.fast_operators() { if self.fast_operators() {
if let Some(func) = get_builtin_op_assignment_fn(op_assign, args[0], args[1]) { if let Some(func) = get_builtin_op_assignment_fn(op_assign_token, args[0], args[1])
{
// Built-in found // Built-in found
let op = op_assign.literal_syntax(); let op = op_assign_token.literal_syntax();
let context = (self, op, None, &*global, lib, *op_pos, level).into(); let context = (self, op, None, &*global, lib, *op_pos, level).into();
return func(context, args).map(|_| ()); return func(context, args).map(|_| ());
} }
} }
let op_assign = op_assign.literal_syntax(); let op_assign = op_assign_token.literal_syntax();
let op = op.literal_syntax(); let op = op_token.literal_syntax();
let token = Some(op_assign_token);
match self.exec_native_fn_call( match self.exec_native_fn_call(
global, caches, lib, op_assign, hash, args, true, true, *op_pos, level, global, caches, lib, op_assign, token, hash, args, true, *op_pos, level,
) { ) {
Ok(_) => (), Ok(_) => (),
Err(err) if matches!(*err, ERR::ErrorFunctionNotFound(ref f, ..) if f.starts_with(op_assign)) => Err(err) if matches!(*err, ERR::ErrorFunctionNotFound(ref f, ..) if f.starts_with(op_assign)) =>
{ {
// Expand to `var = var op rhs` // Expand to `var = var op rhs`
let token = Some(op_token);
*args[0] = self *args[0] = self
.exec_native_fn_call( .exec_native_fn_call(
global, caches, lib, op, *hash_op, args, true, false, *op_pos, level, global, caches, lib, op, token, *hash_op, args, true, *op_pos, level,
) )
.map_err(|err| err.fill_position(op_info.pos))? .map_err(|err| err.fill_position(op_info.pos))?
.0 .0

130
src/func/call.rs
View File

@ -127,48 +127,24 @@ pub fn ensure_no_data_race(fn_name: &str, args: &FnCallArgs, is_ref_mut: bool) -
Ok(()) Ok(())
} }
impl Engine {
/// Generate the signature for a function call. /// Generate the signature for a function call.
#[inline] #[inline]
#[must_use] #[must_use]
fn gen_fn_call_signature(engine: &Engine, fn_name: &str, args: &[&mut Dynamic]) -> String { fn gen_fn_call_signature(&self, fn_name: &str, args: &[&mut Dynamic]) -> String {
format!( format!(
"{fn_name} ({})", "{fn_name} ({})",
args.iter() args.iter()
.map(|a| if a.is::<ImmutableString>() { .map(|a| if a.is::<ImmutableString>() {
"&str | ImmutableString | String" "&str | ImmutableString | String"
} else { } else {
engine.map_type_name(a.type_name()) self.map_type_name(a.type_name())
}) })
.collect::<FnArgsVec<_>>() .collect::<FnArgsVec<_>>()
.join(", ") .join(", ")
) )
} }
/// Generate the signature for a namespace-qualified function call.
///
/// Not available under `no_module`.
#[cfg(not(feature = "no_module"))]
#[inline]
#[must_use]
fn gen_qualified_fn_call_signature(
engine: &Engine,
namespace: &crate::ast::Namespace,
fn_name: &str,
args: &[&mut Dynamic],
) -> String {
let (ns, sep) = (
namespace.to_string(),
if namespace.is_empty() {
""
} else {
crate::tokenizer::Token::DoubleColon.literal_syntax()
},
);
format!("{ns}{sep}{}", gen_fn_call_signature(engine, fn_name, args))
}
impl Engine {
/// Resolve a normal (non-qualified) function call. /// Resolve a normal (non-qualified) function call.
/// ///
/// Search order: /// Search order:
@ -184,11 +160,10 @@ impl Engine {
caches: &'s mut Caches, caches: &'s mut Caches,
local_entry: &'s mut Option<FnResolutionCacheEntry>, local_entry: &'s mut Option<FnResolutionCacheEntry>,
lib: &[&Module], lib: &[&Module],
fn_name: &str, op_token: Option<&Token>,
hash_base: u64, hash_base: u64,
args: Option<&mut FnCallArgs>, args: Option<&mut FnCallArgs>,
allow_dynamic: bool, allow_dynamic: bool,
op_assignment_token: Option<&Token>,
) -> Option<&'s FnResolutionCacheEntry> { ) -> Option<&'s FnResolutionCacheEntry> {
if hash_base == 0 { if hash_base == 0 {
return None; return None;
@ -273,25 +248,24 @@ impl Engine {
} }
// Try to find a built-in version // Try to find a built-in version
let builtin = args.and_then(|args| { let builtin =
if let Some(op_assign) = op_assignment_token { args.and_then(|args| match op_token {
Some(token) if token.is_op_assignment() => {
let (first_arg, rest_args) = args.split_first().unwrap(); let (first_arg, rest_args) = args.split_first().unwrap();
get_builtin_op_assignment_fn(op_assign, *first_arg, rest_args[0]) get_builtin_op_assignment_fn(token, *first_arg, rest_args[0])
.map(|f| FnResolutionCacheEntry { .map(|f| FnResolutionCacheEntry {
func: CallableFunction::from_fn_builtin(f), func: CallableFunction::from_fn_builtin(f),
source: None, source: None,
}) })
} else if let Some(ref operator) = Token::lookup_from_syntax(fn_name) { }
get_builtin_binary_op_fn(operator, args[0], args[1]).map(|f| { Some(token) => get_builtin_binary_op_fn(token, args[0], args[1])
FnResolutionCacheEntry { .map(|f| FnResolutionCacheEntry {
func: CallableFunction::from_fn_builtin(f), func: CallableFunction::from_fn_builtin(f),
source: None, source: None,
} }),
})
} else { None => None,
None
}
}); });
return if cache.filter.is_absent_and_set(hash) { return if cache.filter.is_absent_and_set(hash) {
@ -345,21 +319,15 @@ impl Engine {
caches: &mut Caches, caches: &mut Caches,
lib: &[&Module], lib: &[&Module],
name: &str, name: &str,
op_token: Option<&Token>,
hash: u64, hash: u64,
args: &mut FnCallArgs, args: &mut FnCallArgs,
is_ref_mut: bool, is_ref_mut: bool,
is_op_assign: bool,
pos: Position, pos: Position,
level: usize, level: usize,
) -> RhaiResultOf<(Dynamic, bool)> { ) -> RhaiResultOf<(Dynamic, bool)> {
self.track_operation(global, pos)?; self.track_operation(global, pos)?;
let op_assign = if is_op_assign {
Token::lookup_from_syntax(name)
} else {
None
};
// Check if function access already in the cache // Check if function access already in the cache
let local_entry = &mut None; let local_entry = &mut None;
@ -368,11 +336,10 @@ impl Engine {
caches, caches,
local_entry, local_entry,
lib, lib,
name, op_token,
hash, hash,
Some(args), Some(args),
true, true,
op_assign.as_ref(),
); );
if func.is_some() { if func.is_some() {
@ -543,7 +510,7 @@ impl Engine {
// Raise error // Raise error
_ => { _ => {
Err(ERR::ErrorFunctionNotFound(gen_fn_call_signature(self, name, args), pos).into()) Err(ERR::ErrorFunctionNotFound(self.gen_fn_call_signature(name, args), pos).into())
} }
} }
} }
@ -566,6 +533,7 @@ impl Engine {
caches: &mut Caches, caches: &mut Caches,
lib: &[&Module], lib: &[&Module],
fn_name: &str, fn_name: &str,
op_token: Option<&Token>,
hashes: FnCallHashes, hashes: FnCallHashes,
args: &mut FnCallArgs, args: &mut FnCallArgs,
is_ref_mut: bool, is_ref_mut: bool,
@ -646,11 +614,10 @@ impl Engine {
caches, caches,
local_entry, local_entry,
lib, lib,
fn_name, None,
hashes.script(), hashes.script(),
None, None,
false, false,
None,
) )
.cloned() .cloned()
{ {
@ -718,8 +685,9 @@ impl Engine {
// Native function call // Native function call
let hash = hashes.native(); let hash = hashes.native();
self.exec_native_fn_call( self.exec_native_fn_call(
global, caches, lib, fn_name, hash, args, is_ref_mut, false, pos, level, global, caches, lib, fn_name, op_token, hash, args, is_ref_mut, pos, level,
) )
} }
@ -810,6 +778,7 @@ impl Engine {
caches, caches,
lib, lib,
fn_name, fn_name,
None,
new_hash, new_hash,
&mut args, &mut args,
false, false,
@ -865,6 +834,7 @@ impl Engine {
caches, caches,
lib, lib,
&fn_name, &fn_name,
None,
new_hash, new_hash,
&mut args, &mut args,
is_ref_mut, is_ref_mut,
@ -965,6 +935,7 @@ impl Engine {
caches, caches,
lib, lib,
fn_name, fn_name,
None,
hash, hash,
&mut args, &mut args,
is_ref_mut, is_ref_mut,
@ -992,10 +963,10 @@ impl Engine {
lib: &[&Module], lib: &[&Module],
this_ptr: &mut Option<&mut Dynamic>, this_ptr: &mut Option<&mut Dynamic>,
fn_name: &str, fn_name: &str,
op_token: Option<&Token>,
first_arg: Option<&Expr>, first_arg: Option<&Expr>,
args_expr: &[Expr], args_expr: &[Expr],
hashes: FnCallHashes, hashes: FnCallHashes,
is_operator_call: bool,
capture_scope: bool, capture_scope: bool,
pos: Position, pos: Position,
level: usize, level: usize,
@ -1009,7 +980,7 @@ impl Engine {
let redirected; // Handle call() - Redirect function call let redirected; // Handle call() - Redirect function call
match name { match name {
_ if is_operator_call => (), _ if op_token.is_some() => (),
// Handle call() // Handle call()
KEYWORD_FN_PTR_CALL if total_args >= 1 => { KEYWORD_FN_PTR_CALL if total_args >= 1 => {
@ -1204,8 +1175,8 @@ impl Engine {
return self return self
.exec_fn_call( .exec_fn_call(
scope, global, caches, lib, name, hashes, &mut args, is_ref_mut, false, pos, scope, global, caches, lib, name, op_token, hashes, &mut args, is_ref_mut,
level, false, pos, level,
) )
.map(|(v, ..)| v); .map(|(v, ..)| v);
} }
@ -1267,7 +1238,8 @@ impl Engine {
} }
self.exec_fn_call( self.exec_fn_call(
None, global, caches, lib, name, hashes, &mut args, is_ref_mut, false, pos, level, None, global, caches, lib, name, op_token, hashes, &mut args, is_ref_mut, false, pos,
level,
) )
.map(|(v, ..)| v) .map(|(v, ..)| v)
} }
@ -1441,11 +1413,19 @@ impl Engine {
Some(f) => unreachable!("unknown function type: {:?}", f), Some(f) => unreachable!("unknown function type: {:?}", f),
None => Err(ERR::ErrorFunctionNotFound( None => {
gen_qualified_fn_call_signature(self, namespace, fn_name, &args), let sig = if namespace.is_empty() {
pos, self.gen_fn_call_signature(fn_name, &args)
} else {
format!(
"{namespace}{}{}",
crate::tokenizer::Token::DoubleColon.literal_syntax(),
self.gen_fn_call_signature(fn_name, &args)
) )
.into()), };
Err(ERR::ErrorFunctionNotFound(sig, pos).into())
}
} }
} }
@ -1507,13 +1487,18 @@ impl Engine {
level: usize, level: usize,
) -> RhaiResult { ) -> RhaiResult {
let FnCallExpr { let FnCallExpr {
#[cfg(not(feature = "no_module"))]
namespace,
name, name,
hashes, hashes,
args, args,
op_token, op_token,
capture_parent_scope: capture,
.. ..
} = expr; } = expr;
let op_token = op_token.as_ref();
// Short-circuit native binary operator call if under Fast Operators mode // Short-circuit native binary operator call if under Fast Operators mode
if op_token.is_some() && self.fast_operators() && args.len() == 2 { if op_token.is_some() && self.fast_operators() && args.len() == 2 {
let mut lhs = self let mut lhs = self
@ -1538,16 +1523,16 @@ impl Engine {
return self return self
.exec_fn_call( .exec_fn_call(
None, global, caches, lib, name, *hashes, operands, false, false, pos, level, None, global, caches, lib, name, op_token, *hashes, operands, false, false,
pos, level,
) )
.map(|(v, ..)| v); .map(|(v, ..)| v);
} }
#[cfg(not(feature = "no_module"))] #[cfg(not(feature = "no_module"))]
if !expr.namespace.is_empty() { if !namespace.is_empty() {
// Qualified function call // Qualified function call
let hash = hashes.native(); let hash = hashes.native();
let namespace = &expr.namespace;
return self.make_qualified_function_call( return self.make_qualified_function_call(
scope, global, caches, lib, this_ptr, namespace, name, args, hash, pos, level, scope, global, caches, lib, this_ptr, namespace, name, args, hash, pos, level,
@ -1561,19 +1546,8 @@ impl Engine {
); );
self.make_function_call( self.make_function_call(
scope, scope, global, caches, lib, this_ptr, name, op_token, first_arg, args, *hashes,
global, *capture, pos, level,
caches,
lib,
this_ptr,
name,
first_arg,
args,
*hashes,
expr.op_token.is_some(),
expr.capture_parent_scope,
pos,
level,
) )
} }
} }

5
src/func/native.rs
View File

@ -377,6 +377,8 @@ impl<'a> NativeCallContext<'a> {
let caches = &mut Caches::new(); let caches = &mut Caches::new();
let fn_name = fn_name.as_ref(); let fn_name = fn_name.as_ref();
let op_token = Token::lookup_symbol_from_syntax(fn_name);
let op_token = op_token.as_ref();
let args_len = args.len(); let args_len = args.len();
if native_only { if native_only {
@ -387,10 +389,10 @@ impl<'a> NativeCallContext<'a> {
caches, caches,
self.lib, self.lib,
fn_name, fn_name,
op_token,
calc_fn_hash(None, fn_name, args_len), calc_fn_hash(None, fn_name, args_len),
args, args,
is_ref_mut, is_ref_mut,
false,
Position::NONE, Position::NONE,
self.level + 1, self.level + 1,
) )
@ -416,6 +418,7 @@ impl<'a> NativeCallContext<'a> {
caches, caches,
self.lib, self.lib,
fn_name, fn_name,
op_token,
hash, hash,
args, args,
is_ref_mut, is_ref_mut,

4
src/optimizer.rs
View File

@ -152,10 +152,10 @@ impl<'a> OptimizerState<'a> {
&mut self.caches, &mut self.caches,
lib, lib,
fn_name, fn_name,
None,
calc_fn_hash(None, fn_name, arg_values.len()), calc_fn_hash(None, fn_name, arg_values.len()),
&mut arg_values.iter_mut().collect::<StaticVec<_>>(), &mut arg_values.iter_mut().collect::<StaticVec<_>>(),
false, false,
false,
Position::NONE, Position::NONE,
0, 0,
) )
@ -438,7 +438,7 @@ fn optimize_stmt(stmt: &mut Stmt, state: &mut OptimizerState, preserve_result: b
if !x.0.is_op_assignment() if !x.0.is_op_assignment()
&& x.1.lhs.is_variable_access(true) && x.1.lhs.is_variable_access(true)
&& matches!(&x.1.rhs, Expr::FnCall(x2, ..) && matches!(&x.1.rhs, Expr::FnCall(x2, ..)
if Token::lookup_from_syntax(&x2.name).map_or(false, |t| t.has_op_assignment()) if Token::lookup_symbol_from_syntax(&x2.name).map_or(false, |t| t.has_op_assignment())
&& x2.args.len() == 2 && x2.args.len() == 2
&& x2.args[0].get_variable_name(true) == x.1.lhs.get_variable_name(true) && x2.args[0].get_variable_name(true) == x.1.lhs.get_variable_name(true)
) => ) =>

58
src/tokenizer.rs
View File

@ -799,9 +799,9 @@ impl Token {
}) })
} }
/// Reverse lookup a token from a piece of syntax. /// Reverse lookup a symbol token from a piece of syntax.
#[must_use] #[must_use]
pub fn lookup_from_syntax(syntax: &str) -> Option<Self> { pub fn lookup_symbol_from_syntax(syntax: &str) -> Option<Self> {
use Token::*; use Token::*;
Some(match syntax { Some(match syntax {
@ -879,19 +879,11 @@ impl Token {
"**" => PowerOf, "**" => PowerOf,
"**=" => PowerOfAssign, "**=" => PowerOfAssign,
#[cfg(feature = "no_object")]
"?." => Reserved(Box::new(syntax.into())),
#[cfg(feature = "no_index")]
"?[" => Reserved(Box::new(syntax.into())),
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
"fn" => Fn, "fn" => Fn,
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
"private" => Private, "private" => Private,
#[cfg(feature = "no_function")]
"fn" | "private" => Reserved(Box::new(syntax.into())),
#[cfg(not(feature = "no_module"))] #[cfg(not(feature = "no_module"))]
"import" => Import, "import" => Import,
#[cfg(not(feature = "no_module"))] #[cfg(not(feature = "no_module"))]
@ -899,31 +891,43 @@ impl Token {
#[cfg(not(feature = "no_module"))] #[cfg(not(feature = "no_module"))]
"as" => As, "as" => As,
_ => return None,
})
}
/// Is a piece of syntax a reserved keyword?
#[must_use]
pub fn is_reserved_keyword(syntax: &str) -> bool {
match syntax {
#[cfg(feature = "no_object")]
"?." => true,
#[cfg(feature = "no_index")]
"?[" => true,
#[cfg(feature = "no_function")]
"fn" | "private" => true,
#[cfg(feature = "no_module")] #[cfg(feature = "no_module")]
"import" | "export" | "as" => Reserved(Box::new(syntax.into())), "import" | "export" | "as" => true,
// List of reserved operators // List of reserved operators
"===" | "!==" | "->" | "<-" | "?" | ":=" | ":;" | "~" | "!." | "::<" | "(*" | "*)" "===" | "!==" | "->" | "<-" | "?" | ":=" | ":;" | "~" | "!." | "::<" | "(*" | "*)"
| "#" | "#!" | "@" | "$" | "++" | "--" | "..." | "<|" | "|>" => { | "#" | "#!" | "@" | "$" | "++" | "--" | "..." | "<|" | "|>" => true,
Reserved(Box::new(syntax.into()))
}
// List of reserved keywords // List of reserved keywords
"public" | "protected" | "super" | "new" | "use" | "module" | "package" | "var" "public" | "protected" | "super" | "new" | "use" | "module" | "package" | "var"
| "static" | "shared" | "with" | "is" | "goto" | "exit" | "match" | "case" | "static" | "shared" | "with" | "is" | "goto" | "exit" | "match" | "case"
| "default" | "void" | "null" | "nil" | "spawn" | "thread" | "go" | "sync" | "default" | "void" | "null" | "nil" | "spawn" | "thread" | "go" | "sync"
| "async" | "await" | "yield" => Reserved(Box::new(syntax.into())), | "async" | "await" | "yield" => true,
KEYWORD_PRINT | KEYWORD_DEBUG | KEYWORD_TYPE_OF | KEYWORD_EVAL | KEYWORD_FN_PTR KEYWORD_PRINT | KEYWORD_DEBUG | KEYWORD_TYPE_OF | KEYWORD_EVAL | KEYWORD_FN_PTR
| KEYWORD_FN_PTR_CALL | KEYWORD_FN_PTR_CURRY | KEYWORD_THIS | KEYWORD_IS_DEF_VAR => { | KEYWORD_FN_PTR_CALL | KEYWORD_FN_PTR_CURRY | KEYWORD_THIS | KEYWORD_IS_DEF_VAR => {
Reserved(Box::new(syntax.into())) true
} }
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
crate::engine::KEYWORD_IS_DEF_FN => Reserved(Box::new(syntax.into())), crate::engine::KEYWORD_IS_DEF_FN => true,
_ => return None, _ => false,
}) }
} }
/// Is this token [`EOF`][Token::EOF]? /// Is this token [`EOF`][Token::EOF]?
@ -1708,11 +1712,11 @@ fn get_next_token_inner(
// letter or underscore ... // letter or underscore ...
#[cfg(not(feature = "unicode-xid-ident"))] #[cfg(not(feature = "unicode-xid-ident"))]
('a'..='z' | '_' | 'A'..='Z', ..) => { ('a'..='z' | '_' | 'A'..='Z', ..) => {
return Some(get_identifier(stream, pos, start_pos, c)); return Some(get_token_as_identifier(stream, pos, start_pos, c));
} }
#[cfg(feature = "unicode-xid-ident")] #[cfg(feature = "unicode-xid-ident")]
(ch, ..) if unicode_xid::UnicodeXID::is_xid_start(ch) || ch == '_' => { (ch, ..) if unicode_xid::UnicodeXID::is_xid_start(ch) || ch == '_' => {
return Some(get_identifier(stream, pos, start_pos, c)); return Some(get_token_as_identifier(stream, pos, start_pos, c));
} }
// " - string literal // " - string literal
@ -2179,8 +2183,8 @@ fn get_next_token_inner(
Some((Token::EOF, *pos)) Some((Token::EOF, *pos))
} }
/// Get the next identifier. /// Get the next token, parsing it as an identifier.
fn get_identifier( fn get_token_as_identifier(
stream: &mut impl InputStream, stream: &mut impl InputStream,
pos: &mut Position, pos: &mut Position,
start_pos: Position, start_pos: Position,
@ -2199,13 +2203,13 @@ fn get_identifier(
} }
} }
let is_valid_identifier = is_valid_identifier(&identifier); if let Some(token) = Token::lookup_symbol_from_syntax(&identifier) {
if let Some(token) = Token::lookup_from_syntax(&identifier) {
return (token, start_pos); return (token, start_pos);
} else if Token::is_reserved_keyword(&identifier) {
return (Token::Reserved(Box::new(identifier)), start_pos);
} }
if !is_valid_identifier { if !is_valid_identifier(&identifier) {
return ( return (
Token::LexError(LERR::MalformedIdentifier(identifier.to_string()).into()), Token::LexError(LERR::MalformedIdentifier(identifier.to_string()).into()),
start_pos, start_pos,