kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Use SmartString for buffers and literal_syntax can panic.

Browse Source
This commit is contained in:
Stephen Chung 2022-11-22 23:30:43 +08:00
parent 05c7d00a8e
commit d911327242
16 changed files with 225 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/api/mod.rs
View File

@ -163,25 +163,18 @@ impl Engine {
// Active standard keywords cannot be made custom
// Disabled keywords are OK
Some(token) if token.is_standard_keyword() => {
if self.disabled_symbols.is_empty()
|| !self.disabled_symbols.contains(&*token.syntax())
{
if !self.disabled_symbols.contains(token.literal_syntax()) {
return Err(format!("'{keyword}' is a reserved keyword"));
}
}
// Active standard symbols cannot be made custom
Some(token) if token.is_standard_symbol() => {
if self.disabled_symbols.is_empty()
|| !self.disabled_symbols.contains(&*token.syntax())
{
if !self.disabled_symbols.contains(token.literal_syntax()) {
return Err(format!("'{keyword}' is a reserved operator"));
}
}
// Active standard symbols cannot be made custom
Some(token)
if self.disabled_symbols.is_empty()
|| !self.disabled_symbols.contains(&*token.syntax()) =>
{
Some(token) if !self.disabled_symbols.contains(token.literal_syntax()) => {
return Err(format!("'{keyword}' is a reserved symbol"))
}
// Disabled symbols are OK

3
src/api/register.rs
View File

@ -688,8 +688,7 @@ impl Engine {
name: &str,
module: SharedModule,
) {
let separator = crate::tokenizer::Token::DoubleColon.syntax();
let separator = separator.as_ref();
let separator = crate::tokenizer::Token::DoubleColon.literal_syntax();
if name.contains(separator) {
let mut iter = name.splitn(2, separator);

14
src/ast/expr.rs
View File

@ -361,7 +361,7 @@ impl fmt::Debug for Expr {
#[cold]
#[inline(never)]
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let mut display_pos = format!(" @ {:?}", self.start_position());
let mut display_pos = self.start_position();
match self {
Self::DynamicConstant(value, ..) => write!(f, "{value:?}"),
@ -395,7 +395,7 @@ impl fmt::Debug for Expr {
write!(f, "{}{}", x.1, Token::DoubleColon.literal_syntax())?;
let pos = x.1.position();
if !pos.is_none() {
display_pos = format!(" @ {pos:?}");
display_pos = pos;
}
}
f.write_str(&x.3)?;
@ -413,7 +413,7 @@ impl fmt::Debug for Expr {
Self::Stmt(x) => {
let pos = x.span();
if !pos.is_none() {
display_pos = format!(" @ {pos:?}");
display_pos = pos.start();
}
f.write_str("ExprStmtBlock")?;
f.debug_list().entries(x.iter()).finish()
@ -421,7 +421,7 @@ impl fmt::Debug for Expr {
Self::FnCall(x, ..) => fmt::Debug::fmt(x, f),
Self::Index(x, options, pos) => {
if !pos.is_none() {
display_pos = format!(" @ {pos:?}");
display_pos = *pos;
}
let mut f = f.debug_struct("Index");
@ -434,7 +434,7 @@ impl fmt::Debug for Expr {
}
Self::Dot(x, options, pos) => {
if !pos.is_none() {
display_pos = format!(" @ {pos:?}");
display_pos = *pos;
}
let mut f = f.debug_struct("Dot");
@ -454,7 +454,7 @@ impl fmt::Debug for Expr {
};
if !pos.is_none() {
display_pos = format!(" @ {pos:?}");
display_pos = *pos;
}
f.debug_struct(op_name)
@ -466,7 +466,7 @@ impl fmt::Debug for Expr {
Self::Custom(x, ..) => f.debug_tuple("Custom").field(x).finish(),
}?;
f.write_str(&display_pos)
write!(f, " @ {display_pos:?}")
}
}

12
src/eval/chaining.rs
View File

@ -317,7 +317,7 @@ impl Engine {
scope: &mut Scope,
this_ptr: &mut Dynamic,
expr: &Expr,
new_val: &mut Option<(Dynamic, &OpAssignment)>,
new_val: Option<(Dynamic, &OpAssignment)>,
) -> RhaiResult {
let chain_type = ChainType::from(expr);
@ -507,7 +507,7 @@ impl Engine {
target: &mut Target,
rhs: &Expr,
idx_values: &mut FnArgsVec<Dynamic>,
new_val: &mut Option<(Dynamic, &OpAssignment)>,
new_val: Option<(Dynamic, &OpAssignment)>,
) -> RhaiResultOf<(Dynamic, bool)> {
let is_ref_mut = target.is_ref();
let op_pos = parent.position();
@ -576,7 +576,7 @@ impl Engine {
#[cfg(feature = "debugging")]
self.run_debugger(global, caches, scope, this_ptr, parent)?;
let (new_val, op_info) = new_val.take().expect("`Some`");
let (new_val, op_info) = new_val.expect("`Some`");
let idx_val = &mut idx_values.pop().unwrap();
let idx = &mut idx_val.clone();
@ -686,12 +686,12 @@ impl Engine {
unreachable!("function call in dot chain should not be namespace-qualified")
}
// {xxx:map}.id op= ???
Expr::Property(x, pos) if target.is_map() && new_val.is_some() => {
Expr::Property(x, pos) if new_val.is_some() && target.is_map() => {
#[cfg(feature = "debugging")]
self.run_debugger(global, caches, scope, this_ptr, rhs)?;
let index = &mut x.2.clone().into();
let (new_val, op_info) = new_val.take().expect("`Some`");
let (new_val, op_info) = new_val.expect("`Some`");
{
let val_target = &mut self.get_indexed_mut(
global, caches, target, index, *pos, op_pos, true, false,
@ -720,7 +720,7 @@ impl Engine {
self.run_debugger(global, caches, scope, this_ptr, rhs)?;
let ((getter, hash_get), (setter, hash_set), name) = &**x;
let (mut new_val, op_info) = new_val.take().expect("`Some`");
let (mut new_val, op_info) = new_val.expect("`Some`");
if op_info.is_op_assignment() {
let args = &mut [target.as_mut()];

24
src/eval/data_check.rs
View File

@ -8,7 +8,7 @@ use std::borrow::Borrow;
#[cfg(feature = "no_std")]
use std::prelude::v1::*;
impl Engine {
impl Dynamic {
/// Recursively calculate the sizes of a value.
///
/// Sizes returned are `(` [`Array`][crate::Array], [`Map`][crate::Map] and [`String`] `)`.
@ -16,20 +16,20 @@ impl Engine {
/// # Panics
///
/// Panics if any interior data is shared (should never happen).
pub(crate) fn calc_data_sizes(value: &Dynamic, _top: bool) -> (usize, usize, usize) {
match value.0 {
pub(crate) fn calc_data_sizes(&self, _top: bool) -> (usize, usize, usize) {
match self.0 {
#[cfg(not(feature = "no_index"))]
Union::Array(ref arr, ..) => {
arr.iter()
.fold((0, 0, 0), |(ax, mx, sx), value| match value.0 {
Union::Array(..) => {
let (a, m, s) = Self::calc_data_sizes(value, false);
let (a, m, s) = value.calc_data_sizes(false);
(ax + a + 1, mx + m, sx + s)
}
Union::Blob(ref a, ..) => (ax + 1 + a.len(), mx, sx),
#[cfg(not(feature = "no_object"))]
Union::Map(..) => {
let (a, m, s) = Self::calc_data_sizes(value, false);
let (a, m, s) = value.calc_data_sizes(false);
(ax + a + 1, mx + m, sx + s)
}
Union::Str(ref s, ..) => (ax + 1, mx, sx + s.len()),
@ -44,13 +44,13 @@ impl Engine {
.fold((0, 0, 0), |(ax, mx, sx), value| match value.0 {
#[cfg(not(feature = "no_index"))]
Union::Array(..) => {
let (a, m, s) = Self::calc_data_sizes(value, false);
let (a, m, s) = value.calc_data_sizes(false);
(ax + a, mx + m + 1, sx + s)
}
#[cfg(not(feature = "no_index"))]
Union::Blob(ref a, ..) => (ax + a.len(), mx, sx),
Union::Map(..) => {
let (a, m, s) = Self::calc_data_sizes(value, false);
let (a, m, s) = value.calc_data_sizes(false);
(ax + a, mx + m + 1, sx + s)
}
Union::Str(ref s, ..) => (ax, mx + 1, sx + s.len()),
@ -59,17 +59,17 @@ impl Engine {
}
Union::Str(ref s, ..) => (0, 0, s.len()),
#[cfg(not(feature = "no_closure"))]
Union::Shared(..) if _top => {
Self::calc_data_sizes(&*value.read_lock::<Dynamic>().unwrap(), true)
}
Union::Shared(..) if _top => self.read_lock::<Dynamic>().unwrap().calc_data_sizes(true),
#[cfg(not(feature = "no_closure"))]
Union::Shared(..) => {
unreachable!("shared values discovered within data: {}", value)
unreachable!("shared values discovered within data: {}", self)
}
_ => (0, 0, 0),
}
}
}
impl Engine {
/// Raise an error if any data size exceeds limit.
///
/// [`Position`] in [`EvalAltResult`][crate::EvalAltResult] is always [`NONE`][Position::NONE]
@ -125,7 +125,7 @@ impl Engine {
return Ok(value);
}
let sizes = Self::calc_data_sizes(value.borrow(), true);
let sizes = value.borrow().calc_data_sizes(true);
self.raise_err_if_over_data_size_limit(sizes)
.map(|_| value)

10
src/eval/expr.rs
View File

@ -307,7 +307,7 @@ impl Engine {
#[cfg(not(feature = "unchecked"))]
if self.has_data_size_limit() {
let val_sizes = Self::calc_data_sizes(&value, true);
let val_sizes = value.calc_data_sizes(true);
total_data_sizes = (
total_data_sizes.0 + val_sizes.0,
@ -339,7 +339,7 @@ impl Engine {
#[cfg(not(feature = "unchecked"))]
if self.has_data_size_limit() {
let delta = Self::calc_data_sizes(&value, true);
let delta = value.calc_data_sizes(true);
total_data_sizes = (
total_data_sizes.0 + delta.0,
total_data_sizes.1 + delta.1,
@ -411,13 +411,11 @@ impl Engine {
#[cfg(not(feature = "no_index"))]
Expr::Index(..) => {
self.eval_dot_index_chain(global, caches, scope, this_ptr, expr, &mut None)
self.eval_dot_index_chain(global, caches, scope, this_ptr, expr, None)
}
#[cfg(not(feature = "no_object"))]
Expr::Dot(..) => {
self.eval_dot_index_chain(global, caches, scope, this_ptr, expr, &mut None)
}
Expr::Dot(..) => self.eval_dot_index_chain(global, caches, scope, this_ptr, expr, None),
_ => unreachable!("expression cannot be evaluated: {:?}", expr),
}

4
src/eval/mod.rs
View File

@ -21,7 +21,9 @@ pub use eval_context::EvalContext;
#[cfg(not(feature = "no_function"))]
pub use global_state::GlobalConstants;
pub use global_state::GlobalRuntimeState;
pub use target::{calc_index, calc_offset_len, Target};
#[cfg(not(feature = "no_index"))]
pub use target::calc_offset_len;
pub use target::{calc_index, Target};
#[cfg(feature = "unchecked")]
mod unchecked {

2
src/eval/stmt.rs
View File

@ -258,7 +258,7 @@ impl Engine {
rhs_val = self.get_interned_string(value).into();
}
let _new_val = &mut Some((rhs_val, op_info));
let _new_val = Some((rhs_val, op_info));
// Must be either `var[index] op= val` or `var.prop op= val`
match lhs {

2
src/eval/target.rs
View File

@ -13,8 +13,8 @@ use std::{
/// Negative starting positions count from the end.
///
/// Values going over bounds are limited to the actual length.
#[cfg(not(feature = "no_index"))]
#[inline]
#[allow(dead_code)]
pub fn calc_offset_len(length: usize, start: crate::INT, len: crate::INT) -> (usize, usize) {
let start = if start < 0 {
let abs_start = start.unsigned_abs();

21
src/func/builtin.rs
View File

@ -4,11 +4,12 @@ use super::call::FnCallArgs;
use super::native::FnBuiltin;
use crate::tokenizer::{Token, Token::*};
use crate::{
Dynamic, ExclusiveRange, ImmutableString, InclusiveRange, NativeCallContext, RhaiResult, INT,
Dynamic, ExclusiveRange, ImmutableString, InclusiveRange, NativeCallContext, RhaiResult,
SmartString, INT,
};
use std::any::TypeId;
#[cfg(feature = "no_std")]
use std::prelude::v1::*;
use std::{any::TypeId, fmt::Write};
#[cfg(not(feature = "no_float"))]
use crate::FLOAT;
@ -241,7 +242,9 @@ pub fn get_builtin_binary_op_fn(op: &Token, x: &Dynamic, y: &Dynamic) -> Option<
let x = args[0].as_char().expect(BUILTIN);
let y = args[1].as_char().expect(BUILTIN);
let result = format!("{x}{y}");
let mut result = SmartString::new_const();
result.push(x);
result.push(y);
#[cfg(not(feature = "unchecked"))]
_ctx.engine()
@ -399,7 +402,10 @@ pub fn get_builtin_binary_op_fn(op: &Token, x: &Dynamic, y: &Dynamic) -> Option<
Plus => Some(|_ctx, args| {
let x = args[0].as_char().expect(BUILTIN);
let y = &*args[1].read_lock::<ImmutableString>().expect(BUILTIN);
let result = format!("{x}{y}");
let mut result = SmartString::new_const();
result.push(x);
result.push_str(y);
#[cfg(not(feature = "unchecked"))]
_ctx.engine()
@ -679,7 +685,12 @@ pub fn get_builtin_op_assignment_fn(op: &Token, x: &Dynamic, y: &Dynamic) -> Opt
PlusAssign => Some(|_, args| {
let y = args[1].as_char().expect(BUILTIN);
let x = &mut *args[0].write_lock::<Dynamic>().expect(BUILTIN);
Ok((*x = format!("{x}{y}").into()).into())
let mut buf = SmartString::new_const();
write!(&mut buf, "{y}").unwrap();
buf.push(y);
Ok((*x = buf.into()).into())
}),
_ => None,
};

4
src/packages/array_basic.rs
View File

@ -247,8 +247,8 @@ pub mod array_functions {
let mut arr_len = array.len();
let mut arr = Dynamic::from_array(mem::take(array));
let (mut a1, mut m1, mut s1) = crate::Engine::calc_data_sizes(&arr, true);
let (a2, m2, s2) = crate::Engine::calc_data_sizes(&item, true);
let (mut a1, mut m1, mut s1) = arr.calc_data_sizes(true);
let (a2, m2, s2) = item.calc_data_sizes(true);
{
let mut guard = arr.write_lock::<Array>().unwrap();

10
src/packages/lang_core.rs
View File

@ -269,9 +269,13 @@ fn collect_fn_metadata(
.iter_script_fn()
.filter(|(s, a, n, p, f)| filter(*s, *a, n, *p, f))
.for_each(|(.., f)| list.push(make_metadata(dict, namespace.into(), f).into()));
for (ns, m) in module.iter_sub_modules() {
let ns = format!(
"{namespace}{}{ns}",
for (name, m) in module.iter_sub_modules() {
use std::fmt::Write;
let mut ns = crate::SmartString::new_const();
write!(
&mut ns,
"{namespace}{}{name}",
crate::tokenizer::Token::DoubleColon.literal_syntax()
);
scan_module(dict, list, &ns, &**m, filter);

40
src/packages/string_basic.rs
View File

@ -1,7 +1,7 @@
use crate::plugin::*;
use crate::{def_package, FnPtr, INT};
use crate::{def_package, FnPtr, SmartString, INT};
use std::any::TypeId;
use std::fmt::{Binary, LowerHex, Octal};
use std::fmt::{Binary, LowerHex, Octal, Write};
#[cfg(feature = "no_std")]
use std::prelude::v1::*;
@ -69,7 +69,9 @@ mod print_debug_functions {
/// Convert the value of the `item` into a string in debug format.
#[rhai_fn(name = "to_debug", pure)]
pub fn to_debug_generic(ctx: NativeCallContext, item: &mut Dynamic) -> ImmutableString {
ctx.engine().map_type_name(&format!("{item:?}")).into()
let mut buf = SmartString::new_const();
write!(&mut buf, "{item:?}").unwrap();
ctx.engine().map_type_name(&buf).into()
}
/// Return the empty string.
@ -86,7 +88,9 @@ mod print_debug_functions {
/// Convert the string into debug format.
#[rhai_fn(name = "debug", name = "to_debug", pure)]
pub fn debug_string(string: &mut ImmutableString) -> ImmutableString {
format!("{string:?}").into()
let mut buf = SmartString::new_const();
write!(&mut buf, "{string:?}").unwrap();
buf.into()
}
/// Return the character into a string.
@ -97,7 +101,9 @@ mod print_debug_functions {
/// Convert the string into debug format.
#[rhai_fn(name = "debug", name = "to_debug")]
pub fn debug_char(character: char) -> ImmutableString {
format!("{character:?}").into()
let mut buf = SmartString::new_const();
buf.push(character);
buf.into()
}
/// Convert the function pointer into a string in debug format.
@ -114,7 +120,9 @@ mod print_debug_functions {
/// Convert the boolean value into a string in debug format.
#[rhai_fn(name = "debug", name = "to_debug")]
pub fn debug_bool(value: bool) -> ImmutableString {
format!("{value:?}").into()
let mut buf = SmartString::new_const();
write!(&mut buf, "{value:?}").unwrap();
buf.into()
}
/// Return the empty string.
@ -147,14 +155,18 @@ mod print_debug_functions {
#[rhai_fn(name = "debug", name = "to_debug")]
pub fn debug_f64(number: f64) -> ImmutableString {
let number = crate::types::FloatWrapper::new(number);
format!("{number:?}").into()
let mut buf = SmartString::new_const();
write!(&mut buf, "{number:?}").unwrap();
buf.into()
}
/// Convert the value of `number` into a string.
#[cfg(not(feature = "no_float"))]
#[rhai_fn(name = "debug", name = "to_debug")]
pub fn debug_f32(number: f32) -> ImmutableString {
let number = crate::types::FloatWrapper::new(number);
format!("{number:?}").into()
let mut buf = SmartString::new_const();
write!(&mut buf, "{number:?}").unwrap();
buf.into()
}
/// Convert the array into a string.
@ -217,13 +229,19 @@ mod print_debug_functions {
#[export_module]
mod number_formatting {
fn to_hex<T: LowerHex>(value: T) -> ImmutableString {
format!("{value:x}").into()
let mut buf = SmartString::new_const();
write!(&mut buf, "{value:x}").unwrap();
buf.into()
}
fn to_octal<T: Octal>(value: T) -> ImmutableString {
format!("{value:o}").into()
let mut buf = SmartString::new_const();
write!(&mut buf, "{value:o}").unwrap();
buf.into()
}
fn to_binary<T: Binary>(value: T) -> ImmutableString {
format!("{value:b}").into()
let mut buf = SmartString::new_const();
write!(&mut buf, "{value:b}").unwrap();
buf.into()
}
/// Convert the `value` into a string in hex format.

13
src/packages/string_more.rs
View File

@ -33,7 +33,9 @@ mod string_functions {
if s.is_empty() {
string.clone()
} else {
format!("{string}{s}").into()
let mut buf = SmartString::from(string.as_str());
buf.push_str(&s);
buf.into()
}
}
#[rhai_fn(name = "+=", name = "append")]
@ -41,7 +43,9 @@ mod string_functions {
let s = print_with_func(FUNC_TO_STRING, &ctx, &mut item);
if !s.is_empty() {
*string = format!("{string}{s}").into();
let mut buf = SmartString::from(string.as_str());
buf.push_str(&s);
*string = buf.into();
}
}
#[rhai_fn(name = "+", pure)]
@ -74,7 +78,10 @@ mod string_functions {
}
#[rhai_fn(name = "+")]
pub fn add_prepend_char(character: char, string: &str) -> ImmutableString {
format!("{character}{string}").into()
let mut buf = SmartString::new_const();
buf.push(character);
buf.push_str(string);
buf.into()
}
#[rhai_fn(name = "+")]

131
src/parser.rs
View File

@ -333,8 +333,12 @@ impl ParseSettings {
#[cfg(not(feature = "no_function"))]
#[inline]
#[must_use]
pub fn make_anonymous_fn(hash: u64) -> String {
format!("{}{:016x}", crate::engine::FN_ANONYMOUS, hash)
pub fn make_anonymous_fn(hash: u64) -> Identifier {
use std::fmt::Write;
let mut buf = Identifier::new_const();
write!(&mut buf, "{}{:016x}", crate::engine::FN_ANONYMOUS, hash).unwrap();
buf
}
/// Is this function an anonymous function?
@ -387,7 +391,7 @@ impl Expr {
};
Err(
PERR::MismatchedType("a boolean expression".to_string(), type_name.to_string())
PERR::MismatchedType("a boolean expression".into(), type_name.into())
.into_err(self.start_position()),
)
}
@ -405,7 +409,7 @@ impl Expr {
};
Err(
PERR::MismatchedType("an iterable value".to_string(), type_name.to_string())
PERR::MismatchedType("an iterable value".into(), type_name.into())
.into_err(self.start_position()),
)
}
@ -426,8 +430,8 @@ fn ensure_not_statement_expr(
fn ensure_not_assignment(input: &mut TokenStream) -> ParseResult<()> {
match input.peek().expect(NEVER_ENDS) {
(Token::Equals, pos) => Err(LexError::ImproperSymbol(
"=".to_string(),
"Possibly a typo of '=='?".to_string(),
"=".into(),
"Possibly a typo of '=='?".into(),
)
.into_err(*pos)),
_ => Ok(()),
@ -438,15 +442,15 @@ fn ensure_not_assignment(input: &mut TokenStream) -> ParseResult<()> {
///
/// # Panics
///
/// Panics if the next token is not the expected one.
/// Panics if the next token is not the expected one, or either tokens is not a literal symbol.
fn eat_token(input: &mut TokenStream, expected_token: Token) -> Position {
let (t, pos) = input.next().expect(NEVER_ENDS);
if t != expected_token {
unreachable!(
"{} expected but gets {} at {}",
expected_token.syntax(),
t.syntax(),
expected_token.literal_syntax(),
t.literal_syntax(),
pos
);
}
@ -588,7 +592,7 @@ impl Engine {
&& !self.global_sub_modules.contains_key(root)
{
return Err(
PERR::ModuleUndefined(root.to_string()).into_err(namespace.position())
PERR::ModuleUndefined(root.into()).into_err(namespace.position())
);
}
@ -655,8 +659,9 @@ impl Engine {
&& !state.global_imports.iter().any(|m| m.as_str() == root)
&& !self.global_sub_modules.contains_key(root)
{
return Err(PERR::ModuleUndefined(root.to_string())
.into_err(namespace.position()));
return Err(
PERR::ModuleUndefined(root.into()).into_err(namespace.position())
);
}
namespace.set_index(index);
@ -901,7 +906,7 @@ impl Engine {
if self.max_array_size() > 0 && array.len() >= self.max_array_size() {
return Err(PERR::LiteralTooLarge(
"Size of array literal".to_string(),
"Size of array literal".into(),
self.max_array_size(),
)
.into_err(input.peek().expect(NEVER_ENDS).1));
@ -1037,7 +1042,7 @@ impl Engine {
if self.max_map_size() > 0 && map.len() >= self.max_map_size() {
return Err(PERR::LiteralTooLarge(
"Number of properties in object map literal".to_string(),
"Number of properties in object map literal".into(),
self.max_map_size(),
)
.into_err(input.peek().expect(NEVER_ENDS).1));
@ -1185,7 +1190,7 @@ impl Engine {
(.., pos) => {
return Err(PERR::MissingToken(
Token::DoubleArrow.into(),
"in this switch case".to_string(),
"in this switch case".into(),
)
.into_err(pos))
}
@ -1213,7 +1218,7 @@ impl Engine {
} else {
for expr in case_expr_list {
let value = expr.get_literal_value().ok_or_else(|| {
PERR::ExprExpected("a literal".to_string()).into_err(expr.start_position())
PERR::ExprExpected("a literal".into()).into_err(expr.start_position())
})?;
let mut range_value: Option<RangeCase> = None;
@ -1315,9 +1320,7 @@ impl Engine {
let root_expr = match token {
_ if !(state.expr_filter)(token) => {
return Err(
LexError::UnexpectedInput(token.syntax().to_string()).into_err(settings.pos)
)
return Err(LexError::UnexpectedInput(token.to_string()).into_err(settings.pos))
}
Token::EOF => return Err(PERR::UnexpectedEOF.into_err(settings.pos)),
@ -1674,11 +1677,7 @@ impl Engine {
token => unreachable!("Token::LexError expected but gets {:?}", token),
},
_ => {
return Err(
LexError::UnexpectedInput(token.syntax().to_string()).into_err(settings.pos)
)
}
_ => return Err(LexError::UnexpectedInput(token.to_string()).into_err(settings.pos)),
};
if !(state.expr_filter)(&input.peek().expect(NEVER_ENDS).0) {
@ -1716,12 +1715,11 @@ impl Engine {
(Expr::Variable(x, ..), Token::Bang) if !x.1.is_empty() => {
return match input.peek().expect(NEVER_ENDS) {
(Token::LeftParen | Token::Unit, ..) => {
Err(LexError::UnexpectedInput(Token::Bang.syntax().to_string())
.into_err(tail_pos))
Err(LexError::UnexpectedInput(Token::Bang.into()).into_err(tail_pos))
}
_ => Err(LexError::ImproperSymbol(
"!".to_string(),
"'!' cannot be used to call module functions".to_string(),
"!".into(),
"'!' cannot be used to call module functions".into(),
)
.into_err(tail_pos)),
};
@ -1732,7 +1730,7 @@ impl Engine {
(Token::LeftParen | Token::Unit, ..) => (),
(_, pos) => {
return Err(PERR::MissingToken(
Token::LeftParen.syntax().into(),
Token::LeftParen.into(),
"to start arguments list of function call".into(),
)
.into_err(*pos))
@ -1821,11 +1819,9 @@ impl Engine {
Self::make_dot_expr(state, expr, rhs, ASTFlags::NONE, op_flags, tail_pos)?
}
// Unknown postfix operator
(expr, token) => unreachable!(
"unknown postfix operator '{}' for {:?}",
token.syntax(),
expr
),
(expr, token) => {
unreachable!("unknown postfix operator '{}' for {:?}", token, expr)
}
}
}
@ -1863,7 +1859,7 @@ impl Engine {
&& !self.global_sub_modules.contains_key(root)
{
return Err(
PERR::ModuleUndefined(root.to_string()).into_err(namespace.position())
PERR::ModuleUndefined(root.into()).into_err(namespace.position())
);
}
@ -1890,7 +1886,7 @@ impl Engine {
let (token, token_pos) = input.peek().expect(NEVER_ENDS);
if !(state.expr_filter)(token) {
return Err(LexError::UnexpectedInput(token.syntax().to_string()).into_err(*token_pos));
return Err(LexError::UnexpectedInput(token.to_string()).into_err(*token_pos));
}
let mut settings = settings;
@ -2092,8 +2088,8 @@ impl Engine {
}
// ??? && ??? = rhs, ??? || ??? = rhs, xxx ?? xxx = rhs
Expr::And(..) | Expr::Or(..) | Expr::Coalesce(..) => Err(LexError::ImproperSymbol(
"=".to_string(),
"Possibly a typo of '=='?".to_string(),
"=".into(),
"Possibly a typo of '=='?".into(),
)
.into_err(op_pos)),
// expr = rhs
@ -2362,7 +2358,7 @@ impl Engine {
#[cfg(not(feature = "unchecked"))]
settings.ensure_level_within_max_limit(state.max_expr_depth)?;
let op = op_token.syntax();
let op = op_token.to_string();
let hash = calc_fn_hash(None, &op, 2);
let is_valid_script_function = is_valid_function_name(&op);
let operator_token = if is_valid_script_function {
@ -2379,7 +2375,7 @@ impl Engine {
let mut op_base = FnCallExpr {
#[cfg(not(feature = "no_module"))]
namespace: Default::default(),
name: state.get_interned_string(op.as_ref()),
name: state.get_interned_string(&op),
hashes: FnCallHashes::from_native(hash),
args,
op_token: operator_token,
@ -2492,7 +2488,7 @@ impl Engine {
settings.pos = *fwd_pos;
let settings = settings.level_up();
required_token = match parse_func(&segments, &*fwd_token.syntax(), &mut user_state) {
required_token = match parse_func(&segments, &fwd_token.to_string(), &mut user_state) {
Ok(Some(seg))
if seg.starts_with(CUSTOM_SYNTAX_MARKER_SYNTAX_VARIANT)
&& seg.len() > CUSTOM_SYNTAX_MARKER_SYNTAX_VARIANT.len() =>
@ -2551,8 +2547,7 @@ impl Engine {
}
(.., pos) => {
return Err(
PERR::MissingSymbol("Expecting 'true' or 'false'".to_string())
.into_err(pos),
PERR::MissingSymbol("Expecting 'true' or 'false'".into()).into_err(pos)
)
}
},
@ -2564,8 +2559,7 @@ impl Engine {
}
(.., pos) => {
return Err(
PERR::MissingSymbol("Expecting an integer number".to_string())
.into_err(pos),
PERR::MissingSymbol("Expecting an integer number".into()).into_err(pos)
)
}
},
@ -2577,10 +2571,10 @@ impl Engine {
tokens.push(state.get_interned_string(CUSTOM_SYNTAX_MARKER_FLOAT));
}
(.., pos) => {
return Err(PERR::MissingSymbol(
"Expecting a floating-point number".to_string(),
return Err(
PERR::MissingSymbol("Expecting a floating-point number".into())
.into_err(pos),
)
.into_err(pos))
}
},
CUSTOM_SYNTAX_MARKER_STRING => match input.next().expect(NEVER_ENDS) {
@ -2591,20 +2585,26 @@ impl Engine {
tokens.push(state.get_interned_string(CUSTOM_SYNTAX_MARKER_STRING));
}
(.., pos) => {
return Err(
PERR::MissingSymbol("Expecting a string".to_string()).into_err(pos)
)
return Err(PERR::MissingSymbol("Expecting a string".into()).into_err(pos))
}
},
s => match input.next().expect(NEVER_ENDS) {
(Token::LexError(err), pos) => return Err(err.into_err(pos)),
(t, ..) if &*t.syntax() == s => {
(Token::Identifier(t), ..)
| (Token::Reserved(t), ..)
| (Token::Custom(t), ..)
if *t == s =>
{
segments.push(required_token.clone());
tokens.push(required_token.clone().into());
}
(t, ..) if t.is_literal() && t.literal_syntax() == s => {
segments.push(required_token.clone());
tokens.push(required_token.clone().into());
}
(.., pos) => {
return Err(PERR::MissingToken(
s.to_string(),
s.into(),
format!("for '{}' expression", segments[0]),
)
.into_err(pos))
@ -2805,9 +2805,7 @@ impl Engine {
let (counter_name, counter_pos) = parse_var_name(input)?;
if counter_name == name {
return Err(
PERR::DuplicatedVariable(counter_name.to_string()).into_err(counter_pos)
);
return Err(PERR::DuplicatedVariable(counter_name.into()).into_err(counter_pos));
}
let (has_close_paren, pos) = match_token(input, Token::RightParen);
@ -2893,7 +2891,7 @@ impl Engine {
let (name, pos) = parse_var_name(input)?;
if !self.allow_shadowing() && state.stack.iter().any(|(v, ..)| v == name) {
return Err(PERR::VariableExists(name.to_string()).into_err(pos));
return Err(PERR::VariableExists(name.into()).into_err(pos));
}
if let Some(ref filter) = self.def_var_filter {
@ -2914,10 +2912,10 @@ impl Engine {
match filter(false, info, context) {
Ok(true) => (),
Ok(false) => return Err(PERR::ForbiddenVariable(name.to_string()).into_err(pos)),
Ok(false) => return Err(PERR::ForbiddenVariable(name.into()).into_err(pos)),
Err(err) => match *err {
EvalAltResult::ErrorParsing(e, pos) => return Err(e.into_err(pos)),
_ => return Err(PERR::ForbiddenVariable(name.to_string()).into_err(pos)),
_ => return Err(PERR::ForbiddenVariable(name.into()).into_err(pos)),
},
}
}
@ -3361,7 +3359,7 @@ impl Engine {
(.., pos) => Err(PERR::MissingToken(
Token::Fn.into(),
format!("following '{}'", Token::Private.syntax()),
format!("following '{}'", Token::Private),
)
.into_err(pos)),
}
@ -3564,7 +3562,7 @@ impl Engine {
eat_token(input, Token::Unit);
true
}
(.., pos) => return Err(PERR::FnMissingParams(name.to_string()).into_err(*pos)),
(.., pos) => return Err(PERR::FnMissingParams(name.into()).into_err(*pos)),
};
let mut params = StaticVec::<(ImmutableString, _)>::new_const();
@ -3577,8 +3575,9 @@ impl Engine {
(Token::RightParen, ..) => break,
(Token::Identifier(s), pos) => {
if params.iter().any(|(p, _)| p.as_str() == &*s) {
return Err(PERR::FnDuplicatedParam(name.to_string(), s.to_string())
.into_err(pos));
return Err(
PERR::FnDuplicatedParam(name.into(), s.to_string()).into_err(pos)
);
}
let s = state.get_interned_string(*s);
state.stack.push(s.clone(), ());
@ -3611,7 +3610,7 @@ impl Engine {
settings.is_breakable = false;
self.parse_block(input, state, lib, settings.level_up())?
}
(.., pos) => return Err(PERR::FnMissingBody(name.to_string()).into_err(*pos)),
(.., pos) => return Err(PERR::FnMissingBody(name.into()).into_err(*pos)),
}
.into();
@ -3851,9 +3850,7 @@ impl Engine {
match input.peek().expect(NEVER_ENDS) {
(Token::EOF, ..) => (),
// Return error if the expression doesn't end
(token, pos) => {
return Err(LexError::UnexpectedInput(token.syntax().to_string()).into_err(*pos))
}
(token, pos) => return Err(LexError::UnexpectedInput(token.to_string()).into_err(*pos)),
}
let mut statements = StmtBlockContainer::new_const();

96
src/tokenizer.rs
View File

@ -9,7 +9,6 @@ use crate::{Engine, Identifier, LexError, SmartString, StaticVec, INT, UNSIGNED_
#[cfg(feature = "no_std")]
use std::prelude::v1::*;
use std::{
borrow::Cow,
cell::RefCell,
char, fmt,
iter::{FusedIterator, Peekable},
@ -591,12 +590,62 @@ pub enum Token {
impl fmt::Display for Token {
#[inline(always)]
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.syntax())
use Token::*;
match self {
IntegerConstant(i) => write!(f, "{i}"),
#[cfg(not(feature = "no_float"))]
FloatConstant(v) => write!(f, "{v}"),
#[cfg(feature = "decimal")]
DecimalConstant(d) => write!(f, "{d}"),
StringConstant(s) => write!(f, r#""{s}""#),
InterpolatedString(..) => f.write_str("string"),
CharConstant(c) => write!(f, "{c}"),
Identifier(s) => f.write_str(s),
Reserved(s) => f.write_str(s),
#[cfg(not(feature = "no_custom_syntax"))]
Custom(s) => f.write_str(s),
LexError(err) => write!(f, "{err}"),
Comment(s) => f.write_str(s),
EOF => f.write_str("{EOF}"),
token => f.write_str(token.literal_syntax()),
}
}
}
impl Token {
/// Is the token a literal symbol?
#[must_use]
pub const fn is_literal(&self) -> bool {
use Token::*;
match self {
IntegerConstant(..) => false,
#[cfg(not(feature = "no_float"))]
FloatConstant(..) => false,
#[cfg(feature = "decimal")]
DecimalConstant(..) => false,
StringConstant(..)
| InterpolatedString(..)
| CharConstant(..)
| Identifier(..)
| Reserved(..) => false,
#[cfg(not(feature = "no_custom_syntax"))]
Custom(..) => false,
LexError(..) | Comment(..) => false,
EOF => false,
_ => true,
}
}
/// Get the literal syntax of the token.
///
/// # Panics
///
/// Panics if the token is not a literal symbol.
#[must_use]
pub const fn literal_syntax(&self) -> &'static str {
use Token::*;
@ -690,34 +739,7 @@ impl Token {
#[cfg(not(feature = "no_module"))]
As => "as",
_ => "ERROR: NOT A KEYWORD",
}
}
/// Get the syntax of the token.
#[must_use]
pub fn syntax(&self) -> Cow<'static, str> {
use Token::*;
match self {
IntegerConstant(i) => i.to_string().into(),
#[cfg(not(feature = "no_float"))]
FloatConstant(f) => f.to_string().into(),
#[cfg(feature = "decimal")]
DecimalConstant(d) => d.to_string().into(),
StringConstant(s) => format!("\"{s}\"").into(),
InterpolatedString(..) => "string".into(),
CharConstant(c) => c.to_string().into(),
Identifier(s) => s.to_string().into(),
Reserved(s) => s.to_string().into(),
#[cfg(not(feature = "no_custom_syntax"))]
Custom(s) => s.to_string().into(),
LexError(err) => err.to_string().into(),
Comment(s) => s.to_string().into(),
EOF => "{EOF}".into(),
token => token.literal_syntax().into(),
_ => panic!("token is not a literal symbol"),
}
}
@ -1127,7 +1149,7 @@ impl Token {
impl From<Token> for String {
#[inline(always)]
fn from(token: Token) -> Self {
token.syntax().into()
token.to_string()
}
}
@ -2399,7 +2421,7 @@ impl<'a> Iterator for TokenIterator<'a> {
Some((Token::Reserved(s), pos)) => (match
(s.as_str(),
#[cfg(not(feature = "no_custom_syntax"))]
(!self.engine.custom_keywords.is_empty() && self.engine.custom_keywords.contains_key(&*s)),
self.engine.custom_keywords.contains_key(&*s),
#[cfg(feature = "no_custom_syntax")]
false
)
@ -2436,7 +2458,7 @@ impl<'a> Iterator for TokenIterator<'a> {
#[cfg(feature = "no_custom_syntax")]
(.., true) => unreachable!("no custom operators"),
// Reserved keyword that is not custom and disabled.
(token, false) if !self.engine.disabled_symbols.is_empty() && self.engine.disabled_symbols.contains(token) => {
(token, false) if self.engine.disabled_symbols.contains(token) => {
let msg = format!("reserved {} '{token}' is disabled", if is_valid_identifier(token) { "keyword"} else {"symbol"});
Token::LexError(LERR::ImproperSymbol(s.to_string(), msg).into())
},
@ -2445,13 +2467,13 @@ impl<'a> Iterator for TokenIterator<'a> {
}, pos),
// Custom keyword
#[cfg(not(feature = "no_custom_syntax"))]
Some((Token::Identifier(s), pos)) if !self.engine.custom_keywords.is_empty() && self.engine.custom_keywords.contains_key(&*s) => {
Some((Token::Identifier(s), pos)) if self.engine.custom_keywords.contains_key(&*s) => {
(Token::Custom(s), pos)
}
// Custom keyword/symbol - must be disabled
#[cfg(not(feature = "no_custom_syntax"))]
Some((token, pos)) if !self.engine.custom_keywords.is_empty() && self.engine.custom_keywords.contains_key(token.literal_syntax()) => {
if !self.engine.disabled_symbols.is_empty() && self.engine.disabled_symbols.contains(token.literal_syntax()) {
Some((token, pos)) if token.is_literal() && self.engine.custom_keywords.contains_key(token.literal_syntax()) => {
if self.engine.disabled_symbols.contains(token.literal_syntax()) {
// Disabled standard keyword/symbol
(Token::Custom(Box::new(token.literal_syntax().into())), pos)
} else {
@ -2460,7 +2482,7 @@ impl<'a> Iterator for TokenIterator<'a> {
}
}
// Disabled symbol
Some((token, pos)) if !self.engine.disabled_symbols.is_empty() && self.engine.disabled_symbols.contains(token.literal_syntax()) => {
Some((token, pos)) if token.is_literal() && self.engine.disabled_symbols.contains(token.literal_syntax()) => {
(Token::Reserved(Box::new(token.literal_syntax().into())), pos)
}
// Normal symbol