Revise documentation.

This commit is contained in:
Stephen Chung 2022-11-29 15:50:58 +08:00
parent 0c85f0c796
commit c509cc896d
11 changed files with 146 additions and 114 deletions

View File

@ -6,23 +6,20 @@ use std::num::{NonZeroU64, NonZeroUsize};
#[cfg(feature = "no_std")] #[cfg(feature = "no_std")]
use std::prelude::v1::*; use std::prelude::v1::*;
#[cfg(debug_assertions)]
pub mod default_limits { pub mod default_limits {
#[cfg(debug_assertions)]
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
pub const MAX_CALL_STACK_DEPTH: usize = 8; pub const MAX_CALL_STACK_DEPTH: usize = 8;
#[cfg(debug_assertions)]
pub const MAX_EXPR_DEPTH: usize = 32; pub const MAX_EXPR_DEPTH: usize = 32;
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
#[cfg(debug_assertions)]
pub const MAX_FUNCTION_EXPR_DEPTH: usize = 16; pub const MAX_FUNCTION_EXPR_DEPTH: usize = 16;
}
#[cfg(not(debug_assertions))] #[cfg(not(debug_assertions))]
pub mod default_limits {
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
pub const MAX_CALL_STACK_DEPTH: usize = 64; pub const MAX_CALL_STACK_DEPTH: usize = 64;
#[cfg(not(debug_assertions))]
pub const MAX_EXPR_DEPTH: usize = 64; pub const MAX_EXPR_DEPTH: usize = 64;
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
#[cfg(not(debug_assertions))]
pub const MAX_FUNCTION_EXPR_DEPTH: usize = 32; pub const MAX_FUNCTION_EXPR_DEPTH: usize = 32;
} }
@ -55,7 +52,7 @@ pub struct Limits {
#[cfg(not(feature = "no_module"))] #[cfg(not(feature = "no_module"))]
pub max_modules: usize, pub max_modules: usize,
/// Maximum length of a [string][crate::ImmutableString]. /// Maximum length of a [string][crate::ImmutableString].
pub max_string_size: Option<NonZeroUsize>, pub max_string_len: Option<NonZeroUsize>,
/// Maximum length of an [array][crate::Array]. /// Maximum length of an [array][crate::Array].
/// ///
/// Not available under `no_index`. /// Not available under `no_index`.
@ -83,7 +80,7 @@ impl Limits {
max_operations: None, max_operations: None,
#[cfg(not(feature = "no_module"))] #[cfg(not(feature = "no_module"))]
max_modules: usize::MAX, max_modules: usize::MAX,
max_string_size: None, max_string_len: None,
#[cfg(not(feature = "no_index"))] #[cfg(not(feature = "no_index"))]
max_array_size: None, max_array_size: None,
#[cfg(not(feature = "no_object"))] #[cfg(not(feature = "no_object"))]
@ -104,7 +101,7 @@ impl Engine {
/// Is there a data size limit set? /// Is there a data size limit set?
#[inline(always)] #[inline(always)]
pub(crate) const fn has_data_size_limit(&self) -> bool { pub(crate) const fn has_data_size_limit(&self) -> bool {
self.limits.max_string_size.is_some() self.limits.max_string_len.is_some()
|| { || {
#[cfg(not(feature = "no_index"))] #[cfg(not(feature = "no_index"))]
{ {
@ -222,19 +219,19 @@ impl Engine {
#[cfg(feature = "no_function")] #[cfg(feature = "no_function")]
return 0; return 0;
} }
/// Set the maximum length of [strings][crate::ImmutableString] (0 for unlimited). /// Set the maximum length, in bytes, of [strings][crate::ImmutableString] (0 for unlimited).
/// ///
/// Not available under `unchecked`. /// Not available under `unchecked`.
#[inline(always)] #[inline(always)]
pub fn set_max_string_size(&mut self, max_size: usize) -> &mut Self { pub fn set_max_string_size(&mut self, max_len: usize) -> &mut Self {
self.limits.max_string_size = NonZeroUsize::new(max_size); self.limits.max_string_len = NonZeroUsize::new(max_len);
self self
} }
/// The maximum length of [strings][crate::ImmutableString] (0 for unlimited). /// The maximum length, in bytes, of [strings][crate::ImmutableString] (0 for unlimited).
#[inline] #[inline]
#[must_use] #[must_use]
pub const fn max_string_size(&self) -> usize { pub const fn max_string_size(&self) -> usize {
match self.limits.max_string_size { match self.limits.max_string_len {
Some(n) => n.get(), Some(n) => n.get(),
None => 0, None => 0,
} }

View File

@ -1,3 +1,4 @@
//! Placeholder settings for [`Engine`]'s limitations.
#![cfg(feature = "unchecked")] #![cfg(feature = "unchecked")]
use crate::Engine; use crate::Engine;
@ -5,7 +6,7 @@ use crate::Engine;
impl Engine { impl Engine {
/// The maximum levels of function calls allowed for a script. /// The maximum levels of function calls allowed for a script.
/// ///
/// Always returns [`usize::MAX`] under `unchecked`. /// Always returns [`usize::MAX`].
#[inline(always)] #[inline(always)]
#[must_use] #[must_use]
pub const fn max_call_levels(&self) -> usize { pub const fn max_call_levels(&self) -> usize {
@ -13,7 +14,7 @@ impl Engine {
} }
/// The maximum number of operations allowed for a script to run (0 for unlimited). /// The maximum number of operations allowed for a script to run (0 for unlimited).
/// ///
/// Always returns zero under `unchecked`. /// Always returns zero.
#[inline(always)] #[inline(always)]
#[must_use] #[must_use]
pub const fn max_operations(&self) -> u64 { pub const fn max_operations(&self) -> u64 {
@ -21,7 +22,7 @@ impl Engine {
} }
/// The maximum number of imported [modules][crate::Module] allowed for a script. /// The maximum number of imported [modules][crate::Module] allowed for a script.
/// ///
/// Always returns [`usize::MAX`] under `unchecked`. /// Always returns [`usize::MAX`].
#[inline(always)] #[inline(always)]
#[must_use] #[must_use]
pub const fn max_modules(&self) -> usize { pub const fn max_modules(&self) -> usize {
@ -29,7 +30,7 @@ impl Engine {
} }
/// The depth limit for expressions (0 for unlimited). /// The depth limit for expressions (0 for unlimited).
/// ///
/// Always returns zero under `unchecked`. /// Always returns zero.
#[inline(always)] #[inline(always)]
#[must_use] #[must_use]
pub const fn max_expr_depth(&self) -> usize { pub const fn max_expr_depth(&self) -> usize {
@ -37,7 +38,7 @@ impl Engine {
} }
/// The depth limit for expressions in functions (0 for unlimited). /// The depth limit for expressions in functions (0 for unlimited).
/// ///
/// Always returns zero under `unchecked`. /// Always returns zero.
#[inline(always)] #[inline(always)]
#[must_use] #[must_use]
pub const fn max_function_expr_depth(&self) -> usize { pub const fn max_function_expr_depth(&self) -> usize {
@ -45,7 +46,7 @@ impl Engine {
} }
/// The maximum length of [strings][crate::ImmutableString] (0 for unlimited). /// The maximum length of [strings][crate::ImmutableString] (0 for unlimited).
/// ///
/// Always returns zero under `unchecked`. /// Always returns zero.
#[inline(always)] #[inline(always)]
#[must_use] #[must_use]
pub const fn max_string_size(&self) -> usize { pub const fn max_string_size(&self) -> usize {
@ -53,7 +54,7 @@ impl Engine {
} }
/// The maximum length of [arrays][crate::Array] (0 for unlimited). /// The maximum length of [arrays][crate::Array] (0 for unlimited).
/// ///
/// Always returns zero under `unchecked`. /// Always returns zero.
#[inline(always)] #[inline(always)]
#[must_use] #[must_use]
pub const fn max_array_size(&self) -> usize { pub const fn max_array_size(&self) -> usize {
@ -61,7 +62,7 @@ impl Engine {
} }
/// The maximum size of [object maps][crate::Map] (0 for unlimited). /// The maximum size of [object maps][crate::Map] (0 for unlimited).
/// ///
/// Always returns zero under `unchecked`. /// Always returns zero.
#[inline(always)] #[inline(always)]
#[must_use] #[must_use]
pub const fn max_map_size(&self) -> usize { pub const fn max_map_size(&self) -> usize {

View File

@ -80,7 +80,7 @@ impl Engine {
) -> RhaiResultOf<()> { ) -> RhaiResultOf<()> {
if self if self
.limits .limits
.max_string_size .max_string_len
.map_or(false, |max| s > max.get()) .map_or(false, |max| s > max.get())
{ {
return Err( return Err(

View File

@ -501,13 +501,15 @@ impl Engine {
// 2) Global modules - packages // 2) Global modules - packages
// 3) Imported modules - functions marked with global namespace // 3) Imported modules - functions marked with global namespace
// 4) Global sub-modules - functions marked with global namespace // 4) Global sub-modules - functions marked with global namespace
let func = self let iter_func = self
.global_modules .global_modules
.iter() .iter()
.find_map(|m| m.get_iter(iter_type)); .find_map(|m| m.get_iter(iter_type));
#[cfg(not(feature = "no_module"))] #[cfg(not(feature = "no_module"))]
let func = func.or_else(|| global.get_iter(iter_type)).or_else(|| { let iter_func = iter_func
.or_else(|| global.get_iter(iter_type))
.or_else(|| {
self.global_sub_modules self.global_sub_modules
.as_deref() .as_deref()
.into_iter() .into_iter()
@ -515,7 +517,7 @@ impl Engine {
.find_map(|(_, m)| m.get_qualified_iter(iter_type)) .find_map(|(_, m)| m.get_qualified_iter(iter_type))
}); });
let func = func.ok_or_else(|| ERR::ErrorFor(expr.start_position()))?; let iter_func = iter_func.ok_or_else(|| ERR::ErrorFor(expr.start_position()))?;
// Restore scope at end of statement // Restore scope at end of statement
let orig_scope_len = scope.len(); let orig_scope_len = scope.len();
@ -536,7 +538,7 @@ impl Engine {
let mut result = Dynamic::UNIT; let mut result = Dynamic::UNIT;
for (x, iter_value) in func(iter_obj).enumerate() { for (x, iter_value) in iter_func(iter_obj).enumerate() {
// Increment counter // Increment counter
if counter_index < usize::MAX { if counter_index < usize::MAX {
// As the variable increments from 0, this should always work // As the variable increments from 0, this should always work
@ -604,10 +606,7 @@ impl Engine {
Stmt::TryCatch(x, ..) => { Stmt::TryCatch(x, ..) => {
let TryCatchBlock { let TryCatchBlock {
try_block, try_block,
catch_var: catch_var,
Ident {
name: catch_var, ..
},
catch_block, catch_block,
} = &**x; } = &**x;
@ -659,7 +658,7 @@ impl Engine {
}); });
if !catch_var.is_empty() { if !catch_var.is_empty() {
scope.push(catch_var.clone(), err_value); scope.push(catch_var.name.clone(), err_value);
} }
self.eval_stmt_block(global, caches, scope, this_ptr, catch_block, true) self.eval_stmt_block(global, caches, scope, this_ptr, catch_block, true)
@ -768,7 +767,7 @@ impl Engine {
#[cfg(not(feature = "no_module"))] #[cfg(not(feature = "no_module"))]
if let Some(alias) = _alias { if let Some(alias) = _alias {
scope.add_alias_by_index(scope.len() - 1, alias.name.as_str().into()); scope.add_alias_by_index(scope.len() - 1, alias.as_str().into());
} }
Ok(Dynamic::UNIT) Ok(Dynamic::UNIT)

View File

@ -186,6 +186,7 @@ impl<'a> Target<'a> {
Self::RefMut(r) => r.is_shared(), Self::RefMut(r) => r.is_shared(),
Self::SharedValue { .. } => true, Self::SharedValue { .. } => true,
Self::TempValue(value) => value.is_shared(), Self::TempValue(value) => value.is_shared(),
#[cfg(not(feature = "no_index"))]
Self::Bit { .. } Self::Bit { .. }
| Self::BitField { .. } | Self::BitField { .. }
| Self::BlobByte { .. } | Self::BlobByte { .. }

View File

@ -204,16 +204,16 @@ impl CallableFunction {
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
impl From<crate::ast::ScriptFnDef> for CallableFunction { impl From<crate::ast::ScriptFnDef> for CallableFunction {
#[inline(always)] #[inline(always)]
fn from(_func: crate::ast::ScriptFnDef) -> Self { fn from(func: crate::ast::ScriptFnDef) -> Self {
Self::Script(_func.into()) Self::Script(func.into())
} }
} }
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
impl From<Shared<crate::ast::ScriptFnDef>> for CallableFunction { impl From<Shared<crate::ast::ScriptFnDef>> for CallableFunction {
#[inline(always)] #[inline(always)]
fn from(_func: Shared<crate::ast::ScriptFnDef>) -> Self { fn from(func: Shared<crate::ast::ScriptFnDef>) -> Self {
Self::Script(_func) Self::Script(func)
} }
} }

View File

@ -1464,7 +1464,7 @@ impl Engine {
// Restore the strings interner by swapping it back // Restore the strings interner by swapping it back
std::mem::swap(state.interned_strings, new_state.interned_strings); std::mem::swap(state.interned_strings, new_state.interned_strings);
let (expr, func) = result?; let (expr, f) = result?;
#[cfg(not(feature = "no_closure"))] #[cfg(not(feature = "no_closure"))]
new_state new_state
@ -1490,8 +1490,8 @@ impl Engine {
} }
})?; })?;
let hash_script = calc_fn_hash(None, &func.name, func.params.len()); let hash_script = calc_fn_hash(None, &f.name, f.params.len());
lib.insert(hash_script, func.into()); lib.insert(hash_script, f.into());
expr expr
} }
@ -2170,42 +2170,42 @@ impl Engine {
)), )),
// lhs.nnn::func(...) - syntax error // lhs.nnn::func(...) - syntax error
#[cfg(not(feature = "no_module"))] #[cfg(not(feature = "no_module"))]
(.., Expr::FnCall(func, ..)) if func.is_qualified() => { (.., Expr::FnCall(f, ..)) if f.is_qualified() => {
Err(PERR::PropertyExpected.into_err(func.namespace.position())) Err(PERR::PropertyExpected.into_err(f.namespace.position()))
} }
// lhs.Fn() or lhs.eval() // lhs.Fn() or lhs.eval()
(.., Expr::FnCall(func, func_pos)) (.., Expr::FnCall(f, func_pos))
if func.args.is_empty() if f.args.is_empty()
&& [crate::engine::KEYWORD_FN_PTR, crate::engine::KEYWORD_EVAL] && [crate::engine::KEYWORD_FN_PTR, crate::engine::KEYWORD_EVAL]
.contains(&func.name.as_str()) => .contains(&f.name.as_str()) =>
{ {
let err_msg = format!( let err_msg = format!(
"'{}' should not be called in method style. Try {}(...);", "'{}' should not be called in method style. Try {}(...);",
func.name, func.name f.name, f.name
); );
Err(LexError::ImproperSymbol(func.name.to_string(), err_msg).into_err(func_pos)) Err(LexError::ImproperSymbol(f.name.to_string(), err_msg).into_err(func_pos))
} }
// lhs.func!(...) // lhs.func!(...)
(.., Expr::FnCall(func, func_pos)) if func.capture_parent_scope => { (.., Expr::FnCall(f, func_pos)) if f.capture_parent_scope => {
Err(PERR::MalformedCapture( Err(PERR::MalformedCapture(
"method-call style does not support running within the caller's scope".into(), "method-call style does not support running within the caller's scope".into(),
) )
.into_err(func_pos)) .into_err(func_pos))
} }
// lhs.func(...) // lhs.func(...)
(lhs, Expr::FnCall(mut func, func_pos)) => { (lhs, Expr::FnCall(mut f, func_pos)) => {
// Recalculate hash // Recalculate hash
func.hashes = if is_valid_function_name(&func.name) { f.hashes = if is_valid_function_name(&f.name) {
FnCallHashes::from_all( FnCallHashes::from_all(
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
calc_fn_hash(None, &func.name, func.args.len()), calc_fn_hash(None, &f.name, f.args.len()),
calc_fn_hash(None, &func.name, func.args.len() + 1), calc_fn_hash(None, &f.name, f.args.len() + 1),
) )
} else { } else {
FnCallHashes::from_native(calc_fn_hash(None, &func.name, func.args.len() + 1)) FnCallHashes::from_native(calc_fn_hash(None, &f.name, f.args.len() + 1))
}; };
let rhs = Expr::MethodCall(func, func_pos); let rhs = Expr::MethodCall(f, func_pos);
Ok(Expr::Dot(BinaryExpr { lhs, rhs }.into(), op_flags, op_pos)) Ok(Expr::Dot(BinaryExpr { lhs, rhs }.into(), op_flags, op_pos))
} }
// lhs.dot_lhs.dot_rhs or lhs.dot_lhs[idx_rhs] // lhs.dot_lhs.dot_rhs or lhs.dot_lhs[idx_rhs]
@ -2224,8 +2224,8 @@ impl Engine {
} }
// lhs.module::func().dot_rhs or lhs.module::func()[idx_rhs] - syntax error // lhs.module::func().dot_rhs or lhs.module::func()[idx_rhs] - syntax error
#[cfg(not(feature = "no_module"))] #[cfg(not(feature = "no_module"))]
Expr::FnCall(func, ..) if func.is_qualified() => { Expr::FnCall(f, ..) if f.is_qualified() => {
Err(PERR::PropertyExpected.into_err(func.namespace.position())) Err(PERR::PropertyExpected.into_err(f.namespace.position()))
} }
// lhs.id.dot_rhs or lhs.id[idx_rhs] // lhs.id.dot_rhs or lhs.id[idx_rhs]
Expr::Variable(..) | Expr::Property(..) => { Expr::Variable(..) | Expr::Property(..) => {
@ -2243,24 +2243,20 @@ impl Engine {
Ok(Expr::Dot(BinaryExpr { lhs, rhs }.into(), op_flags, op_pos)) Ok(Expr::Dot(BinaryExpr { lhs, rhs }.into(), op_flags, op_pos))
} }
// lhs.func().dot_rhs or lhs.func()[idx_rhs] // lhs.func().dot_rhs or lhs.func()[idx_rhs]
Expr::FnCall(mut func, func_pos) => { Expr::FnCall(mut f, func_pos) => {
// Recalculate hash // Recalculate hash
func.hashes = if is_valid_function_name(&func.name) { f.hashes = if is_valid_function_name(&f.name) {
FnCallHashes::from_all( FnCallHashes::from_all(
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
calc_fn_hash(None, &func.name, func.args.len()), calc_fn_hash(None, &f.name, f.args.len()),
calc_fn_hash(None, &func.name, func.args.len() + 1), calc_fn_hash(None, &f.name, f.args.len() + 1),
) )
} else { } else {
FnCallHashes::from_native(calc_fn_hash( FnCallHashes::from_native(calc_fn_hash(None, &f.name, f.args.len() + 1))
None,
&func.name,
func.args.len() + 1,
))
}; };
let new_lhs = BinaryExpr { let new_lhs = BinaryExpr {
lhs: Expr::MethodCall(func, func_pos), lhs: Expr::MethodCall(f, func_pos),
rhs: x.rhs, rhs: x.rhs,
} }
.into(); .into();
@ -3303,7 +3299,7 @@ impl Engine {
max_expr_depth: self.max_function_expr_depth(), max_expr_depth: self.max_function_expr_depth(),
}; };
let func = self.parse_fn( let f = self.parse_fn(
input, input,
new_state, new_state,
lib, lib,
@ -3312,22 +3308,21 @@ impl Engine {
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
#[cfg(feature = "metadata")] #[cfg(feature = "metadata")]
comments, comments,
); )?;
// Restore parse state // Restore parse state
let func = func?;
let hash = calc_fn_hash(None, &func.name, func.params.len()); let hash = calc_fn_hash(None, &f.name, f.params.len());
if !lib.is_empty() && lib.contains_key(&hash) { if !lib.is_empty() && lib.contains_key(&hash) {
return Err(PERR::FnDuplicatedDefinition( return Err(PERR::FnDuplicatedDefinition(
func.name.to_string(), f.name.to_string(),
func.params.len(), f.params.len(),
) )
.into_err(pos)); .into_err(pos));
} }
lib.insert(hash, func.into()); lib.insert(hash, f.into());
Ok(Stmt::Noop(pos)) Ok(Stmt::Noop(pos))
} }

View File

@ -852,7 +852,7 @@ impl From<Token> for String {
#[derive(Debug, Clone, Eq, PartialEq, Default)] #[derive(Debug, Clone, Eq, PartialEq, Default)]
pub struct TokenizeState { pub struct TokenizeState {
/// Maximum length of a string. /// Maximum length of a string.
pub max_string_size: Option<NonZeroUsize>, pub max_string_len: Option<NonZeroUsize>,
/// Can the next token be a unary operator? /// Can the next token be a unary operator?
pub next_token_cannot_be_unary: bool, pub next_token_cannot_be_unary: bool,
/// Shared object to allow controlling the tokenizer externally. /// Shared object to allow controlling the tokenizer externally.
@ -879,6 +879,18 @@ pub trait InputStream {
fn peek_next(&mut self) -> Option<char>; fn peek_next(&mut self) -> Option<char>;
} }
/// Return error if the string is longer than the maximum length.
#[inline]
fn ensure_string_len_within_limit(max: Option<NonZeroUsize>, value: &str) -> Result<(), LexError> {
if let Some(max) = max {
if value.len() > max.get() {
return Err(LexError::StringTooLong(max.get()));
}
}
Ok(())
}
/// _(internals)_ Parse a string literal ended by a specified termination character. /// _(internals)_ Parse a string literal ended by a specified termination character.
/// Exported under the `internals` feature only. /// Exported under the `internals` feature only.
/// ///
@ -968,11 +980,8 @@ pub fn parse_string_literal(
break; break;
} }
if let Some(max) = state.max_string_size { ensure_string_len_within_limit(state.max_string_len, &result)
if result.len() > max.get() { .map_err(|err| (err, start))?;
return Err((LexError::StringTooLong(max.get()), *pos));
}
}
// Close wrapper // Close wrapper
if termination_char == next_char && escape.is_empty() { if termination_char == next_char && escape.is_empty() {
@ -1107,11 +1116,7 @@ pub fn parse_string_literal(
} }
} }
if let Some(max) = state.max_string_size { ensure_string_len_within_limit(state.max_string_len, &result).map_err(|err| (err, start))?;
if result.len() > max.get() {
return Err((LexError::StringTooLong(max.get()), *pos));
}
}
Ok((result, interpolated, first_char)) Ok((result, interpolated, first_char))
} }
@ -1430,11 +1435,17 @@ fn get_next_token_inner(
// letter or underscore ... // letter or underscore ...
#[cfg(not(feature = "unicode-xid-ident"))] #[cfg(not(feature = "unicode-xid-ident"))]
('a'..='z' | '_' | 'A'..='Z', ..) => { ('a'..='z' | '_' | 'A'..='Z', ..) => {
return Some(get_token_as_identifier(stream, pos, start_pos, c)); return Some(
parse_identifier_token(stream, pos, start_pos, c)
.unwrap_or_else(|err| (Token::LexError(err.into()), start_pos)),
);
} }
#[cfg(feature = "unicode-xid-ident")] #[cfg(feature = "unicode-xid-ident")]
(ch, ..) if unicode_xid::UnicodeXID::is_xid_start(ch) || ch == '_' => { (ch, ..) if unicode_xid::UnicodeXID::is_xid_start(ch) || ch == '_' => {
return Some(get_token_as_identifier(stream, pos, start_pos, c)); return Some(
parse_identifier_token(stream, pos, start_pos, c)
.unwrap_or_else(|err| (Token::LexError(err.into()), start_pos)),
);
} }
// " - string literal // " - string literal
@ -1902,12 +1913,12 @@ fn get_next_token_inner(
} }
/// Get the next token, parsing it as an identifier. /// Get the next token, parsing it as an identifier.
fn get_token_as_identifier( fn parse_identifier_token(
stream: &mut impl InputStream, stream: &mut impl InputStream,
pos: &mut Position, pos: &mut Position,
start_pos: Position, start_pos: Position,
first_char: char, first_char: char,
) -> (Token, Position) { ) -> Result<(Token, Position), LexError> {
let mut identifier = SmartString::new_const(); let mut identifier = SmartString::new_const();
identifier.push(first_char); identifier.push(first_char);
@ -1922,19 +1933,20 @@ fn get_token_as_identifier(
} }
if let Some(token) = Token::lookup_symbol_from_syntax(&identifier) { if let Some(token) = Token::lookup_symbol_from_syntax(&identifier) {
return (token, start_pos); return Ok((token, start_pos));
} else if Token::is_reserved_keyword(&identifier) { }
return (Token::Reserved(Box::new(identifier)), start_pos); if Token::is_reserved_keyword(&identifier) {
return Ok((Token::Reserved(Box::new(identifier)), start_pos));
} }
if !is_valid_identifier(&identifier) { if !is_valid_identifier(&identifier) {
return ( return Ok((
Token::LexError(LERR::MalformedIdentifier(identifier.to_string()).into()), Token::LexError(LERR::MalformedIdentifier(identifier.to_string()).into()),
start_pos, start_pos,
); ));
} }
(Token::Identifier(identifier.into()), start_pos) Ok((Token::Identifier(identifier.into()), start_pos))
} }
/// Is a keyword allowed as a function? /// Is a keyword allowed as a function?
@ -2236,10 +2248,7 @@ impl Engine {
TokenIterator { TokenIterator {
engine: self, engine: self,
state: TokenizeState { state: TokenizeState {
#[cfg(not(feature = "unchecked"))] max_string_len: NonZeroUsize::new(self.max_string_size()),
max_string_size: self.limits.max_string_size,
#[cfg(feature = "unchecked")]
max_string_size: None,
next_token_cannot_be_unary: false, next_token_cannot_be_unary: false,
tokenizer_control: buffer, tokenizer_control: buffer,
comment_level: 0, comment_level: 0,

View File

@ -19,7 +19,7 @@ pub enum LexError {
UnexpectedInput(String), UnexpectedInput(String),
/// A string literal is not terminated before a new-line or EOF. /// A string literal is not terminated before a new-line or EOF.
UnterminatedString, UnterminatedString,
/// An identifier is in an invalid format. /// An identifier or string literal is longer than the maximum allowed length.
StringTooLong(usize), StringTooLong(usize),
/// An string/character/numeric escape sequence is in an invalid format. /// An string/character/numeric escape sequence is in an invalid format.
MalformedEscapeSequence(String), MalformedEscapeSequence(String),
@ -44,11 +44,7 @@ impl fmt::Display for LexError {
Self::MalformedChar(s) => write!(f, "Invalid character: '{s}'"), Self::MalformedChar(s) => write!(f, "Invalid character: '{s}'"),
Self::MalformedIdentifier(s) => write!(f, "Variable name is not proper: '{s}'"), Self::MalformedIdentifier(s) => write!(f, "Variable name is not proper: '{s}'"),
Self::UnterminatedString => f.write_str("Open string is not terminated"), Self::UnterminatedString => f.write_str("Open string is not terminated"),
Self::StringTooLong(max) => write!( Self::StringTooLong(max) => write!(f, "String is too long (max {max})"),
f,
"Length of string literal exceeds the maximum limit ({})",
max
),
Self::ImproperSymbol(s, d) if d.is_empty() => { Self::ImproperSymbol(s, d) if d.is_empty() => {
write!(f, "Invalid symbol encountered: '{s}'") write!(f, "Invalid symbol encountered: '{s}'")
} }
@ -262,7 +258,7 @@ impl From<LexError> for ParseErrorType {
fn from(err: LexError) -> Self { fn from(err: LexError) -> Self {
match err { match err {
LexError::StringTooLong(max) => { LexError::StringTooLong(max) => {
Self::LiteralTooLarge("Length of string literal".to_string(), max) Self::LiteralTooLarge("Length of string".to_string(), max)
} }
_ => Self::BadInput(err), _ => Self::BadInput(err),
} }

View File

@ -1,3 +1,4 @@
//! Script character position type.
#![cfg(not(feature = "no_position"))] #![cfg(not(feature = "no_position"))]
#[cfg(feature = "no_std")] #[cfg(feature = "no_std")]
@ -48,6 +49,8 @@ impl Position {
Self { line, pos: _pos } Self { line, pos: _pos }
} }
/// Get the line number (1-based), or [`None`] if there is no position. /// Get the line number (1-based), or [`None`] if there is no position.
///
/// Always returns [`None`] under `no_position`.
#[inline] #[inline]
#[must_use] #[must_use]
pub const fn line(self) -> Option<usize> { pub const fn line(self) -> Option<usize> {
@ -58,6 +61,8 @@ impl Position {
} }
} }
/// Get the character position (1-based), or [`None`] if at beginning of a line. /// Get the character position (1-based), or [`None`] if at beginning of a line.
///
/// Always returns [`None`] under `no_position`.
#[inline] #[inline]
#[must_use] #[must_use]
pub const fn position(self) -> Option<usize> { pub const fn position(self) -> Option<usize> {
@ -100,18 +105,24 @@ impl Position {
} }
} }
/// Is this [`Position`] at the beginning of a line? /// Is this [`Position`] at the beginning of a line?
///
/// Always returns `false` under `no_position`.
#[inline] #[inline]
#[must_use] #[must_use]
pub const fn is_beginning_of_line(self) -> bool { pub const fn is_beginning_of_line(self) -> bool {
self.pos == 0 && !self.is_none() self.pos == 0 && !self.is_none()
} }
/// Is there no [`Position`]? /// Is there no [`Position`]?
///
/// Always returns `true` under `no_position`.
#[inline] #[inline]
#[must_use] #[must_use]
pub const fn is_none(self) -> bool { pub const fn is_none(self) -> bool {
self.line == 0 && self.pos == 0 self.line == 0 && self.pos == 0
} }
/// Returns an fallback [`Position`] if it is [`NONE`][Position::NONE]? /// Returns an fallback [`Position`] if it is [`NONE`][Position::NONE]?
///
/// Always returns the fallback under `no_position`.
#[inline] #[inline]
#[must_use] #[must_use]
pub const fn or_else(self, pos: Self) -> Self { pub const fn or_else(self, pos: Self) -> Self {
@ -219,18 +230,24 @@ impl Span {
Self { start, end } Self { start, end }
} }
/// Is this [`Span`] non-existent? /// Is this [`Span`] non-existent?
///
/// Always returns `true` under `no_position`.
#[inline] #[inline]
#[must_use] #[must_use]
pub const fn is_none(&self) -> bool { pub const fn is_none(&self) -> bool {
self.start.is_none() && self.end.is_none() self.start.is_none() && self.end.is_none()
} }
/// Get the [`Span`]'s starting [position][Position]. /// Get the [`Span`]'s starting [position][Position].
///
/// Always returns [`Position::NONE`] under `no_position`.
#[inline(always)] #[inline(always)]
#[must_use] #[must_use]
pub const fn start(&self) -> Position { pub const fn start(&self) -> Position {
self.start self.start
} }
/// Get the [`Span`]'s ending [position][Position]. /// Get the [`Span`]'s ending [position][Position].
///
/// Always returns [`Position::NONE`] under `no_position`.
#[inline(always)] #[inline(always)]
#[must_use] #[must_use]
pub const fn end(&self) -> Position { pub const fn end(&self) -> Position {

View File

@ -1,3 +1,4 @@
//! Placeholder script character position type.
#![cfg(feature = "no_position")] #![cfg(feature = "no_position")]
#![allow(unused_variables)] #![allow(unused_variables)]
@ -25,12 +26,16 @@ impl Position {
Self Self
} }
/// Get the line number (1-based), or [`None`] if there is no position. /// Get the line number (1-based), or [`None`] if there is no position.
///
/// Always returns [`None`].
#[inline(always)] #[inline(always)]
#[must_use] #[must_use]
pub const fn line(self) -> Option<usize> { pub const fn line(self) -> Option<usize> {
None None
} }
/// Get the character position (1-based), or [`None`] if at beginning of a line. /// Get the character position (1-based), or [`None`] if at beginning of a line.
///
/// Always returns [`None`].
#[inline(always)] #[inline(always)]
#[must_use] #[must_use]
pub const fn position(self) -> Option<usize> { pub const fn position(self) -> Option<usize> {
@ -46,18 +51,24 @@ impl Position {
#[inline(always)] #[inline(always)]
pub(crate) fn new_line(&mut self) {} pub(crate) fn new_line(&mut self) {}
/// Is this [`Position`] at the beginning of a line? /// Is this [`Position`] at the beginning of a line?
///
/// Always returns `false`.
#[inline(always)] #[inline(always)]
#[must_use] #[must_use]
pub const fn is_beginning_of_line(self) -> bool { pub const fn is_beginning_of_line(self) -> bool {
false false
} }
/// Is there no [`Position`]? /// Is there no [`Position`]?
///
/// Always returns `true`.
#[inline(always)] #[inline(always)]
#[must_use] #[must_use]
pub const fn is_none(self) -> bool { pub const fn is_none(self) -> bool {
true true
} }
/// Returns an fallback [`Position`] if it is [`NONE`][Position::NONE]? /// Returns an fallback [`Position`] if it is [`NONE`][Position::NONE]?
///
/// Always returns the fallback.
#[inline(always)] #[inline(always)]
#[must_use] #[must_use]
pub const fn or_else(self, pos: Self) -> Self { pub const fn or_else(self, pos: Self) -> Self {
@ -130,22 +141,28 @@ impl Span {
Self Self
} }
/// Is this [`Span`] non-existent? /// Is this [`Span`] non-existent?
///
/// Always returns `true`.
#[inline(always)] #[inline(always)]
#[must_use] #[must_use]
pub const fn is_none(&self) -> bool { pub const fn is_none(&self) -> bool {
true true
} }
/// Get the [`Span`]'s starting [position][Position]. /// Get the [`Span`]'s starting [position][Position].
///
/// Always returns [`Position::NONE`].
#[inline(always)] #[inline(always)]
#[must_use] #[must_use]
pub const fn start(&self) -> Position { pub const fn start(&self) -> Position {
Position Position::NONE
} }
/// Get the [`Span`]'s ending [position][Position]. /// Get the [`Span`]'s ending [position][Position].
///
/// Always returns [`Position::NONE`].
#[inline(always)] #[inline(always)]
#[must_use] #[must_use]
pub const fn end(&self) -> Position { pub const fn end(&self) -> Position {
Position Position::NONE
} }
} }