kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Use SmartString in parsing.

Browse Source
This commit is contained in:
Stephen Chung 2022-02-26 17:28:58 +08:00
parent 8205547d8a
commit 9ef522b699
6 changed files with 51 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/ast/flags.rs
View File

@ -4,15 +4,6 @@ use bitflags::bitflags;
#[cfg(feature = "no_std")]
use std::prelude::v1::*;
/// A type representing the access mode of a function.
#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub enum FnAccess {
/// Private function.
Private,
/// Public function.
Public,
}
bitflags! {
/// _(internals)_ A type that holds a configuration option with bit-flags.
/// Exported under the `internals` feature only.

4
src/ast/mod.rs
View File

@ -9,13 +9,13 @@ pub mod stmt;
pub use ast::{ASTNode, AST};
pub use expr::{BinaryExpr, CustomExpr, Expr, FnCallExpr, FnCallHashes};
pub use flags::{ASTFlags, FnAccess};
pub use flags::ASTFlags;
pub use ident::Ident;
#[cfg(not(feature = "no_module"))]
#[cfg(not(feature = "no_function"))]
pub use script_fn::EncapsulatedEnviron;
#[cfg(not(feature = "no_function"))]
pub use script_fn::{ScriptFnDef, ScriptFnMetadata};
pub use script_fn::{FnAccess, ScriptFnDef, ScriptFnMetadata};
pub use stmt::{
ConditionalStmtBlock, OpAssignment, Stmt, StmtBlock, StmtBlockContainer, SwitchCases,
TryCatchBlock,

11
src/ast/script_fn.rs
View File

@ -1,12 +1,21 @@
//! Module defining script-defined functions.
#![cfg(not(feature = "no_function"))]
use super::{FnAccess, StmtBlock};
use super::StmtBlock;
use crate::{Identifier, StaticVec};
#[cfg(feature = "no_std")]
use std::prelude::v1::*;
use std::{fmt, hash::Hash};
/// A type representing the access mode of a function.
#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub enum FnAccess {
/// Private function.
Private,
/// Public function.
Public,
}
/// _(internals)_ Encapsulated AST environment.
/// Exported under the `internals` feature only.
///

2
src/lib.rs
View File

@ -365,7 +365,7 @@ pub type StaticVec<T> = smallvec::SmallVec<[T; 3]>;
///
/// Under `no_closure`, this type aliases to [`StaticVec`][crate::StaticVec] instead.
#[cfg(not(feature = "no_closure"))]
type FnArgsVec<T> = smallvec::SmallVec<[T; 8]>;
type FnArgsVec<T> = smallvec::SmallVec<[T; 5]>;
/// Inline arguments storage for function calls.
/// This type aliases to [`StaticVec`][crate::StaticVec].

32
src/parser.rs
View File

@ -19,7 +19,7 @@ use crate::types::StringsInterner;
use crate::{
calc_fn_hash, Dynamic, Engine, EvalAltResult, EvalContext, ExclusiveRange, Identifier,
ImmutableString, InclusiveRange, LexError, OptimizationLevel, ParseError, Position, Scope,
Shared, StaticVec, AST, INT, PERR,
Shared, SmartString, StaticVec, AST, INT, PERR,
};
#[cfg(feature = "no_std")]
use std::prelude::v1::*;
@ -387,7 +387,7 @@ fn match_token(input: &mut TokenStream, token: Token) -> (bool, Position) {
}
/// Parse a variable name.
fn parse_var_name(input: &mut TokenStream) -> ParseResult<(Box<str>, Position)> {
fn parse_var_name(input: &mut TokenStream) -> ParseResult<(SmartString, Position)> {
match input.next().expect(NEVER_ENDS) {
// Variable name
(Token::Identifier(s), pos) => Ok((s, pos)),
@ -403,7 +403,7 @@ fn parse_var_name(input: &mut TokenStream) -> ParseResult<(Box<str>, Position)>
}
/// Parse a symbol.
fn parse_symbol(input: &mut TokenStream) -> ParseResult<(Box<str>, Position)> {
fn parse_symbol(input: &mut TokenStream) -> ParseResult<(SmartString, Position)> {
match input.next().expect(NEVER_ENDS) {
// Symbol
(token, pos) if token.is_standard_symbol() => Ok((token.literal_syntax().into(), pos)),
@ -2059,7 +2059,7 @@ fn parse_binary_op(
Token::Custom(c) => state
.engine
.custom_keywords
.get(c.as_ref())
.get(c)
.cloned()
.ok_or_else(|| PERR::Reserved(c.to_string()).into_err(*current_pos))?,
Token::Reserved(c) if !is_valid_identifier(c.chars()) => {
@ -2084,7 +2084,7 @@ fn parse_binary_op(
Token::Custom(c) => state
.engine
.custom_keywords
.get(c.as_ref())
.get(c)
.cloned()
.ok_or_else(|| PERR::Reserved(c.to_string()).into_err(*next_pos))?,
Token::Reserved(c) if !is_valid_identifier(c.chars()) => {
@ -2182,7 +2182,7 @@ fn parse_binary_op(
if state
.engine
.custom_keywords
.get(s.as_ref())
.get(s.as_str())
.map_or(false, Option::is_some) =>
{
let hash = calc_fn_hash(&s, 2);
@ -2630,14 +2630,12 @@ fn parse_let(
// let name ...
let (name, pos) = parse_var_name(input)?;
if !settings.default_options.allow_shadowing
&& state.stack.iter().any(|(v, ..)| v == name.as_ref())
{
if !settings.default_options.allow_shadowing && state.stack.iter().any(|(v, ..)| v == &name) {
return Err(PERR::VariableExists(name.to_string()).into_err(pos));
}
if let Some(ref filter) = state.engine.def_var_filter {
let will_shadow = state.stack.iter().any(|(v, ..)| v == name.as_ref());
let will_shadow = state.stack.iter().any(|(v, ..)| v == &name);
let level = settings.level;
let is_const = access == AccessMode::ReadOnly;
let info = VarDefInfo {
@ -2820,7 +2818,7 @@ fn parse_block(
}
};
let mut statements = Vec::with_capacity(8);
let mut statements = StaticVec::new();
let prev_entry_stack_len = state.block_stack_len;
state.block_stack_len = state.stack.len();
@ -2924,7 +2922,7 @@ fn parse_stmt(
#[cfg(not(feature = "no_function"))]
#[cfg(feature = "metadata")]
let comments = {
let mut comments = Vec::<Box<str>>::new();
let mut comments = StaticVec::<SmartString>::new();
let mut comments_pos = Position::NONE;
// Handle doc-comments.
@ -3201,7 +3199,7 @@ fn parse_fn(
settings: ParseSettings,
#[cfg(not(feature = "no_function"))]
#[cfg(feature = "metadata")]
comments: Vec<Box<str>>,
comments: StaticVec<SmartString>,
) -> ParseResult<ScriptFnDef> {
#[cfg(not(feature = "unchecked"))]
settings.ensure_level_within_max_limit(state.max_expr_depth)?;
@ -3285,7 +3283,13 @@ fn parse_fn(
comments: if comments.is_empty() {
None
} else {
Some(comments.into())
Some(
comments
.into_iter()
.map(|s| s.to_string().into_boxed_str())
.collect::<Vec<_>>()
.into_boxed_slice(),
)
},
})
}

41
src/tokenizer.rs
View File

@ -5,7 +5,7 @@ use crate::engine::{
KEYWORD_FN_PTR_CURRY, KEYWORD_IS_DEF_VAR, KEYWORD_PRINT, KEYWORD_THIS, KEYWORD_TYPE_OF,
};
use crate::func::native::OnParseTokenCallback;
use crate::{Engine, LexError, StaticVec, INT, UNSIGNED_INT};
use crate::{Engine, LexError, SmartString, StaticVec, INT, UNSIGNED_INT};
#[cfg(feature = "no_std")]
use std::prelude::v1::*;
use std::{
@ -363,13 +363,13 @@ pub enum Token {
#[cfg(feature = "decimal")]
DecimalConstant(rust_decimal::Decimal),
/// An identifier.
Identifier(Box<str>),
Identifier(SmartString),
/// A character constant.
CharConstant(char),
/// A string constant.
StringConstant(Box<str>),
StringConstant(SmartString),
/// An interpolated string.
InterpolatedString(Box<str>),
InterpolatedString(SmartString),
/// `{`
LeftBrace,
/// `}`
@ -536,11 +536,11 @@ pub enum Token {
/// A lexer error.
LexError(LexError),
/// A comment block.
Comment(Box<str>),
Comment(SmartString),
/// A reserved symbol.
Reserved(Box<str>),
Reserved(SmartString),
/// A custom keyword.
Custom(Box<str>),
Custom(SmartString),
/// End of the input stream.
EOF,
}
@ -1022,9 +1022,9 @@ impl Token {
/// Convert a token into a function name, if possible.
#[cfg(not(feature = "no_function"))]
#[inline]
pub(crate) fn into_function_name_for_override(self) -> Result<Box<str>, Self> {
pub(crate) fn into_function_name_for_override(self) -> Result<SmartString, Self> {
match self {
Self::Custom(s) | Self::Identifier(s) if is_valid_function_name(&*s) => Ok(s),
Self::Custom(s) | Self::Identifier(s) if is_valid_function_name(&s) => Ok(s),
_ => Err(self),
}
}
@ -1112,9 +1112,9 @@ pub fn parse_string_literal(
verbatim: bool,
allow_line_continuation: bool,
allow_interpolation: bool,
) -> Result<(Box<str>, bool, Position), (LexError, Position)> {
let mut result = String::with_capacity(12);
let mut escape = String::with_capacity(12);
) -> Result<(SmartString, bool, Position), (LexError, Position)> {
let mut result = SmartString::new();
let mut escape = SmartString::new();
let start = *pos;
let mut first_char = Position::NONE;
@ -1146,7 +1146,6 @@ pub fn parse_string_literal(
break;
}
None => {
result += &escape;
pos.advance();
state.is_within_text_terminated_by = None;
return Err((LERR::UnterminatedString, start));
@ -1242,7 +1241,7 @@ pub fn parse_string_literal(
result.push(
char::from_u32(out_val)
.ok_or_else(|| (LERR::MalformedEscapeSequence(seq), *pos))?,
.ok_or_else(|| (LERR::MalformedEscapeSequence(seq.to_string()), *pos))?,
);
}
@ -1283,7 +1282,7 @@ pub fn parse_string_literal(
_ if !escape.is_empty() => {
escape.push(next_char);
return Err((LERR::MalformedEscapeSequence(escape), *pos));
return Err((LERR::MalformedEscapeSequence(escape.to_string()), *pos));
}
// Whitespace to skip
@ -1309,7 +1308,7 @@ pub fn parse_string_literal(
}
}
Ok((result.into(), interpolated, first_char))
Ok((result, interpolated, first_char))
}
/// Consume the next character.
@ -2291,18 +2290,18 @@ impl<'a> Iterator for TokenIterator<'a> {
(Token::Custom(s), pos)
}
// Custom keyword/symbol - must be disabled
Some((token, pos)) if self.engine.custom_keywords.contains_key(&*token.syntax()) => {
if self.engine.disabled_symbols.contains(&*token.syntax()) {
Some((token, pos)) if self.engine.custom_keywords.contains_key(token.literal_syntax()) => {
if self.engine.disabled_symbols.contains(token.literal_syntax()) {
// Disabled standard keyword/symbol
(Token::Custom(token.syntax().into()), pos)
(Token::Custom(token.literal_syntax().into()), pos)
} else {
// Active standard keyword - should never be a custom keyword!
unreachable!("{:?} is an active keyword", token)
}
}
// Disabled symbol
Some((token, pos)) if self.engine.disabled_symbols.contains(&*token.syntax()) => {
(Token::Reserved(token.syntax().into()), pos)
Some((token, pos)) if self.engine.disabled_symbols.contains(token.literal_syntax()) => {
(Token::Reserved(token.literal_syntax().into()), pos)
}
// Normal symbol
Some(r) => r,