From 28743594d016cb895cec6327fbdfb10c690cf0cc Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Sat, 13 Aug 2022 18:07:42 +0800 Subject: [PATCH] Use interned strings for AST nodes. --- src/ast/expr.rs | 14 ++- src/ast/ident.rs | 12 +-- src/ast/script_fn.rs | 10 +- src/ast/stmt.rs | 2 +- src/eval/expr.rs | 6 +- src/eval/global_state.rs | 4 +- src/eval/stmt.rs | 6 +- src/module/mod.rs | 4 +- src/packages/lang_core.rs | 4 +- src/parser.rs | 193 ++++++++++++++++++++------------------ src/types/interner.rs | 97 +++---------------- 11 files changed, 147 insertions(+), 205 deletions(-) diff --git a/src/ast/expr.rs b/src/ast/expr.rs index 16269430..e5208eee 100644 --- a/src/ast/expr.rs +++ b/src/ast/expr.rs @@ -60,7 +60,7 @@ pub struct CustomExpr { /// List of keywords. pub inputs: StaticVec, /// List of tokens actually parsed. - pub tokens: StaticVec, + pub tokens: StaticVec, /// Is the current [`Scope`][crate::Scope] possibly modified by this custom statement /// (e.g. introducing a new variable)? pub scope_may_be_changed: bool, @@ -183,7 +183,7 @@ pub struct FnCallExpr { #[cfg(not(feature = "no_module"))] pub namespace: super::Namespace, /// Function name. - pub name: Identifier, + pub name: ImmutableString, /// Pre-calculated hashes. pub hashes: FnCallHashes, /// List of function call argument expressions. @@ -392,14 +392,18 @@ pub enum Expr { /// This is to avoid reading a pointer redirection during each variable access. Variable( #[cfg(not(feature = "no_module"))] - Box<(Option, super::Namespace, u64, Identifier)>, - #[cfg(feature = "no_module")] Box<(Option, (), u64, Identifier)>, + Box<(Option, super::Namespace, u64, ImmutableString)>, + #[cfg(feature = "no_module")] Box<(Option, (), u64, ImmutableString)>, Option, Position, ), /// Property access - ((getter, hash), (setter, hash), prop) Property( - Box<((Identifier, u64), (Identifier, u64), ImmutableString)>, + Box<( + (ImmutableString, u64), + (ImmutableString, u64), + ImmutableString, + )>, Position, ), /// xxx `.` method `(` expr `,` ... `)` diff --git a/src/ast/ident.rs b/src/ast/ident.rs index 415de2bc..13ab4ef9 100644 --- a/src/ast/ident.rs +++ b/src/ast/ident.rs @@ -1,6 +1,6 @@ //! Module defining script identifiers. -use crate::{Identifier, Position}; +use crate::{ImmutableString, Position}; #[cfg(feature = "no_std")] use std::prelude::v1::*; use std::{ @@ -14,7 +14,7 @@ use std::{ #[derive(Clone, Eq, PartialEq, Hash)] pub struct Ident { /// Identifier name. - pub name: Identifier, + pub name: ImmutableString, /// Position. pub pos: Position, } @@ -34,7 +34,7 @@ impl AsRef for Ident { } impl Deref for Ident { - type Target = Identifier; + type Target = ImmutableString; #[inline(always)] fn deref(&self) -> &Self::Target { @@ -50,12 +50,6 @@ impl DerefMut for Ident { } impl Ident { - /// An empty [`Ident`]. - pub const EMPTY: Self = Self { - name: Identifier::new_const(), - pos: Position::NONE, - }; - /// Get the name of the identifier as a string slice. #[inline(always)] #[must_use] diff --git a/src/ast/script_fn.rs b/src/ast/script_fn.rs index f56a7450..77b67107 100644 --- a/src/ast/script_fn.rs +++ b/src/ast/script_fn.rs @@ -2,7 +2,7 @@ #![cfg(not(feature = "no_function"))] use super::{FnAccess, StmtBlock}; -use crate::{Identifier, SmartString, StaticVec}; +use crate::{Identifier, ImmutableString, StaticVec}; #[cfg(feature = "no_std")] use std::prelude::v1::*; use std::{fmt, hash::Hash}; @@ -38,11 +38,11 @@ pub struct ScriptFnDef { #[cfg(not(feature = "no_function"))] pub environ: Option, /// Function name. - pub name: Identifier, + pub name: ImmutableString, /// Function access mode. pub access: FnAccess, /// Names of function parameters. - pub params: StaticVec, + pub params: StaticVec, /// _(metadata)_ Function doc-comments (if any). /// Exported under the `metadata` feature only. /// @@ -71,7 +71,7 @@ impl fmt::Display for ScriptFnDef { self.name, self.params .iter() - .map(SmartString::as_str) + .map(|s| s.as_str()) .collect::>() .join(", ") ) @@ -132,7 +132,7 @@ impl<'a> From<&'a ScriptFnDef> for ScriptFnMetadata<'a> { fn from(value: &'a ScriptFnDef) -> Self { Self { name: &value.name, - params: value.params.iter().map(SmartString::as_str).collect(), + params: value.params.iter().map(|s| s.as_str()).collect(), access: value.access, #[cfg(feature = "metadata")] comments: value.comments.iter().map(<_>::as_ref).collect(), diff --git a/src/ast/stmt.rs b/src/ast/stmt.rs index e853701a..c634bfc1 100644 --- a/src/ast/stmt.rs +++ b/src/ast/stmt.rs @@ -594,7 +594,7 @@ pub enum Stmt { /// This variant does not map to any language structure. It is currently only used only to /// convert a normal variable into a shared variable when the variable is _captured_ by a closure. #[cfg(not(feature = "no_closure"))] - Share(Box, Position), + Share(crate::ImmutableString, Position), } impl Default for Stmt { diff --git a/src/eval/expr.rs b/src/eval/expr.rs index 49ca5292..51265efc 100644 --- a/src/eval/expr.rs +++ b/src/eval/expr.rs @@ -94,7 +94,7 @@ impl Engine { if namespace.len() == 1 && namespace.root() == crate::engine::KEYWORD_GLOBAL { if let Some(ref constants) = global.constants { if let Some(value) = - crate::func::locked_write(constants).get_mut(var_name) + crate::func::locked_write(constants).get_mut(var_name.as_str()) { let mut target: Target = value.clone().into(); // Module variables are constant @@ -155,7 +155,7 @@ impl Engine { if lib .iter() .flat_map(|&m| m.iter_script_fn()) - .any(|(_, _, f, ..)| f == v.3) => + .any(|(_, _, f, ..)| f == v.3.as_str()) => { let val: Dynamic = crate::FnPtr::new_unchecked(v.3.as_str(), Default::default()).into(); @@ -497,7 +497,7 @@ impl Engine { // The first token acts as the custom syntax's key let key_token = custom.tokens.first().unwrap(); // The key should exist, unless the AST is compiled in a different Engine - let custom_def = self.custom_syntax.get(key_token).ok_or_else(|| { + let custom_def = self.custom_syntax.get(key_token.as_str()).ok_or_else(|| { Box::new(ERR::ErrorCustomSyntax( format!("Invalid custom syntax prefix: {key_token}"), custom.tokens.iter().map(<_>::to_string).collect(), diff --git a/src/eval/global_state.rs b/src/eval/global_state.rs index 225f724c..ebee0e0a 100644 --- a/src/eval/global_state.rs +++ b/src/eval/global_state.rs @@ -1,6 +1,6 @@ //! Global runtime state. -use crate::{Dynamic, Engine, Identifier}; +use crate::{Dynamic, Engine, Identifier, ImmutableString}; #[cfg(feature = "no_std")] use std::prelude::v1::*; use std::{fmt, marker::PhantomData}; @@ -9,7 +9,7 @@ use std::{fmt, marker::PhantomData}; #[cfg(not(feature = "no_module"))] #[cfg(not(feature = "no_function"))] pub type GlobalConstants = - crate::Shared>>; + crate::Shared>>; /// _(internals)_ Global runtime states. /// Exported under the `internals` feature only. diff --git a/src/eval/stmt.rs b/src/eval/stmt.rs index 45ed998a..8e8ab47c 100644 --- a/src/eval/stmt.rs +++ b/src/eval/stmt.rs @@ -913,7 +913,7 @@ impl Engine { #[cfg(not(feature = "no_module"))] if let Some(alias) = _alias { - scope.add_alias_by_index(scope.len() - 1, alias.name.clone()); + scope.add_alias_by_index(scope.len() - 1, alias.name.as_str().into()); } Ok(Dynamic::UNIT) @@ -995,11 +995,11 @@ impl Engine { // Export statement #[cfg(not(feature = "no_module"))] Stmt::Export(x, ..) => { - let (Ident { name, pos, .. }, alias) = &**x; + let (Ident { name, pos, .. }, Ident { name: alias, .. }) = &**x; // Mark scope variables as public if let Some((index, ..)) = scope.get_index(name) { let alias = if alias.is_empty() { name } else { alias }.clone(); - scope.add_alias_by_index(index, alias); + scope.add_alias_by_index(index, alias.into()); Ok(Dynamic::UNIT) } else { Err(ERR::ErrorVariableNotFound(name.to_string(), *pos).into()) diff --git a/src/module/mod.rs b/src/module/mod.rs index 9507fc16..37828a66 100644 --- a/src/module/mod.rs +++ b/src/module/mod.rs @@ -759,12 +759,12 @@ impl Module { let num_params = fn_def.params.len(); let hash_script = crate::calc_fn_hash(&fn_def.name, num_params); #[cfg(feature = "metadata")] - let params_info = fn_def.params.iter().cloned().collect(); + let params_info = fn_def.params.iter().map(Into::into).collect(); self.functions.insert( hash_script, FuncInfo { metadata: FnMetadata { - name: fn_def.name.clone(), + name: fn_def.name.as_str().into(), namespace: FnNamespace::Internal, access: fn_def.access, params: num_params, diff --git a/src/packages/lang_core.rs b/src/packages/lang_core.rs index e2a1b62c..379fd4c7 100644 --- a/src/packages/lang_core.rs +++ b/src/packages/lang_core.rs @@ -143,7 +143,7 @@ fn collect_fn_metadata( if !namespace.is_empty() { map.insert("namespace".into(), dict.get(namespace).into()); } - map.insert("name".into(), dict.get(&func.name).into()); + map.insert("name".into(), dict.get(func.name.as_str()).into()); map.insert( "access".into(), dict.get(match func.access { @@ -160,7 +160,7 @@ fn collect_fn_metadata( "params".into(), func.params .iter() - .map(|p| dict.get(p).into()) + .map(|p| dict.get(p.as_str()).into()) .collect::() .into(), ); diff --git a/src/parser.rs b/src/parser.rs index 44116d33..d9ee3996 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -43,13 +43,22 @@ const NEVER_ENDS: &str = "`Token`"; /// Unroll `switch` ranges no larger than this. const SMALL_SWITCH_RANGE: usize = 16; +#[derive(Debug, Default, Clone)] +pub struct InternedStrings<'e> { + pub main: StringsInterner<'e>, + #[cfg(not(feature = "no_object"))] + pub getters: StringsInterner<'e>, + #[cfg(not(feature = "no_object"))] + pub setters: StringsInterner<'e>, +} + /// _(internals)_ A type that encapsulates the current state of the parser. /// Exported under the `internals` feature only. pub struct ParseState<'e> { /// Input stream buffer containing the next character to read. pub tokenizer_control: TokenizerControl, - /// Interned strings. - interned_strings: StringsInterner<'e>, + /// String interners. + interned_strings: InternedStrings<'e>, /// External [scope][Scope] with constants. pub scope: &'e Scope<'e>, /// Global runtime state. @@ -71,10 +80,10 @@ pub struct ParseState<'e> { pub allow_capture: bool, /// Encapsulates a local stack with imported [module][crate::Module] names. #[cfg(not(feature = "no_module"))] - pub imports: StaticVec, + pub imports: StaticVec, /// List of globally-imported [module][crate::Module] names. #[cfg(not(feature = "no_module"))] - pub global_imports: StaticVec, + pub global_imports: StaticVec, /// Maximum levels of expression nesting (0 for unlimited). #[cfg(not(feature = "unchecked"))] pub max_expr_depth: usize, @@ -109,7 +118,7 @@ impl<'e> ParseState<'e> { pub fn new( engine: &Engine, scope: &'e Scope, - interned_strings: StringsInterner<'e>, + interners: InternedStrings<'e>, tokenizer_control: TokenizerControl, ) -> Self { Self { @@ -119,7 +128,7 @@ impl<'e> ParseState<'e> { external_vars: Vec::new(), #[cfg(not(feature = "no_closure"))] allow_capture: true, - interned_strings, + interned_strings: interners, scope, global: GlobalRuntimeState::new(engine), stack: Scope::new(), @@ -235,28 +244,10 @@ impl<'e> ParseState<'e> { .iter() .rev() .enumerate() - .find(|&(.., n)| n == name) + .find(|(.., n)| n.as_str() == name) .and_then(|(i, ..)| NonZeroUsize::new(i + 1)) } - /// Get an interned identifier, creating one if it is not yet interned. - #[inline(always)] - #[must_use] - pub fn get_identifier(&mut self, text: impl AsRef + Into) -> Identifier { - self.get_identifier_with_prefix("", text).into() - } - - /// Get an interned identifier, creating one if it is not yet interned. - #[inline(always)] - #[must_use] - pub fn get_identifier_with_prefix( - &mut self, - prefix: impl AsRef, - text: impl AsRef + Into, - ) -> Identifier { - self.interned_strings.get_with_prefix(prefix, text).into() - } - /// Get an interned string, creating one if it is not yet interned. #[inline(always)] #[allow(dead_code)] @@ -265,19 +256,35 @@ impl<'e> ParseState<'e> { &mut self, text: impl AsRef + Into, ) -> ImmutableString { - self.get_interned_string_with_prefix("", text) + self.interned_strings.main.get(text) } - /// Get an interned string, creating one if it is not yet interned. + /// Get an interned property getter, creating one if it is not yet interned. + #[cfg(not(feature = "no_object"))] #[inline(always)] #[allow(dead_code)] #[must_use] - pub fn get_interned_string_with_prefix( + pub fn get_interned_getter( &mut self, - prefix: impl AsRef, text: impl AsRef + Into, ) -> ImmutableString { - self.interned_strings.get_with_prefix(prefix, text) + self.interned_strings + .getters + .get_with_mapper(|s| crate::engine::make_getter(s.as_ref()).into(), text) + } + + /// Get an interned property setter, creating one if it is not yet interned. + #[cfg(not(feature = "no_object"))] + #[inline(always)] + #[allow(dead_code)] + #[must_use] + pub fn get_interned_setter( + &mut self, + text: impl AsRef + Into, + ) -> ImmutableString { + self.interned_strings + .setters + .get_with_mapper(|s| crate::engine::make_setter(s.as_ref()).into(), text) } } @@ -353,18 +360,14 @@ impl Expr { #[cfg(not(feature = "no_module"))] Self::Variable(x, ..) if !x.1.is_empty() => unreachable!("qualified property"), Self::Variable(x, .., pos) => { - let ident = x.3; - let getter = state.get_identifier_with_prefix(crate::engine::FN_GET, &ident); + let ident = x.3.clone(); + let getter = state.get_interned_getter(ident.as_str()); let hash_get = calc_fn_hash(&getter, 1); - let setter = state.get_identifier_with_prefix(crate::engine::FN_SET, &ident); + let setter = state.get_interned_setter(ident.as_str()); let hash_set = calc_fn_hash(&setter, 2); Self::Property( - Box::new(( - (getter, hash_get), - (setter, hash_set), - state.get_interned_string(&ident), - )), + Box::new(((getter, hash_get), (setter, hash_set), ident)), pos, ) } @@ -539,7 +542,7 @@ impl Engine { input: &mut TokenStream, state: &mut ParseState, lib: &mut FnLib, - id: Identifier, + id: ImmutableString, no_args: bool, capture_parent_scope: bool, #[cfg(not(feature = "no_module"))] namespace: crate::ast::Namespace, @@ -591,7 +594,7 @@ impl Engine { if settings.options.contains(LangOptions::STRICT_VAR) && index.is_none() && !is_global - && !state.global_imports.iter().any(|m| m == root) + && !state.global_imports.iter().any(|m| m.as_str() == root) && !self.global_sub_modules.contains_key(root) { return Err( @@ -615,7 +618,7 @@ impl Engine { args.shrink_to_fit(); return Ok(FnCallExpr { - name: state.get_identifier(id), + name: id, capture_parent_scope, #[cfg(not(feature = "no_module"))] namespace, @@ -659,7 +662,7 @@ impl Engine { if settings.options.contains(LangOptions::STRICT_VAR) && index.is_none() && !is_global - && !state.global_imports.iter().any(|m| m == root) + && !state.global_imports.iter().any(|m| m.as_str() == root) && !self.global_sub_modules.contains_key(root) { return Err(PERR::ModuleUndefined(root.to_string()) @@ -686,7 +689,7 @@ impl Engine { args.shrink_to_fit(); return Ok(FnCallExpr { - name: state.get_identifier(id), + name: state.get_interned_string(id), capture_parent_scope, #[cfg(not(feature = "no_module"))] namespace, @@ -1054,8 +1057,9 @@ impl Engine { } let expr = self.parse_expr(input, state, lib, settings.level_up())?; - let name = state.get_identifier(name); template.insert(name.clone(), crate::Dynamic::UNIT); + + let name = state.get_interned_string(name); map.push((Ident { name, pos }, expr)); match input.peek().expect(NEVER_ENDS) { @@ -1380,12 +1384,12 @@ impl Engine { // | ... #[cfg(not(feature = "no_function"))] Token::Pipe | Token::Or if settings.options.contains(LangOptions::ANON_FN) => { - let interned_strings = std::mem::take(&mut state.interned_strings); + let interners = std::mem::take(&mut state.interned_strings); let mut new_state = ParseState::new( self, state.scope, - interned_strings, + interners, state.tokenizer_control.clone(), ); @@ -1564,7 +1568,7 @@ impl Engine { state.allow_capture = true; } Expr::Variable( - (None, ns, 0, state.get_identifier(s)).into(), + (None, ns, 0, state.get_interned_string(s)).into(), None, settings.pos, ) @@ -1578,7 +1582,7 @@ impl Engine { state.allow_capture = true; } Expr::Variable( - (None, ns, 0, state.get_identifier(s)).into(), + (None, ns, 0, state.get_interned_string(s)).into(), None, settings.pos, ) @@ -1605,7 +1609,7 @@ impl Engine { } }); Expr::Variable( - (index, ns, 0, state.get_identifier(s)).into(), + (index, ns, 0, state.get_interned_string(s)).into(), short_index, settings.pos, ) @@ -1629,7 +1633,7 @@ impl Engine { // Function call is allowed to have reserved keyword Token::LeftParen | Token::Bang | Token::Unit if is_keyword_function(&s) => { Expr::Variable( - (None, ns, 0, state.get_identifier(s)).into(), + (None, ns, 0, state.get_interned_string(s)).into(), None, settings.pos, ) @@ -1637,7 +1641,7 @@ impl Engine { // Access to `this` as a variable is OK within a function scope #[cfg(not(feature = "no_function"))] _ if &*s == KEYWORD_THIS && settings.in_fn_scope => Expr::Variable( - (None, ns, 0, state.get_identifier(s)).into(), + (None, ns, 0, state.get_interned_string(s)).into(), None, settings.pos, ), @@ -1764,7 +1768,7 @@ impl Engine { namespace.push(var_name_def); Expr::Variable( - (None, namespace, 0, state.get_identifier(id2)).into(), + (None, namespace, 0, state.get_interned_string(id2)).into(), None, pos2, ) @@ -1842,7 +1846,7 @@ impl Engine { if settings.options.contains(LangOptions::STRICT_VAR) && index.is_none() && !is_global - && !state.global_imports.iter().any(|m| m == root) + && !state.global_imports.iter().any(|m| m.as_str() == root) && !self.global_sub_modules.contains_key(root) { return Err( @@ -1909,7 +1913,7 @@ impl Engine { args.shrink_to_fit(); Ok(FnCallExpr { - name: state.get_identifier("-"), + name: state.get_interned_string("-"), hashes: FnCallHashes::from_native(calc_fn_hash("-", 1)), args, pos, @@ -1936,7 +1940,7 @@ impl Engine { args.shrink_to_fit(); Ok(FnCallExpr { - name: state.get_identifier("+"), + name: state.get_interned_string("+"), hashes: FnCallHashes::from_native(calc_fn_hash("+", 1)), args, pos, @@ -1954,7 +1958,7 @@ impl Engine { args.shrink_to_fit(); Ok(FnCallExpr { - name: state.get_identifier("!"), + name: state.get_interned_string("!"), hashes: FnCallHashes::from_native(calc_fn_hash("!", 1)), args, pos, @@ -2329,7 +2333,7 @@ impl Engine { let hash = calc_fn_hash(&op, 2); let op_base = FnCallExpr { - name: state.get_identifier(op.as_ref()), + name: state.get_interned_string(op.as_ref()), hashes: FnCallHashes::from_native(hash), pos, ..Default::default() @@ -2401,7 +2405,7 @@ impl Engine { FnCallExpr { hashes: calc_fn_hash(OP_CONTAINS, 2).into(), args, - name: state.get_identifier(OP_CONTAINS), + name: state.get_interned_string(OP_CONTAINS), ..op_base } .into_fn_call_expr(pos) @@ -2460,14 +2464,14 @@ impl Engine { if syntax.scope_may_be_changed { // Add a barrier variable to the stack so earlier variables will not be matched. // Variable searches stop at the first barrier. - let marker = state.get_identifier(SCOPE_SEARCH_BARRIER_MARKER); + let marker = state.get_interned_string(SCOPE_SEARCH_BARRIER_MARKER); state.stack.push(marker, ()); } let parse_func = &*syntax.parse; let mut required_token: ImmutableString = key.into(); - tokens.push(required_token.clone().into()); + tokens.push(required_token.clone()); segments.push(required_token.clone()); loop { @@ -2491,7 +2495,7 @@ impl Engine { match required_token.as_str() { CUSTOM_SYNTAX_MARKER_IDENT => { let (name, pos) = parse_var_name(input)?; - let name = state.get_identifier(name); + let name = state.get_interned_string(name); #[cfg(not(feature = "no_module"))] let ns = crate::ast::Namespace::NONE; @@ -2499,19 +2503,19 @@ impl Engine { let ns = (); segments.push(name.clone().into()); - tokens.push(state.get_identifier(CUSTOM_SYNTAX_MARKER_IDENT)); + tokens.push(state.get_interned_string(CUSTOM_SYNTAX_MARKER_IDENT)); inputs.push(Expr::Variable((None, ns, 0, name).into(), None, pos)); } CUSTOM_SYNTAX_MARKER_SYMBOL => { let (symbol, pos) = parse_symbol(input)?; let symbol = state.get_interned_string(symbol); segments.push(symbol.clone()); - tokens.push(state.get_identifier(CUSTOM_SYNTAX_MARKER_SYMBOL)); + tokens.push(state.get_interned_string(CUSTOM_SYNTAX_MARKER_SYMBOL)); inputs.push(Expr::StringConstant(symbol, pos)); } CUSTOM_SYNTAX_MARKER_EXPR => { inputs.push(self.parse_expr(input, state, lib, settings)?); - let keyword = state.get_identifier(CUSTOM_SYNTAX_MARKER_EXPR); + let keyword = state.get_interned_string(CUSTOM_SYNTAX_MARKER_EXPR); segments.push(keyword.clone().into()); tokens.push(keyword); } @@ -2519,7 +2523,7 @@ impl Engine { match self.parse_block(input, state, lib, settings)? { block @ Stmt::Block(..) => { inputs.push(Expr::Stmt(Box::new(block.into()))); - let keyword = state.get_identifier(CUSTOM_SYNTAX_MARKER_BLOCK); + let keyword = state.get_interned_string(CUSTOM_SYNTAX_MARKER_BLOCK); segments.push(keyword.clone().into()); tokens.push(keyword); } @@ -2530,7 +2534,7 @@ impl Engine { (b @ (Token::True | Token::False), pos) => { inputs.push(Expr::BoolConstant(b == Token::True, pos)); segments.push(state.get_interned_string(b.literal_syntax())); - tokens.push(state.get_identifier(CUSTOM_SYNTAX_MARKER_BOOL)); + tokens.push(state.get_interned_string(CUSTOM_SYNTAX_MARKER_BOOL)); } (.., pos) => { return Err( @@ -2543,7 +2547,7 @@ impl Engine { (Token::IntegerConstant(i), pos) => { inputs.push(Expr::IntegerConstant(i, pos)); segments.push(i.to_string().into()); - tokens.push(state.get_identifier(CUSTOM_SYNTAX_MARKER_INT)); + tokens.push(state.get_interned_string(CUSTOM_SYNTAX_MARKER_INT)); } (.., pos) => { return Err( @@ -2557,7 +2561,7 @@ impl Engine { (Token::FloatConstant(f), pos) => { inputs.push(Expr::FloatConstant(f, pos)); segments.push(f.to_string().into()); - tokens.push(state.get_identifier(CUSTOM_SYNTAX_MARKER_FLOAT)); + tokens.push(state.get_interned_string(CUSTOM_SYNTAX_MARKER_FLOAT)); } (.., pos) => { return Err(PERR::MissingSymbol( @@ -2571,7 +2575,7 @@ impl Engine { let s = state.get_interned_string(s); inputs.push(Expr::StringConstant(s.clone(), pos)); segments.push(s); - tokens.push(state.get_identifier(CUSTOM_SYNTAX_MARKER_STRING)); + tokens.push(state.get_interned_string(CUSTOM_SYNTAX_MARKER_STRING)); } (.., pos) => { return Err( @@ -2835,11 +2839,11 @@ impl Engine { state.stack.push(name.clone(), ()); } let counter_var = Ident { - name: state.get_identifier(counter_name), + name: state.get_interned_string(counter_name), pos: counter_pos, }; - let loop_var = state.get_identifier(name); + let loop_var = state.get_interned_string(name); state.stack.push(loop_var.clone(), ()); let loop_var = Ident { name: loop_var, @@ -2912,7 +2916,7 @@ impl Engine { } } - let name = state.get_identifier(name); + let name = state.get_interned_string(name); // let name = ... let expr = if match_token(input, Token::Equals).0 { @@ -2978,14 +2982,18 @@ impl Engine { // import expr ... let expr = self.parse_expr(input, state, lib, settings.level_up())?; - // import expr as ... + // import expr; if !match_token(input, Token::As).0 { - return Ok(Stmt::Import((expr, Ident::EMPTY).into(), settings.pos)); + let empty = Ident { + name: state.get_interned_string(""), + pos: Position::NONE, + }; + return Ok(Stmt::Import((expr, empty).into(), settings.pos)); } // import expr as name ... let (name, pos) = parse_var_name(input)?; - let name = state.get_identifier(name); + let name = state.get_interned_string(name); state.imports.push(name.clone()); Ok(Stmt::Import( @@ -3038,11 +3046,11 @@ impl Engine { let export = ( Ident { - name: state.get_identifier(id), + name: state.get_interned_string(id), pos: id_pos, }, Ident { - name: state.get_identifier(alias.as_ref().map_or("", <_>::as_ref)), + name: state.get_interned_string(alias.as_ref().map_or("", <_>::as_ref)), pos: alias_pos, }, ); @@ -3251,12 +3259,12 @@ impl Engine { match input.next().expect(NEVER_ENDS) { (Token::Fn, pos) => { - let interned_strings = std::mem::take(&mut state.interned_strings); + let interners = std::mem::take(&mut state.interned_strings); let mut new_state = ParseState::new( self, state.scope, - interned_strings, + interners, state.tokenizer_control.clone(), ); @@ -3451,11 +3459,14 @@ impl Engine { .into_err(err_pos)); } - let name = state.get_identifier(name); + let name = state.get_interned_string(name); state.stack.push(name.clone(), ()); Ident { name, pos } } else { - Ident::EMPTY + Ident { + name: state.get_interned_string(""), + pos: Position::NONE, + } }; // try { try_block } catch ( var ) { catch_block } @@ -3515,7 +3526,7 @@ impl Engine { (.., pos) => return Err(PERR::FnMissingParams(name.to_string()).into_err(*pos)), }; - let mut params = StaticVec::new_const(); + let mut params = StaticVec::<(ImmutableString, _)>::new_const(); if !no_params { let sep_err = format!("to separate the parameters of function '{name}'"); @@ -3524,11 +3535,11 @@ impl Engine { match input.next().expect(NEVER_ENDS) { (Token::RightParen, ..) => break, (Token::Identifier(s), pos) => { - if params.iter().any(|(p, _)| p == &*s) { + if params.iter().any(|(p, _)| p.as_str() == &*s) { return Err(PERR::FnDuplicatedParam(name.to_string(), s.to_string()) .into_err(pos)); } - let s = state.get_identifier(s); + let s = state.get_interned_string(s); state.stack.push(s.clone(), ()); params.push((s, pos)); } @@ -3567,7 +3578,7 @@ impl Engine { params.shrink_to_fit(); Ok(ScriptFnDef { - name: state.get_identifier(name), + name: state.get_interned_string(name), access, params, body, @@ -3617,7 +3628,7 @@ impl Engine { ); let expr = FnCallExpr { - name: state.get_identifier(crate::engine::KEYWORD_FN_PTR_CURRY), + name: state.get_interned_string(crate::engine::KEYWORD_FN_PTR_CURRY), hashes: FnCallHashes::from_native(calc_fn_hash( crate::engine::KEYWORD_FN_PTR_CURRY, num_externals + 1, @@ -3634,7 +3645,7 @@ impl Engine { statements.extend( externals .into_iter() - .map(|crate::ast::Ident { name, pos }| Stmt::Share(name.into(), pos)), + .map(|crate::ast::Ident { name, pos }| Stmt::Share(name, pos)), ); statements.push(Stmt::Expr(expr.into())); Expr::Stmt(crate::ast::StmtBlock::new(statements, pos, Position::NONE).into()) @@ -3653,18 +3664,18 @@ impl Engine { settings.ensure_level_within_max_limit(state.max_expr_depth)?; let mut settings = settings; - let mut params_list = StaticVec::new_const(); + let mut params_list = StaticVec::::new_const(); if input.next().expect(NEVER_ENDS).0 != Token::Or && !match_token(input, Token::Pipe).0 { loop { match input.next().expect(NEVER_ENDS) { (Token::Pipe, ..) => break, (Token::Identifier(s), pos) => { - if params_list.iter().any(|p| p == &*s) { + if params_list.iter().any(|p| p.as_str() == &*s) { return Err(PERR::FnDuplicatedParam("".to_string(), s.to_string()) .into_err(pos)); } - let s = state.get_identifier(s); + let s = state.get_interned_string(s); state.stack.push(s.clone(), ()); params_list.push(s); } @@ -3722,7 +3733,7 @@ impl Engine { params.iter().for_each(|p| p.hash(hasher)); body.hash(hasher); let hash = hasher.finish(); - let fn_name = state.get_identifier(make_anonymous_fn(hash)); + let fn_name = state.get_interned_string(make_anonymous_fn(hash)); // Define the function let script = ScriptFnDef { diff --git a/src/types/interner.rs b/src/types/interner.rs index e8711e68..548f469a 100644 --- a/src/types/interner.rs +++ b/src/types/interner.rs @@ -26,12 +26,6 @@ pub struct StringsInterner<'a> { max: usize, /// Normal strings. strings: BTreeMap, - /// Property getters. - #[cfg(not(feature = "no_object"))] - getters: BTreeMap, - /// Property setters. - #[cfg(not(feature = "no_object"))] - setters: BTreeMap, /// Take care of the lifetime parameter. dummy: PhantomData<&'a ()>, } @@ -51,71 +45,38 @@ impl StringsInterner<'_> { Self::new_with_capacity(MAX_INTERNED_STRINGS) } - /// Create a new [`StringsInterner`] with a maximum capacity. + /// Create a new [`StringsInterner`] with maximum capacity. #[inline] #[must_use] pub fn new_with_capacity(capacity: usize) -> Self { Self { max: capacity, strings: BTreeMap::new(), - #[cfg(not(feature = "no_object"))] - getters: BTreeMap::new(), - #[cfg(not(feature = "no_object"))] - setters: BTreeMap::new(), dummy: PhantomData, } } - - /// Get an identifier from a text string and prefix, adding it to the interner if necessary. + /// Get an identifier from a text string, adding it to the interner if necessary. #[inline(always)] #[must_use] - pub fn get(&mut self, text: impl AsRef + Into) -> ImmutableString { - self.get_with_prefix("", text) + pub fn get + Into>(&mut self, text: T) -> ImmutableString { + self.get_with_mapper(|s| s.into(), text) } - /// Get an identifier from a text string and prefix, adding it to the interner if necessary. - /// - /// # Prefix - /// - /// Currently recognized prefixes are: - /// - /// * `""` - None (normal string) - /// * `"get$"` - Property getter, not available under `no_object` - /// * `"set$"` - Property setter, not available under `no_object` - /// - /// # Panics - /// - /// Panics if the prefix is not recognized. + /// Get an identifier from a text string, adding it to the interner if necessary. #[inline] #[must_use] - pub fn get_with_prefix + Into>( + pub fn get_with_mapper + Into>( &mut self, - prefix: impl AsRef, + mapper: fn(T) -> ImmutableString, text: T, ) -> ImmutableString { - let prefix = prefix.as_ref(); let key = text.as_ref(); // Do not intern numbers - if prefix == "" && key.bytes().all(|c| c == b'.' || (c >= b'0' && c <= b'9')) { + if key.bytes().all(|c| c == b'.' || (c >= b'0' && c <= b'9')) { return text.into(); } - let (dict, mapper): (_, fn(T) -> ImmutableString) = match prefix { - "" => (&mut self.strings, |s| s.into()), - - #[cfg(not(feature = "no_object"))] - crate::engine::FN_GET => (&mut self.getters, |s| { - crate::engine::make_getter(s.as_ref()).into() - }), - #[cfg(not(feature = "no_object"))] - crate::engine::FN_SET => (&mut self.setters, |s| { - crate::engine::make_setter(s.as_ref()).into() - }), - - _ => unreachable!("unsupported prefix {}", prefix), - }; - if key.len() > MAX_STRING_LEN { return mapper(text); } @@ -124,8 +85,8 @@ impl StringsInterner<'_> { key.hash(hasher); let key = hasher.finish(); - if !dict.is_empty() && dict.contains_key(&key) { - return dict.get(&key).unwrap().clone(); + if !self.strings.is_empty() && self.strings.contains_key(&key) { + return self.strings.get(&key).unwrap().clone(); } let value = mapper(text); @@ -134,7 +95,7 @@ impl StringsInterner<'_> { return value; } - dict.insert(key, value.clone()); + self.strings.insert(key, value.clone()); // If the interner is over capacity, remove the longest entry if self.strings.len() > self.max { @@ -160,37 +121,23 @@ impl StringsInterner<'_> { } /// Number of strings interned. - #[inline] + #[inline(always)] #[must_use] pub fn len(&self) -> usize { - #[cfg(not(feature = "no_object"))] - return self.strings.len() + self.getters.len() + self.setters.len(); - - #[cfg(feature = "no_object")] - return self.strings.len(); + self.strings.len() } /// Number of strings interned. - #[inline] + #[inline(always)] #[must_use] pub fn is_empty(&self) -> bool { - #[cfg(not(feature = "no_object"))] - return self.strings.is_empty() || self.getters.is_empty() || self.setters.is_empty(); - - #[cfg(feature = "no_object")] - return self.strings.is_empty(); + self.strings.is_empty() } /// Clear all interned strings. #[inline] pub fn clear(&mut self) { self.strings.clear(); - - #[cfg(not(feature = "no_object"))] - { - self.getters.clear(); - self.setters.clear(); - } } } @@ -198,12 +145,6 @@ impl AddAssign for StringsInterner<'_> { #[inline(always)] fn add_assign(&mut self, rhs: Self) { self.strings.extend(rhs.strings.into_iter()); - - #[cfg(not(feature = "no_object"))] - { - self.getters.extend(rhs.getters.into_iter()); - self.setters.extend(rhs.setters.into_iter()); - } } } @@ -212,13 +153,5 @@ impl AddAssign<&Self> for StringsInterner<'_> { fn add_assign(&mut self, rhs: &Self) { self.strings .extend(rhs.strings.iter().map(|(&k, v)| (k, v.clone()))); - - #[cfg(not(feature = "no_object"))] - { - self.getters - .extend(rhs.getters.iter().map(|(&k, v)| (k, v.clone()))); - self.setters - .extend(rhs.setters.iter().map(|(&k, v)| (k, v.clone()))); - } } }