kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Use Box<str> internally.

Browse Source
This commit is contained in:
Stephen Chung 2021-11-11 13:55:52 +08:00
parent 6b27ca19d5
commit 0fbc437916
9 changed files with 121 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/ast.rs
View File

@ -77,7 +77,7 @@ pub struct ScriptFnDef {
/// Not available under `no_function`. /// Not available under `no_function`.
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
#[cfg(feature = "metadata")] #[cfg(feature = "metadata")]
pub comments: StaticVec<String>, pub comments: StaticVec<Box<str>>,
} }
impl fmt::Display for ScriptFnDef { impl fmt::Display for ScriptFnDef {
@ -156,7 +156,7 @@ impl<'a> From<&'a ScriptFnDef> for ScriptFnMetadata<'a> {
Self { Self {
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
#[cfg(feature = "metadata")] #[cfg(feature = "metadata")]
comments: value.comments.iter().map(|s| s.as_str()).collect(), comments: value.comments.iter().map(Box::as_ref).collect(),
access: value.access, access: value.access,
name: &value.name, name: &value.name,
params: value.params.iter().map(|s| s.as_str()).collect(), params: value.params.iter().map(|s| s.as_str()).collect(),

10
src/engine_api.rs
View File

@ -1385,7 +1385,7 @@ impl Engine {
Some(&|token, _, _| { Some(&|token, _, _| {
match token { match token {
// If `null` is present, make sure `null` is treated as a variable // If `null` is present, make sure `null` is treated as a variable
Token::Reserved(s) if s == "null" => Token::Identifier(s), Token::Reserved(s) if &*s == "null" => Token::Identifier(s),
_ => token, _ => token,
} }
}) })
@ -2118,7 +2118,7 @@ impl Engine {
signatures.extend( signatures.extend(
self.global_modules self.global_modules
.iter() .iter()
.take(self.global_modules.len() - 1) .skip(1)
.flat_map(|m| m.gen_fn_signatures()), .flat_map(|m| m.gen_fn_signatures()),
); );
} }
@ -2214,10 +2214,10 @@ impl Engine {
/// engine.on_parse_token(|token, _, _| { /// engine.on_parse_token(|token, _, _| {
/// match token { /// match token {
/// // Convert all integer literals to strings /// // Convert all integer literals to strings
/// Token::IntegerConstant(n) => Token::StringConstant(n.to_string()), /// Token::IntegerConstant(n) => Token::StringConstant(n.to_string().into()),
/// // Convert 'begin' .. 'end' to '{' .. '}' /// // Convert 'begin' .. 'end' to '{' .. '}'
/// Token::Identifier(s) if &s == "begin" => Token::LeftBrace, /// Token::Identifier(s) if &*s == "begin" => Token::LeftBrace,
/// Token::Identifier(s) if &s == "end" => Token::RightBrace, /// Token::Identifier(s) if &*s == "end" => Token::RightBrace,
/// // Pass through all other tokens unchanged /// // Pass through all other tokens unchanged
/// _ => token /// _ => token
/// } /// }

6
src/engine_settings.rs
View File

@ -308,18 +308,18 @@ impl Engine {
// Active standard keywords cannot be made custom // Active standard keywords cannot be made custom
// Disabled keywords are OK // Disabled keywords are OK
Some(token) if token.is_standard_keyword() => { Some(token) if token.is_standard_keyword() => {
if !self.disabled_symbols.contains(token.syntax().as_ref()) { if !self.disabled_symbols.contains(&*token.syntax()) {
return Err(format!("'{}' is a reserved keyword", keyword.as_ref())); return Err(format!("'{}' is a reserved keyword", keyword.as_ref()));
} }
} }
// Active standard symbols cannot be made custom // Active standard symbols cannot be made custom
Some(token) if token.is_standard_symbol() => { Some(token) if token.is_standard_symbol() => {
if !self.disabled_symbols.contains(token.syntax().as_ref()) { if !self.disabled_symbols.contains(&*token.syntax()) {
return Err(format!("'{}' is a reserved operator", keyword.as_ref())); return Err(format!("'{}' is a reserved operator", keyword.as_ref()));
} }
} }
// Active standard symbols cannot be made custom // Active standard symbols cannot be made custom
Some(token) if !self.disabled_symbols.contains(token.syntax().as_ref()) => { Some(token) if !self.disabled_symbols.contains(&*token.syntax()) => {
return Err(format!("'{}' is a reserved symbol", keyword.as_ref())) return Err(format!("'{}' is a reserved symbol", keyword.as_ref()))
} }
// Disabled symbols are OK // Disabled symbols are OK

10
src/fn_ptr.rs
View File

@ -174,6 +174,16 @@ impl TryFrom<String> for FnPtr {
} }
} }
impl TryFrom<Box<str>> for FnPtr {
type Error = Box<EvalAltResult>;
#[inline(always)]
fn try_from(value: Box<str>) -> Result<Self, Self::Error> {
let s: Identifier = value.into();
Self::try_from(s)
}
}
impl TryFrom<&str> for FnPtr { impl TryFrom<&str> for FnPtr {
type Error = Box<EvalAltResult>; type Error = Box<EvalAltResult>;

7
src/immutable_string.rs
View File

@ -94,6 +94,13 @@ impl From<&str> for ImmutableString {
Self(value.into()) Self(value.into())
} }
} }
impl From<Box<str>> for ImmutableString {
#[inline(always)]
fn from(value: Box<str>) -> Self {
let value: SmartString = value.into();
Self(value.into())
}
}
impl From<&String> for ImmutableString { impl From<&String> for ImmutableString {
#[inline(always)] #[inline(always)]
fn from(value: &String) -> Self { fn from(value: &String) -> Self {

4
src/module/mod.rs
View File

@ -66,11 +66,11 @@ impl FuncInfo {
let mut sig = format!("{}(", self.name); let mut sig = format!("{}(", self.name);
if !self.param_names.is_empty() { if !self.param_names.is_empty() {
let mut params: StaticVec<String> = let mut params: StaticVec<Box<str>> =
self.param_names.iter().map(|s| s.as_str().into()).collect(); self.param_names.iter().map(|s| s.as_str().into()).collect();
let return_type = params.pop().unwrap_or_else(|| "()".into()); let return_type = params.pop().unwrap_or_else(|| "()".into());
sig.push_str(&params.join(", ")); sig.push_str(&params.join(", "));
if return_type != "()" { if &*return_type != "()" {
sig.push_str(") -> "); sig.push_str(") -> ");
sig.push_str(&return_type); sig.push_str(&return_type);
} else { } else {

78
src/parse.rs
View File

@ -382,13 +382,13 @@ fn match_token(input: &mut TokenStream, token: Token) -> (bool, Position) {
} }
/// Parse a variable name. /// Parse a variable name.
fn parse_var_name(input: &mut TokenStream) -> Result<(String, Position), ParseError> { fn parse_var_name(input: &mut TokenStream) -> Result<(Box<str>, Position), ParseError> {
match input.next().expect(NEVER_ENDS) { match input.next().expect(NEVER_ENDS) {
// Variable name // Variable name
(Token::Identifier(s), pos) => Ok((s, pos)), (Token::Identifier(s), pos) => Ok((s, pos)),
// Reserved keyword // Reserved keyword
(Token::Reserved(s), pos) if is_valid_identifier(s.chars()) => { (Token::Reserved(s), pos) if is_valid_identifier(s.chars()) => {
Err(PERR::Reserved(s).into_err(pos)) Err(PERR::Reserved(s.to_string()).into_err(pos))
} }
// Bad identifier // Bad identifier
(Token::LexError(err), pos) => Err(err.into_err(pos)), (Token::LexError(err), pos) => Err(err.into_err(pos)),
@ -398,7 +398,7 @@ fn parse_var_name(input: &mut TokenStream) -> Result<(String, Position), ParseEr
} }
/// Parse a symbol. /// Parse a symbol.
fn parse_symbol(input: &mut TokenStream) -> Result<(String, Position), ParseError> { fn parse_symbol(input: &mut TokenStream) -> Result<(Box<str>, Position), ParseError> {
match input.next().expect(NEVER_ENDS) { match input.next().expect(NEVER_ENDS) {
// Symbol // Symbol
(token, pos) if token.is_standard_symbol() => Ok((token.literal_syntax().into(), pos)), (token, pos) if token.is_standard_symbol() => Ok((token.literal_syntax().into(), pos)),
@ -860,14 +860,14 @@ fn parse_map_literal(
let (name, pos) = match input.next().expect(NEVER_ENDS) { let (name, pos) = match input.next().expect(NEVER_ENDS) {
(Token::Identifier(s), pos) | (Token::StringConstant(s), pos) => { (Token::Identifier(s), pos) | (Token::StringConstant(s), pos) => {
if map.iter().any(|(p, _)| p.name == s) { if map.iter().any(|(p, _)| p.name == &*s) {
return Err(PERR::DuplicatedProperty(s).into_err(pos)); return Err(PERR::DuplicatedProperty(s.to_string()).into_err(pos));
} }
(s, pos) (s, pos)
} }
(Token::InterpolatedString(_), pos) => return Err(PERR::PropertyExpected.into_err(pos)), (Token::InterpolatedString(_), pos) => return Err(PERR::PropertyExpected.into_err(pos)),
(Token::Reserved(s), pos) if is_valid_identifier(s.chars()) => { (Token::Reserved(s), pos) if is_valid_identifier(s.chars()) => {
return Err(PERR::Reserved(s).into_err(pos)); return Err(PERR::Reserved(s.to_string()).into_err(pos));
} }
(Token::LexError(err), pos) => return Err(err.into_err(pos)), (Token::LexError(err), pos) => return Err(err.into_err(pos)),
(Token::EOF, pos) => { (Token::EOF, pos) => {
@ -1313,20 +1313,20 @@ fn parse_primary(
(None, None, state.get_identifier(s)).into(), (None, None, state.get_identifier(s)).into(),
), ),
// Access to `this` as a variable is OK within a function scope // Access to `this` as a variable is OK within a function scope
_ if s == KEYWORD_THIS && settings.is_function_scope => Expr::Variable( _ if &*s == KEYWORD_THIS && settings.is_function_scope => Expr::Variable(
None, None,
settings.pos, settings.pos,
(None, None, state.get_identifier(s)).into(), (None, None, state.get_identifier(s)).into(),
), ),
// Cannot access to `this` as a variable not in a function scope // Cannot access to `this` as a variable not in a function scope
_ if s == KEYWORD_THIS => { _ if &*s == KEYWORD_THIS => {
let msg = format!("'{}' can only be used in functions", s); let msg = format!("'{}' can only be used in functions", s);
return Err(LexError::ImproperSymbol(s, msg).into_err(settings.pos)); return Err(LexError::ImproperSymbol(s.to_string(), msg).into_err(settings.pos));
} }
_ if is_valid_identifier(s.chars()) => { _ if is_valid_identifier(s.chars()) => {
return Err(PERR::Reserved(s).into_err(settings.pos)) return Err(PERR::Reserved(s.to_string()).into_err(settings.pos))
} }
_ => return Err(LexError::UnexpectedInput(s).into_err(settings.pos)), _ => return Err(LexError::UnexpectedInput(s.to_string()).into_err(settings.pos)),
} }
} }
@ -1840,11 +1840,11 @@ fn parse_binary_op(
Token::Custom(c) => state Token::Custom(c) => state
.engine .engine
.custom_keywords .custom_keywords
.get(c.as_str()) .get(c.as_ref())
.cloned() .cloned()
.ok_or_else(|| PERR::Reserved(c.clone()).into_err(*current_pos))?, .ok_or_else(|| PERR::Reserved(c.to_string()).into_err(*current_pos))?,
Token::Reserved(c) if !is_valid_identifier(c.chars()) => { Token::Reserved(c) if !is_valid_identifier(c.chars()) => {
return Err(PERR::UnknownOperator(c.into()).into_err(*current_pos)) return Err(PERR::UnknownOperator(c.to_string()).into_err(*current_pos))
} }
_ => current_op.precedence(), _ => current_op.precedence(),
}; };
@ -1865,11 +1865,11 @@ fn parse_binary_op(
Token::Custom(c) => state Token::Custom(c) => state
.engine .engine
.custom_keywords .custom_keywords
.get(c.as_str()) .get(c.as_ref())
.cloned() .cloned()
.ok_or_else(|| PERR::Reserved(c.clone()).into_err(*next_pos))?, .ok_or_else(|| PERR::Reserved(c.to_string()).into_err(*next_pos))?,
Token::Reserved(c) if !is_valid_identifier(c.chars()) => { Token::Reserved(c) if !is_valid_identifier(c.chars()) => {
return Err(PERR::UnknownOperator(c.into()).into_err(*next_pos)) return Err(PERR::UnknownOperator(c.to_string()).into_err(*next_pos))
} }
_ => next_op.precedence(), _ => next_op.precedence(),
}; };
@ -1971,7 +1971,7 @@ fn parse_binary_op(
if state if state
.engine .engine
.custom_keywords .custom_keywords
.get(s.as_str()) .get(s.as_ref())
.map_or(false, Option::is_some) => .map_or(false, Option::is_some) =>
{ {
let hash = calc_fn_hash(&s, 2); let hash = calc_fn_hash(&s, 2);
@ -2027,7 +2027,7 @@ fn parse_custom_syntax(
settings.pos = *fwd_pos; settings.pos = *fwd_pos;
let settings = settings.level_up(); let settings = settings.level_up();
required_token = match parse_func(&segments, fwd_token.syntax().as_ref()) { required_token = match parse_func(&segments, &*fwd_token.syntax()) {
Ok(Some(seg)) Ok(Some(seg))
if seg.starts_with(CUSTOM_SYNTAX_MARKER_SYNTAX_VARIANT) if seg.starts_with(CUSTOM_SYNTAX_MARKER_SYNTAX_VARIANT)
&& seg.len() > CUSTOM_SYNTAX_MARKER_SYNTAX_VARIANT.len() => && seg.len() > CUSTOM_SYNTAX_MARKER_SYNTAX_VARIANT.len() =>
@ -2123,7 +2123,7 @@ fn parse_custom_syntax(
}, },
s => match input.next().expect(NEVER_ENDS) { s => match input.next().expect(NEVER_ENDS) {
(Token::LexError(err), pos) => return Err(err.into_err(pos)), (Token::LexError(err), pos) => return Err(err.into_err(pos)),
(t, _) if t.syntax().as_ref() == s => { (t, _) if &*t.syntax() == s => {
segments.push(required_token.clone()); segments.push(required_token.clone());
tokens.push(required_token.clone().into()); tokens.push(required_token.clone().into());
} }
@ -2185,7 +2185,7 @@ fn parse_expr(
match token { match token {
Token::Custom(key) | Token::Reserved(key) | Token::Identifier(key) => { Token::Custom(key) | Token::Reserved(key) | Token::Identifier(key) => {
if let Some((key, syntax)) = state.engine.custom_syntax.get_key_value(key.as_str()) if let Some((key, syntax)) = state.engine.custom_syntax.get_key_value(key.as_ref())
{ {
input.next().expect(NEVER_ENDS); input.next().expect(NEVER_ENDS);
return parse_custom_syntax( return parse_custom_syntax(
@ -2355,7 +2355,7 @@ fn parse_for(
let (counter_name, counter_pos) = parse_var_name(input)?; let (counter_name, counter_pos) = parse_var_name(input)?;
if counter_name == name { if counter_name == name {
return Err(PERR::DuplicatedVariable(counter_name).into_err(counter_pos)); return Err(PERR::DuplicatedVariable(counter_name.to_string()).into_err(counter_pos));
} }
let (has_close_paren, pos) = match_token(input, Token::RightParen); let (has_close_paren, pos) = match_token(input, Token::RightParen);
@ -2560,12 +2560,12 @@ fn parse_export(
let (rename, rename_pos) = if match_token(input, Token::As).0 { let (rename, rename_pos) = if match_token(input, Token::As).0 {
let (name, pos) = parse_var_name(input)?; let (name, pos) = parse_var_name(input)?;
if exports.iter().any(|(_, alias)| alias.name == name) { if exports.iter().any(|(_, alias)| alias.name == name.as_ref()) {
return Err(PERR::DuplicatedVariable(name).into_err(pos)); return Err(PERR::DuplicatedVariable(name.to_string()).into_err(pos));
} }
(name, pos) (Some(name), pos)
} else { } else {
(String::new(), Position::NONE) (None, Position::NONE)
}; };
exports.push(( exports.push((
@ -2574,7 +2574,7 @@ fn parse_export(
pos: id_pos, pos: id_pos,
}, },
Ident { Ident {
name: state.get_identifier(rename), name: state.get_identifier(rename.as_ref().map_or("", |s| s.as_ref())),
pos: rename_pos, pos: rename_pos,
}, },
)); ));
@ -2733,7 +2733,7 @@ fn parse_stmt(
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
#[cfg(feature = "metadata")] #[cfg(feature = "metadata")]
let comments = { let comments = {
let mut comments = StaticVec::<String>::new(); let mut comments = StaticVec::<Box<str>>::new();
let mut comments_pos = Position::NONE; let mut comments_pos = Position::NONE;
// Handle doc-comments. // Handle doc-comments.
@ -2996,7 +2996,7 @@ fn parse_fn(
settings: ParseSettings, settings: ParseSettings,
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
#[cfg(feature = "metadata")] #[cfg(feature = "metadata")]
comments: StaticVec<String>, comments: StaticVec<Box<str>>,
) -> Result<ScriptFnDef, ParseError> { ) -> Result<ScriptFnDef, ParseError> {
let mut settings = settings; let mut settings = settings;
@ -3007,13 +3007,13 @@ fn parse_fn(
let name = match token.into_function_name_for_override() { let name = match token.into_function_name_for_override() {
Ok(r) => r, Ok(r) => r,
Err(Token::Reserved(s)) => return Err(PERR::Reserved(s).into_err(pos)), Err(Token::Reserved(s)) => return Err(PERR::Reserved(s.to_string()).into_err(pos)),
Err(_) => return Err(PERR::FnMissingName.into_err(pos)), Err(_) => return Err(PERR::FnMissingName.into_err(pos)),
}; };
match input.peek().expect(NEVER_ENDS) { match input.peek().expect(NEVER_ENDS) {
(Token::LeftParen, _) => eat_token(input, Token::LeftParen), (Token::LeftParen, _) => eat_token(input, Token::LeftParen),
(_, pos) => return Err(PERR::FnMissingParams(name).into_err(*pos)), (_, pos) => return Err(PERR::FnMissingParams(name.to_string()).into_err(*pos)),
}; };
let mut params = StaticVec::new(); let mut params = StaticVec::new();
@ -3025,8 +3025,10 @@ fn parse_fn(
match input.next().expect(NEVER_ENDS) { match input.next().expect(NEVER_ENDS) {
(Token::RightParen, _) => break, (Token::RightParen, _) => break,
(Token::Identifier(s), pos) => { (Token::Identifier(s), pos) => {
if params.iter().any(|(p, _)| p == &s) { if params.iter().any(|(p, _)| p == &*s) {
return Err(PERR::FnDuplicatedParam(name, s).into_err(pos)); return Err(
PERR::FnDuplicatedParam(name.to_string(), s.to_string()).into_err(pos)
);
} }
let s = state.get_identifier(s); let s = state.get_identifier(s);
state.stack.push((s.clone(), AccessMode::ReadWrite)); state.stack.push((s.clone(), AccessMode::ReadWrite));
@ -3059,7 +3061,7 @@ fn parse_fn(
settings.is_breakable = false; settings.is_breakable = false;
parse_block(input, state, lib, settings.level_up())? parse_block(input, state, lib, settings.level_up())?
} }
(_, pos) => return Err(PERR::FnMissingBody(name).into_err(*pos)), (_, pos) => return Err(PERR::FnMissingBody(name.to_string()).into_err(*pos)),
} }
.into(); .into();
@ -3076,7 +3078,7 @@ fn parse_fn(
.collect(); .collect();
Ok(ScriptFnDef { Ok(ScriptFnDef {
name: state.get_identifier(&name), name: state.get_identifier(name),
access, access,
params, params,
#[cfg(not(feature = "no_closure"))] #[cfg(not(feature = "no_closure"))]
@ -3156,8 +3158,10 @@ fn parse_anon_fn(
match input.next().expect(NEVER_ENDS) { match input.next().expect(NEVER_ENDS) {
(Token::Pipe, _) => break, (Token::Pipe, _) => break,
(Token::Identifier(s), pos) => { (Token::Identifier(s), pos) => {
if params_list.iter().any(|p| p == &s) { if params_list.iter().any(|p| p == &*s) {
return Err(PERR::FnDuplicatedParam("".to_string(), s).into_err(pos)); return Err(
PERR::FnDuplicatedParam("".to_string(), s.to_string()).into_err(pos)
);
} }
let s = state.get_identifier(s); let s = state.get_identifier(s);
state.stack.push((s.clone(), AccessMode::ReadWrite)); state.stack.push((s.clone(), AccessMode::ReadWrite));

28
src/serde/metadata.rs
View File

@ -46,9 +46,9 @@ impl From<crate::FnAccess> for FnAccess {
#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)] #[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
struct FnParam { struct FnParam {
pub name: String, pub name: Box<str>,
#[serde(rename = "type", skip_serializing_if = "Option::is_none")] #[serde(rename = "type", skip_serializing_if = "Option::is_none")]
pub typ: Option<String>, pub typ: Option<Box<str>>,
} }
impl PartialOrd for FnParam { impl PartialOrd for FnParam {
@ -98,10 +98,10 @@ struct FnMetadata {
#[serde(default, skip_serializing_if = "Vec::is_empty")] #[serde(default, skip_serializing_if = "Vec::is_empty")]
pub params: Vec<FnParam>, pub params: Vec<FnParam>,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub return_type: Option<String>, pub return_type: Option<Box<str>>,
pub signature: String, pub signature: String,
#[serde(default, skip_serializing_if = "Vec::is_empty")] #[serde(default, skip_serializing_if = "Vec::is_empty")]
pub doc_comments: Vec<String>, pub doc_comments: Vec<Box<str>>,
} }
impl PartialOrd for FnMetadata { impl PartialOrd for FnMetadata {
@ -142,17 +142,17 @@ impl From<&crate::module::FuncInfo> for FnMetadata {
let mut seg = s.splitn(2, ':'); let mut seg = s.splitn(2, ':');
let name = seg let name = seg
.next() .next()
.map(|s| s.trim().to_string()) .map(|s| s.trim().into())
.unwrap_or_else(|| "_".to_string()); .unwrap_or_else(|| "_".into());
let typ = seg.next().map(|s| s.trim().to_string()); let typ = seg.next().map(|s| s.trim().into());
FnParam { name, typ } FnParam { name, typ }
}) })
.collect(), .collect(),
return_type: info return_type: info
.param_names .param_names
.last() .last()
.map(|s| s.to_string()) .map(|s| s.as_str().into())
.or_else(|| Some("()".to_string())), .or_else(|| Some("()".into())),
signature: info.gen_signature(), signature: info.gen_signature(),
doc_comments: if info.func.is_script() { doc_comments: if info.func.is_script() {
#[cfg(feature = "no_function")] #[cfg(feature = "no_function")]
@ -186,14 +186,14 @@ impl From<crate::ast::ScriptFnMetadata<'_>> for FnMetadata {
params: info params: info
.params .params
.iter() .iter()
.map(|s| FnParam { .map(|&s| FnParam {
name: s.to_string(), name: s.into(),
typ: Some("Dynamic".to_string()), typ: Some("Dynamic".into()),
}) })
.collect(), .collect(),
return_type: Some("Dynamic".to_string()), return_type: Some("Dynamic".into()),
signature: info.to_string(), signature: info.to_string(),
doc_comments: info.comments.iter().map(|s| s.to_string()).collect(), doc_comments: info.comments.iter().map(|&s| s.into()).collect(),
} }
} }
} }

71
src/token.rs
View File

@ -320,13 +320,13 @@ pub enum Token {
#[cfg(feature = "decimal")] #[cfg(feature = "decimal")]
DecimalConstant(Decimal), DecimalConstant(Decimal),
/// An identifier. /// An identifier.
Identifier(String), Identifier(Box<str>),
/// A character constant. /// A character constant.
CharConstant(char), CharConstant(char),
/// A string constant. /// A string constant.
StringConstant(String), StringConstant(Box<str>),
/// An interpolated string. /// An interpolated string.
InterpolatedString(String), InterpolatedString(Box<str>),
/// `{` /// `{`
LeftBrace, LeftBrace,
/// `}` /// `}`
@ -489,11 +489,11 @@ pub enum Token {
/// A lexer error. /// A lexer error.
LexError(LexError), LexError(LexError),
/// A comment block. /// A comment block.
Comment(String), Comment(Box<str>),
/// A reserved symbol. /// A reserved symbol.
Reserved(String), Reserved(Box<str>),
/// A custom keyword. /// A custom keyword.
Custom(String), Custom(Box<str>),
/// End of the input stream. /// End of the input stream.
EOF, EOF,
} }
@ -603,11 +603,11 @@ impl Token {
StringConstant(_) => "string".into(), StringConstant(_) => "string".into(),
InterpolatedString(_) => "string".into(), InterpolatedString(_) => "string".into(),
CharConstant(c) => c.to_string().into(), CharConstant(c) => c.to_string().into(),
Identifier(s) => s.clone().into(), Identifier(s) => s.to_string().into(),
Reserved(s) => s.clone().into(), Reserved(s) => s.to_string().into(),
Custom(s) => s.clone().into(), Custom(s) => s.to_string().into(),
LexError(err) => err.to_string().into(), LexError(err) => err.to_string().into(),
Comment(s) => s.clone().into(), Comment(s) => s.to_string().into(),
EOF => "{EOF}".into(), EOF => "{EOF}".into(),
@ -975,9 +975,9 @@ impl Token {
/// Convert a token into a function name, if possible. /// Convert a token into a function name, if possible.
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
#[inline] #[inline]
pub(crate) fn into_function_name_for_override(self) -> Result<String, Self> { pub(crate) fn into_function_name_for_override(self) -> Result<Box<str>, Self> {
match self { match self {
Self::Custom(s) | Self::Identifier(s) if is_valid_function_name(&s) => Ok(s), Self::Custom(s) | Self::Identifier(s) if is_valid_function_name(&*s) => Ok(s),
_ => Err(self), _ => Err(self),
} }
} }
@ -1077,7 +1077,7 @@ pub fn parse_string_literal(
continuation: bool, continuation: bool,
verbatim: bool, verbatim: bool,
allow_interpolation: bool, allow_interpolation: bool,
) -> Result<(String, bool), (LexError, Position)> { ) -> Result<(Box<str>, bool), (LexError, Position)> {
let mut result = String::with_capacity(12); let mut result = String::with_capacity(12);
let mut escape = String::with_capacity(12); let mut escape = String::with_capacity(12);
@ -1268,7 +1268,7 @@ pub fn parse_string_literal(
} }
} }
Ok((result, interpolated)) Ok((result.into(), interpolated))
} }
/// Consume the next character. /// Consume the next character.
@ -1399,7 +1399,7 @@ fn get_next_token_inner(
if return_comment { if return_comment {
return Some(( return Some((
Token::Comment(comment.expect("`return_comment` is true")), Token::Comment(comment.expect("`return_comment` is true").into()),
start_pos, start_pos,
)); ));
} }
@ -1649,7 +1649,10 @@ fn get_next_token_inner(
let first = chars.next().expect("`chars` is not empty"); let first = chars.next().expect("`chars` is not empty");
if chars.next().is_some() { if chars.next().is_some() {
(Token::LexError(LERR::MalformedChar(result)), start_pos) (
Token::LexError(LERR::MalformedChar(result.to_string())),
start_pos,
)
} else { } else {
(Token::CharConstant(first), start_pos) (Token::CharConstant(first), start_pos)
} }
@ -1773,7 +1776,7 @@ fn get_next_token_inner(
} }
if let Some(comment) = comment { if let Some(comment) = comment {
return Some((Token::Comment(comment), start_pos)); return Some((Token::Comment(comment.into()), start_pos));
} }
} }
('/', '*') => { ('/', '*') => {
@ -1800,7 +1803,7 @@ fn get_next_token_inner(
scan_block_comment(stream, state.comment_level, pos, comment.as_mut()); scan_block_comment(stream, state.comment_level, pos, comment.as_mut());
if let Some(comment) = comment { if let Some(comment) = comment {
return Some((Token::Comment(comment), start_pos)); return Some((Token::Comment(comment.into()), start_pos));
} }
} }
@ -2001,7 +2004,7 @@ fn get_identifier(
)); ));
} }
Some((Token::Identifier(identifier), start_pos)) Some((Token::Identifier(identifier.into()), start_pos))
} }
/// Is this keyword allowed as a function? /// Is this keyword allowed as a function?
@ -2185,29 +2188,29 @@ impl<'a> Iterator for TokenIterator<'a> {
} }
// Reserved keyword/symbol // Reserved keyword/symbol
Some((Token::Reserved(s), pos)) => (match Some((Token::Reserved(s), pos)) => (match
(s.as_str(), self.engine.custom_keywords.contains_key(s.as_str())) (&*s, self.engine.custom_keywords.contains_key(&*s))
{ {
("===", false) => Token::LexError(LERR::ImproperSymbol(s, ("===", false) => Token::LexError(LERR::ImproperSymbol(s.to_string(),
"'===' is not a valid operator. This is not JavaScript! Should it be '=='?".to_string(), "'===' is not a valid operator. This is not JavaScript! Should it be '=='?".to_string(),
)), )),
("!==", false) => Token::LexError(LERR::ImproperSymbol(s, ("!==", false) => Token::LexError(LERR::ImproperSymbol(s.to_string(),
"'!==' is not a valid operator. This is not JavaScript! Should it be '!='?".to_string(), "'!==' is not a valid operator. This is not JavaScript! Should it be '!='?".to_string(),
)), )),
("->", false) => Token::LexError(LERR::ImproperSymbol(s, ("->", false) => Token::LexError(LERR::ImproperSymbol(s.to_string(),
"'->' is not a valid symbol. This is not C or C++!".to_string())), "'->' is not a valid symbol. This is not C or C++!".to_string())),
("<-", false) => Token::LexError(LERR::ImproperSymbol(s, ("<-", false) => Token::LexError(LERR::ImproperSymbol(s.to_string(),
"'<-' is not a valid symbol. This is not Go! Should it be '<='?".to_string(), "'<-' is not a valid symbol. This is not Go! Should it be '<='?".to_string(),
)), )),
(":=", false) => Token::LexError(LERR::ImproperSymbol(s, (":=", false) => Token::LexError(LERR::ImproperSymbol(s.to_string(),
"':=' is not a valid assignment operator. This is not Go or Pascal! Should it be simply '='?".to_string(), "':=' is not a valid assignment operator. This is not Go or Pascal! Should it be simply '='?".to_string(),
)), )),
("::<", false) => Token::LexError(LERR::ImproperSymbol(s, ("::<", false) => Token::LexError(LERR::ImproperSymbol(s.to_string(),
"'::<>' is not a valid symbol. This is not Rust! Should it be '::'?".to_string(), "'::<>' is not a valid symbol. This is not Rust! Should it be '::'?".to_string(),
)), )),
("(*", false) | ("*)", false) => Token::LexError(LERR::ImproperSymbol(s, ("(*", false) | ("*)", false) => Token::LexError(LERR::ImproperSymbol(s.to_string(),
"'(* .. *)' is not a valid comment format. This is not Pascal! Should it be '/* .. */'?".to_string(), "'(* .. *)' is not a valid comment format. This is not Pascal! Should it be '/* .. */'?".to_string(),
)), )),
("#", false) => Token::LexError(LERR::ImproperSymbol(s, ("#", false) => Token::LexError(LERR::ImproperSymbol(s.to_string(),
"'#' is not a valid symbol. Should it be '#{'?".to_string(), "'#' is not a valid symbol. Should it be '#{'?".to_string(),
)), )),
// Reserved keyword/operator that is custom. // Reserved keyword/operator that is custom.
@ -2215,23 +2218,23 @@ impl<'a> Iterator for TokenIterator<'a> {
// Reserved operator that is not custom. // Reserved operator that is not custom.
(token, false) if !is_valid_identifier(token.chars()) => { (token, false) if !is_valid_identifier(token.chars()) => {
let msg = format!("'{}' is a reserved symbol", token); let msg = format!("'{}' is a reserved symbol", token);
Token::LexError(LERR::ImproperSymbol(s, msg)) Token::LexError(LERR::ImproperSymbol(s.to_string(), msg))
}, },
// Reserved keyword that is not custom and disabled. // Reserved keyword that is not custom and disabled.
(token, false) if self.engine.disabled_symbols.contains(token) => { (token, false) if self.engine.disabled_symbols.contains(token) => {
let msg = format!("reserved symbol '{}' is disabled", token); let msg = format!("reserved symbol '{}' is disabled", token);
Token::LexError(LERR::ImproperSymbol(s, msg)) Token::LexError(LERR::ImproperSymbol(s.to_string(), msg))
}, },
// Reserved keyword/operator that is not custom. // Reserved keyword/operator that is not custom.
(_, false) => Token::Reserved(s), (_, false) => Token::Reserved(s),
}, pos), }, pos),
// Custom keyword // Custom keyword
Some((Token::Identifier(s), pos)) if self.engine.custom_keywords.contains_key(s.as_str()) => { Some((Token::Identifier(s), pos)) if self.engine.custom_keywords.contains_key(&*s) => {
(Token::Custom(s), pos) (Token::Custom(s), pos)
} }
// Custom standard keyword/symbol - must be disabled // Custom standard keyword/symbol - must be disabled
Some((token, pos)) if self.engine.custom_keywords.contains_key(token.syntax().as_ref()) => { Some((token, pos)) if self.engine.custom_keywords.contains_key(&*token.syntax()) => {
if self.engine.disabled_symbols.contains(token.syntax().as_ref()) { if self.engine.disabled_symbols.contains(&*token.syntax()) {
// Disabled standard keyword/symbol // Disabled standard keyword/symbol
(Token::Custom(token.syntax().into()), pos) (Token::Custom(token.syntax().into()), pos)
} else { } else {
@ -2240,7 +2243,7 @@ impl<'a> Iterator for TokenIterator<'a> {
} }
} }
// Disabled symbol // Disabled symbol
Some((token, pos)) if self.engine.disabled_symbols.contains(token.syntax().as_ref()) => { Some((token, pos)) if self.engine.disabled_symbols.contains(&*token.syntax()) => {
(Token::Reserved(token.syntax().into()), pos) (Token::Reserved(token.syntax().into()), pos)
} }
// Normal symbol // Normal symbol