kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Use SmartString.

Browse Source
This commit is contained in:
Stephen Chung 2022-09-14 12:11:18 +08:00
parent 396ec7df8a
commit b9478e601c
1 changed files with 20 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
src/tokenizer.rs
View File

@ -1545,14 +1545,17 @@ fn get_next_token_inner(
// digit ... // digit ...
('0'..='9', ..) => { ('0'..='9', ..) => {
let mut result = smallvec::SmallVec::<[char; 16]>::new(); let mut result = SmartString::new_const();
let mut radix_base: Option<u32> = None; let mut radix_base: Option<u32> = None;
let mut valid: fn(char) -> bool = is_numeric_digit; let mut valid: fn(char) -> bool = is_numeric_digit;
result.push(c); result.push(c);
while let Some(next_char) = stream.peek_next() { while let Some(next_char) = stream.peek_next() {
match next_char { match next_char {
ch if valid(ch) || ch == NUMBER_SEPARATOR => { NUMBER_SEPARATOR => {
eat_next(stream, pos);
}
ch if valid(ch) => {
result.push(next_char); result.push(next_char);
eat_next(stream, pos); eat_next(stream, pos);
} }
@ -1649,49 +1652,42 @@ fn get_next_token_inner(
// Parse number // Parse number
return Some(( return Some((
if let Some(radix) = radix_base { if let Some(radix) = radix_base {
let out: String = result let result = &result[2..];
.iter()
.skip(2)
.filter(|&&c| c != NUMBER_SEPARATOR)
.collect();
UNSIGNED_INT::from_str_radix(&out, radix) UNSIGNED_INT::from_str_radix(&result, radix)
.map(|v| v as INT) .map(|v| v as INT)
.map_or_else( .map_or_else(
|_| { |_| {
Token::LexError( Token::LexError(
LERR::MalformedNumber(result.into_iter().collect()).into(), LERR::MalformedNumber(result.to_string()).into(),
) )
}, },
Token::IntegerConstant, Token::IntegerConstant,
) )
} else { } else {
let out: String = let num = INT::from_str(&result).map(Token::IntegerConstant);
result.iter().filter(|&&c| c != NUMBER_SEPARATOR).collect();
let num = INT::from_str(&out).map(Token::IntegerConstant);
// If integer parsing is unnecessary, try float instead // If integer parsing is unnecessary, try float instead
#[cfg(not(feature = "no_float"))] #[cfg(not(feature = "no_float"))]
let num = num.or_else(|_| { let num = num.or_else(|_| {
crate::ast::FloatWrapper::from_str(&out).map(Token::FloatConstant) crate::ast::FloatWrapper::from_str(&result).map(Token::FloatConstant)
}); });
// Then try decimal // Then try decimal
#[cfg(feature = "decimal")] #[cfg(feature = "decimal")]
let num = num.or_else(|_| { let num = num.or_else(|_| {
rust_decimal::Decimal::from_str(&out).map(Token::DecimalConstant) rust_decimal::Decimal::from_str(&result).map(Token::DecimalConstant)
}); });
// Then try decimal in scientific notation // Then try decimal in scientific notation
#[cfg(feature = "decimal")] #[cfg(feature = "decimal")]
let num = num.or_else(|_| { let num = num.or_else(|_| {
rust_decimal::Decimal::from_scientific(&out).map(Token::DecimalConstant) rust_decimal::Decimal::from_scientific(&result)
.map(Token::DecimalConstant)
}); });
num.unwrap_or_else(|_| { num.unwrap_or_else(|_| {
Token::LexError( Token::LexError(LERR::MalformedNumber(result.to_string()).into())
LERR::MalformedNumber(result.into_iter().collect()).into(),
)
}) })
}, },
num_pos, num_pos,
@ -2179,22 +2175,20 @@ fn get_identifier(
start_pos: Position, start_pos: Position,
first_char: char, first_char: char,
) -> (Token, Position) { ) -> (Token, Position) {
let mut result = smallvec::SmallVec::<[char; 8]>::new(); let mut identifier = SmartString::new_const();
result.push(first_char); identifier.push(first_char);
while let Some(next_char) = stream.peek_next() { while let Some(next_char) = stream.peek_next() {
match next_char { match next_char {
x if is_id_continue(x) => { x if is_id_continue(x) => {
result.push(x); identifier.push(x);
eat_next(stream, pos); eat_next(stream, pos);
} }
_ => break, _ => break,
} }
} }
let is_valid_identifier = is_valid_identifier(result.iter().copied()); let is_valid_identifier = is_valid_identifier(identifier.chars());
let identifier: String = result.into_iter().collect();
if let Some(token) = Token::lookup_from_syntax(&identifier) { if let Some(token) = Token::lookup_from_syntax(&identifier) {
return (token, start_pos); return (token, start_pos);
@ -2202,12 +2196,12 @@ fn get_identifier(
if !is_valid_identifier { if !is_valid_identifier {
return ( return (
Token::LexError(LERR::MalformedIdentifier(identifier).into()), Token::LexError(LERR::MalformedIdentifier(identifier.to_string()).into()),
start_pos, start_pos,
); );
} }
(Token::Identifier(identifier.into()), start_pos) (Token::Identifier(identifier), start_pos)
} }
/// Is a keyword allowed as a function? /// Is a keyword allowed as a function?