kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Avoid copying tokens.

Browse Source
This commit is contained in:
Stephen Chung 2020-03-24 11:21:20 +08:00
parent 7b06715299
commit 156ebd7ea4
1 changed files with 65 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
src/parser.rs
View File

@ -731,12 +731,12 @@ impl Token {
/// An iterator on a `Token` stream. /// An iterator on a `Token` stream.
pub struct TokenIterator<'a> { pub struct TokenIterator<'a> {
/// The last token seen. /// Can the next token be a unary operator?
last: Token, can_be_unary: bool,
/// Current position. /// Current position.
pos: Position, pos: Position,
/// The input characters stream. /// The input characters stream.
char_stream: Peekable<Chars<'a>>, stream: Peekable<Chars<'a>>,
} }
impl<'a> TokenIterator<'a> { impl<'a> TokenIterator<'a> {
@ -766,7 +766,7 @@ impl<'a> TokenIterator<'a> {
let mut escape = String::with_capacity(12); let mut escape = String::with_capacity(12);
loop { loop {
let next_char = self.char_stream.next(); let next_char = self.stream.next();
self.advance(); self.advance();
match next_char.ok_or((LERR::UnterminatedString, Position::eof()))? { match next_char.ok_or((LERR::UnterminatedString, Position::eof()))? {
@ -809,7 +809,7 @@ impl<'a> TokenIterator<'a> {
}; };
for _ in 0..len { for _ in 0..len {
let c = self.char_stream.next().ok_or_else(|| { let c = self.stream.next().ok_or_else(|| {
(LERR::MalformedEscapeSequence(seq.to_string()), self.pos) (LERR::MalformedEscapeSequence(seq.to_string()), self.pos)
})?; })?;
@ -860,7 +860,7 @@ impl<'a> TokenIterator<'a> {
fn inner_next(&mut self) -> Option<(Token, Position)> { fn inner_next(&mut self) -> Option<(Token, Position)> {
let mut negated = false; let mut negated = false;
while let Some(c) = self.char_stream.next() { while let Some(c) = self.stream.next() {
self.advance(); self.advance();
let pos = self.pos; let pos = self.pos;
@ -874,23 +874,23 @@ impl<'a> TokenIterator<'a> {
let mut radix_base: Option<u32> = None; let mut radix_base: Option<u32> = None;
result.push(c); result.push(c);
while let Some(&next_char) = self.char_stream.peek() { while let Some(&next_char) = self.stream.peek() {
match next_char { match next_char {
'0'..='9' | '_' => { '0'..='9' | '_' => {
result.push(next_char); result.push(next_char);
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
} }
#[cfg(not(feature = "no_float"))] #[cfg(not(feature = "no_float"))]
'.' => { '.' => {
result.push(next_char); result.push(next_char);
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
while let Some(&next_char_in_float) = self.char_stream.peek() { while let Some(&next_char_in_float) = self.stream.peek() {
match next_char_in_float { match next_char_in_float {
'0'..='9' | '_' => { '0'..='9' | '_' => {
result.push(next_char_in_float); result.push(next_char_in_float);
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
} }
_ => break, _ => break,
@ -902,7 +902,7 @@ impl<'a> TokenIterator<'a> {
if c == '0' => if c == '0' =>
{ {
result.push(next_char); result.push(next_char);
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
let valid = match ch { let valid = match ch {
@ -928,13 +928,13 @@ impl<'a> TokenIterator<'a> {
_ => panic!("unexpected character {}", ch), _ => panic!("unexpected character {}", ch),
}); });
while let Some(&next_char_in_hex) = self.char_stream.peek() { while let Some(&next_char_in_hex) = self.stream.peek() {
if !valid.contains(&next_char_in_hex) { if !valid.contains(&next_char_in_hex) {
break; break;
} }
result.push(next_char_in_hex); result.push(next_char_in_hex);
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
} }
} }
@ -984,11 +984,11 @@ impl<'a> TokenIterator<'a> {
let mut result = Vec::new(); let mut result = Vec::new();
result.push(c); result.push(c);
while let Some(&next_char) = self.char_stream.peek() { while let Some(&next_char) = self.stream.peek() {
match next_char { match next_char {
x if x.is_ascii_alphanumeric() || x == '_' => { x if x.is_ascii_alphanumeric() || x == '_' => {
result.push(x); result.push(x);
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
} }
_ => break, _ => break,
@ -1085,35 +1085,35 @@ impl<'a> TokenIterator<'a> {
// Operators // Operators
'+' => { '+' => {
return Some(( return Some((
match self.char_stream.peek() { match self.stream.peek() {
Some(&'=') => { Some(&'=') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
Token::PlusAssign Token::PlusAssign
} }
_ if self.last.is_next_unary() => Token::UnaryPlus, _ if self.can_be_unary => Token::UnaryPlus,
_ => Token::Plus, _ => Token::Plus,
}, },
pos, pos,
)) ))
} }
'-' => match self.char_stream.peek() { '-' => match self.stream.peek() {
// Negative number? // Negative number?
Some('0'..='9') if self.last.is_next_unary() => negated = true, Some('0'..='9') if self.can_be_unary => negated = true,
Some('0'..='9') => return Some((Token::Minus, pos)), Some('0'..='9') => return Some((Token::Minus, pos)),
Some('=') => { Some('=') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
return Some((Token::MinusAssign, pos)); return Some((Token::MinusAssign, pos));
} }
_ if self.last.is_next_unary() => return Some((Token::UnaryMinus, pos)), _ if self.can_be_unary => return Some((Token::UnaryMinus, pos)),
_ => return Some((Token::Minus, pos)), _ => return Some((Token::Minus, pos)),
}, },
'*' => { '*' => {
return Some(( return Some((
match self.char_stream.peek() { match self.stream.peek() {
Some(&'=') => { Some(&'=') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
Token::MultiplyAssign Token::MultiplyAssign
} }
@ -1122,11 +1122,11 @@ impl<'a> TokenIterator<'a> {
pos, pos,
)) ))
} }
'/' => match self.char_stream.peek() { '/' => match self.stream.peek() {
Some(&'/') => { Some(&'/') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
while let Some(c) = self.char_stream.next() { while let Some(c) = self.stream.next() {
match c { match c {
'\n' => { '\n' => {
self.new_line(); self.new_line();
@ -1138,20 +1138,20 @@ impl<'a> TokenIterator<'a> {
} }
Some(&'*') => { Some(&'*') => {
let mut level = 1; let mut level = 1;
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
while let Some(c) = self.char_stream.next() { while let Some(c) = self.stream.next() {
self.advance(); self.advance();
match c { match c {
'/' => { '/' => {
if let Some('*') = self.char_stream.next() { if let Some('*') = self.stream.next() {
level += 1; level += 1;
} }
self.advance(); self.advance();
} }
'*' => { '*' => {
if let Some('/') = self.char_stream.next() { if let Some('/') = self.stream.next() {
level -= 1; level -= 1;
} }
self.advance(); self.advance();
@ -1166,7 +1166,7 @@ impl<'a> TokenIterator<'a> {
} }
} }
Some(&'=') => { Some(&'=') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
return Some((Token::DivideAssign, pos)); return Some((Token::DivideAssign, pos));
} }
@ -1176,31 +1176,31 @@ impl<'a> TokenIterator<'a> {
':' => return Some((Token::Colon, pos)), ':' => return Some((Token::Colon, pos)),
',' => return Some((Token::Comma, pos)), ',' => return Some((Token::Comma, pos)),
'.' => return Some((Token::Period, pos)), '.' => return Some((Token::Period, pos)),
'=' => match self.char_stream.peek() { '=' => match self.stream.peek() {
Some(&'=') => { Some(&'=') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
return Some((Token::EqualsTo, pos)); return Some((Token::EqualsTo, pos));
} }
_ => return Some((Token::Equals, pos)), _ => return Some((Token::Equals, pos)),
}, },
'<' => match self.char_stream.peek() { '<' => match self.stream.peek() {
Some(&'=') => { Some(&'=') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
return Some((Token::LessThanEqualsTo, pos)); return Some((Token::LessThanEqualsTo, pos));
} }
Some(&'<') => { Some(&'<') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
return match self.char_stream.peek() { return match self.stream.peek() {
Some(&'=') => { Some(&'=') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
Some((Token::LeftShiftAssign, pos)) Some((Token::LeftShiftAssign, pos))
} }
_ => { _ => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
Some((Token::LeftShift, pos)) Some((Token::LeftShift, pos))
} }
@ -1210,23 +1210,23 @@ impl<'a> TokenIterator<'a> {
}, },
'>' => { '>' => {
return Some(( return Some((
match self.char_stream.peek() { match self.stream.peek() {
Some(&'=') => { Some(&'=') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
Token::GreaterThanEqualsTo Token::GreaterThanEqualsTo
} }
Some(&'>') => { Some(&'>') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
match self.char_stream.peek() { match self.stream.peek() {
Some(&'=') => { Some(&'=') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
Token::RightShiftAssign Token::RightShiftAssign
} }
_ => { _ => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
Token::RightShift Token::RightShift
} }
@ -1239,9 +1239,9 @@ impl<'a> TokenIterator<'a> {
} }
'!' => { '!' => {
return Some(( return Some((
match self.char_stream.peek() { match self.stream.peek() {
Some(&'=') => { Some(&'=') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
Token::NotEqualsTo Token::NotEqualsTo
} }
@ -1252,14 +1252,14 @@ impl<'a> TokenIterator<'a> {
} }
'|' => { '|' => {
return Some(( return Some((
match self.char_stream.peek() { match self.stream.peek() {
Some(&'|') => { Some(&'|') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
Token::Or Token::Or
} }
Some(&'=') => { Some(&'=') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
Token::OrAssign Token::OrAssign
} }
@ -1270,14 +1270,14 @@ impl<'a> TokenIterator<'a> {
} }
'&' => { '&' => {
return Some(( return Some((
match self.char_stream.peek() { match self.stream.peek() {
Some(&'&') => { Some(&'&') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
Token::And Token::And
} }
Some(&'=') => { Some(&'=') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
Token::AndAssign Token::AndAssign
} }
@ -1288,9 +1288,9 @@ impl<'a> TokenIterator<'a> {
} }
'^' => { '^' => {
return Some(( return Some((
match self.char_stream.peek() { match self.stream.peek() {
Some(&'=') => { Some(&'=') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
Token::XOrAssign Token::XOrAssign
} }
@ -1301,9 +1301,9 @@ impl<'a> TokenIterator<'a> {
} }
'%' => { '%' => {
return Some(( return Some((
match self.char_stream.peek() { match self.stream.peek() {
Some(&'=') => { Some(&'=') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
Token::ModuloAssign Token::ModuloAssign
} }
@ -1314,9 +1314,9 @@ impl<'a> TokenIterator<'a> {
} }
'~' => { '~' => {
return Some(( return Some((
match self.char_stream.peek() { match self.stream.peek() {
Some(&'=') => { Some(&'=') => {
self.char_stream.next(); self.stream.next();
self.advance(); self.advance();
Token::PowerOfAssign Token::PowerOfAssign
} }
@ -1337,10 +1337,10 @@ impl<'a> TokenIterator<'a> {
impl<'a> Iterator for TokenIterator<'a> { impl<'a> Iterator for TokenIterator<'a> {
type Item = (Token, Position); type Item = (Token, Position);
// TODO - perhaps this could be optimized?
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
self.inner_next().map(|x| { self.inner_next().map(|x| {
self.last = x.0.clone(); // Save the last token
self.can_be_unary = x.0.is_next_unary();
x x
}) })
} }
@ -1349,9 +1349,9 @@ impl<'a> Iterator for TokenIterator<'a> {
/// Tokenize an input text stream. /// Tokenize an input text stream.
pub fn lex(input: &str) -> TokenIterator<'_> { pub fn lex(input: &str) -> TokenIterator<'_> {
TokenIterator { TokenIterator {
last: Token::LexError(Box::new(LERR::InputError("".into()))), can_be_unary: true,
pos: Position::new(1, 0), pos: Position::new(1, 0),
char_stream: input.chars().peekable(), stream: input.chars().peekable(),
} }
} }