kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

add get_identifier

Browse Source
This commit is contained in:
ekicyou 2020-07-29 05:26:57 +09:00
parent 5450f66278
commit 56a8120d75
1 changed files with 37 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
src/token.rs
View File

@ -1076,35 +1076,7 @@ fn get_next_token_inner(
// letter or underscore ...
('A'..='Z', _) | ('a'..='z', _) | ('_', _) => {
let mut result = Vec::new();
result.push(c);
while let Some(next_char) = stream.peek_next() {
match next_char {
x if x.is_ascii_alphanumeric() || x == '_' => {
result.push(x);
eat_next(stream, pos);
}
_ => break,
}
}
let is_valid_identifier = is_valid_identifier(result.iter().cloned());
let identifier: String = result.into_iter().collect();
if !is_valid_identifier {
return Some((
Token::LexError(Box::new(LERR::MalformedIdentifier(identifier))),
start_pos,
));
}
return Some((
Token::lookup_from_syntax(&identifier)
.unwrap_or_else(|| Token::Identifier(identifier)),
start_pos,
));
return get_identifier(stream, pos, start_pos, c);
}
// " - string literal
@ -1400,6 +1372,42 @@ fn get_next_token_inner(
}
}
/// Get the next identifier.
fn get_identifier(
stream: &mut impl InputStream,
pos: &mut Position,
start_pos: Position,
first_char: char,
) -> Option<(Token, Position)> {
let mut result = Vec::new();
result.push(first_char);
while let Some(next_char) = stream.peek_next() {
match next_char {
x if x.is_ascii_alphanumeric() || x == '_' => {
result.push(x);
eat_next(stream, pos);
}
_ => break,
}
}
let is_valid_identifier = is_valid_identifier(result.iter().cloned());
let identifier: String = result.into_iter().collect();
if !is_valid_identifier {
return Some((
Token::LexError(Box::new(LERR::MalformedIdentifier(identifier))),
start_pos,
));
}
return Some((
Token::lookup_from_syntax(&identifier).unwrap_or_else(|| Token::Identifier(identifier)),
start_pos,
));
}
/// A type that implements the `InputStream` trait.
/// Multiple character streams are jointed together to form one single stream.
pub struct MultiInputsStream<'a> {