kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Merge pull request #196 from udoprog/tokenize-numeric-literal

Browse Source 
Avoid scanning over slices to validate numerical literals
This commit is contained in:
Stephen Chung 2020-07-25 16:03:17 +08:00 committed by GitHub
commit 22169582bb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 33 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
src/token.rs
View File

@ -810,6 +810,32 @@ pub fn get_next_token(
result
}
/// Test if the given character is a hex character.
fn is_hex_char(c: char) -> bool {
match c {
'a'..='f' => true,
'A'..='F' => true,
'0'..='9' => true,
_ => false,
}
}
/// Test if the given character is an octal character.
fn is_octal_char(c: char) -> bool {
match c {
'0'..='7' => true,
_ => false,
}
}
/// Test if the given character is a binary character.
fn is_binary_char(c: char) -> bool {
match c {
'0' | '1' => true,
_ => false,
}
}
/// Get the next token.
fn get_next_token_inner(
stream: &mut impl InputStream,
@ -872,18 +898,9 @@ fn get_next_token_inner(
eat_next(stream, pos);
let valid = match ch {
'x' | 'X' => [
'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F',
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '_',
],
'o' | 'O' => [
'0', '1', '2', '3', '4', '5', '6', '7', '_', '_', '_', '_',
'_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_',
],
'b' | 'B' => [
'0', '1', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_',
'_', '_', '_', '_', '_', '_', '_', '_', '_', '_', '_',
],
'x' | 'X' => is_hex_char,
'o' | 'O' => is_octal_char,
'b' | 'B' => is_binary_char,
_ => unreachable!(),
};
@ -895,7 +912,7 @@ fn get_next_token_inner(
});
while let Some(next_char_in_escape_seq) = stream.peek_next() {
if !valid.contains(&next_char_in_escape_seq) {
if !valid(next_char_in_escape_seq) {
break;
}

3
tests/number_literals.rs
View File

@ -14,6 +14,7 @@ fn test_hex_literal() -> Result<(), Box<EvalAltResult>> {
let engine = Engine::new();
assert_eq!(engine.eval::<INT>("let x = 0xf; x")?, 15);
assert_eq!(engine.eval::<INT>("let x = 0Xf; x")?, 15);
assert_eq!(engine.eval::<INT>("let x = 0xff; x")?, 255);
Ok(())
@ -24,6 +25,7 @@ fn test_octal_literal() -> Result<(), Box<EvalAltResult>> {
let engine = Engine::new();
assert_eq!(engine.eval::<INT>("let x = 0o77; x")?, 63);
assert_eq!(engine.eval::<INT>("let x = 0O77; x")?, 63);
assert_eq!(engine.eval::<INT>("let x = 0o1234; x")?, 668);
Ok(())
@ -34,6 +36,7 @@ fn test_binary_literal() -> Result<(), Box<EvalAltResult>> {
let engine = Engine::new();
assert_eq!(engine.eval::<INT>("let x = 0b1111; x")?, 15);
assert_eq!(engine.eval::<INT>("let x = 0B1111; x")?, 15);
assert_eq!(
engine.eval::<INT>("let x = 0b0011_1100_1010_0101; x")?,
15525