kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Revise table-driven parsing.

Browse Source
This commit is contained in:
Stephen Chung 2023-03-15 08:54:34 +08:00
parent 72508a07f5
commit 2aa7b99d1e
1 changed files with 8 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/tokenizer.rs
View File

@ -315,7 +315,7 @@ const MAX_KEYWORD_LEN: usize = 8;
const MIN_KEYWORD_HASH_VALUE: usize = 1; const MIN_KEYWORD_HASH_VALUE: usize = 1;
const MAX_KEYWORD_HASH_VALUE: usize = 152; const MAX_KEYWORD_HASH_VALUE: usize = 152;
const KEYWORD_ASSOC_VALUES: [u8; 257] = [ static KEYWORD_ASSOC_VALUES: [u8; 257] = [
153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153,
153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 115, 153, 100, 153, 110, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 115, 153, 100, 153, 110,
105, 40, 80, 2, 20, 25, 125, 95, 15, 40, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 55, 105, 40, 80, 2, 20, 25, 125, 95, 15, 40, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 55,
@ -331,7 +331,7 @@ const KEYWORD_ASSOC_VALUES: [u8; 257] = [
153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153, 153,
153, 153,
]; ];
const KEYWORDS_LIST: [(&str, Token); 153] = [ static KEYWORDS_LIST: [(&str, Token); 153] = [
("", Token::EOF), ("", Token::EOF),
(">", Token::GreaterThan), (">", Token::GreaterThan),
(">=", Token::GreaterThanEqualsTo), (">=", Token::GreaterThanEqualsTo),
@ -515,7 +515,7 @@ const MAX_RESERVED_LEN: usize = 10;
const MIN_RESERVED_HASH_VALUE: usize = 1; const MIN_RESERVED_HASH_VALUE: usize = 1;
const MAX_RESERVED_HASH_VALUE: usize = 112; const MAX_RESERVED_HASH_VALUE: usize = 112;
const RESERVED_ASSOC_VALUES: [u8; 256] = [ static RESERVED_ASSOC_VALUES: [u8; 256] = [
113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 35, 113, 45, 25, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 35, 113, 45, 25, 113,
113, 113, 60, 55, 50, 50, 113, 15, 0, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 60, 55, 50, 50, 113, 15, 0, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
@ -530,8 +530,7 @@ const RESERVED_ASSOC_VALUES: [u8; 256] = [
113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
]; ];
static RESERVED_LIST: [(&str, bool); 113] = [
const RESERVED_LIST: [(&str, bool); 113] = [
("", false), ("", false),
("~", true), ("~", true),
("is", true), ("is", true),
@ -2351,27 +2350,20 @@ pub const fn is_id_continue(x: char) -> bool {
} }
/// Is a piece of syntax a reserved keyword or reserved symbol? /// Is a piece of syntax a reserved keyword or reserved symbol?
#[inline]
#[must_use] #[must_use]
pub fn is_reserved_keyword_or_symbol(syntax: &str) -> bool { pub fn is_reserved_keyword_or_symbol(syntax: &str) -> bool {
let utf8 = syntax.as_bytes(); let utf8 = syntax.as_bytes();
let len = utf8.len(); let len = utf8.len();
let rounds = len.min(3);
let mut hash_val = len; let mut hash_val = len;
if !(MIN_RESERVED_LEN..=MAX_RESERVED_LEN).contains(&len) { if !(MIN_RESERVED_LEN..=MAX_RESERVED_LEN).contains(&len) {
return false; return false;
} }
match len { for x in 0..rounds {
1 => hash_val += RESERVED_ASSOC_VALUES[utf8[0] as usize] as usize, hash_val += RESERVED_ASSOC_VALUES[utf8[rounds - 1 - x] as usize] as usize;
2 => {
hash_val += RESERVED_ASSOC_VALUES[utf8[1] as usize] as usize;
hash_val += RESERVED_ASSOC_VALUES[utf8[0] as usize] as usize;
}
_ => {
hash_val += RESERVED_ASSOC_VALUES[utf8[2] as usize] as usize;
hash_val += RESERVED_ASSOC_VALUES[utf8[1] as usize] as usize;
hash_val += RESERVED_ASSOC_VALUES[utf8[0] as usize] as usize;
}
} }
if !(MIN_RESERVED_HASH_VALUE..=MAX_RESERVED_HASH_VALUE).contains(&hash_val) { if !(MIN_RESERVED_HASH_VALUE..=MAX_RESERVED_HASH_VALUE).contains(&hash_val) {