kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Fix bug with tokenizing identifiers.

Browse Source
This commit is contained in:
Stephen Chung 2020-12-29 11:37:15 +08:00
parent eca8212f38
commit 41c6f985f5
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/token.rs
View File

@ -1115,7 +1115,7 @@ fn get_next_token_inner(
} }
} }
// 0x????, 0o????, 0b???? // 0x????, 0o????, 0b????
ch @ 'x' | ch @ 'X' | ch @ 'o' | ch @ 'O' | ch @ 'b' | ch @ 'B' ch @ 'x' | ch @ 'o' | ch @ 'b' | ch @ 'X' | ch @ 'O' | ch @ 'B'
if c == '0' => if c == '0' =>
{ {
result.push(next_char); result.push(next_char);
@ -1184,11 +1184,11 @@ fn get_next_token_inner(
// letter or underscore ... // letter or underscore ...
#[cfg(not(feature = "unicode-xid-ident"))] #[cfg(not(feature = "unicode-xid-ident"))]
('A'..='Z', _) | ('a'..='z', _) | ('_', _) => { ('a'..='z', _) | ('_', _) | ('A'..='Z', _) => {
return get_identifier(stream, pos, start_pos, c); return get_identifier(stream, pos, start_pos, c);
} }
#[cfg(feature = "unicode-xid-ident")] #[cfg(feature = "unicode-xid-ident")]
(ch, _) if unicode_xid::UnicodeXID::is_xid_start(ch) => { (ch, _) if unicode_xid::UnicodeXID::is_xid_start(ch) || ch == '_' => {
return get_identifier(stream, pos, start_pos, c); return get_identifier(stream, pos, start_pos, c);
} }