kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

add unicode-xid

Browse Source
This commit is contained in:
ekicyou 2020-07-29 07:03:21 +09:00
parent 79022b1858
commit 9b0375b870
2 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.toml
View File

@ -18,6 +18,7 @@ categories = [ "no-std", "embedded", "parser-implementations" ]
[dependencies] [dependencies]
num-traits = { version = "0.2.11", default-features = false } num-traits = { version = "0.2.11", default-features = false }
unicode-xid = "0.2.1"
[features] [features]
#default = ["unchecked", "sync", "no_optimize", "no_float", "only_i32", "no_index", "no_object", "no_function", "no_module"] #default = ["unchecked", "sync", "no_optimize", "no_float", "only_i32", "no_index", "no_object", "no_function", "no_module"]

13
src/token.rs
View File

@ -1338,6 +1338,9 @@ fn get_next_token_inner(
('\0', _) => unreachable!(), ('\0', _) => unreachable!(),
(ch, _) if ch.is_whitespace() => (), (ch, _) if ch.is_whitespace() => (),
(ch, _) if unicode_xid::UnicodeXID::is_xid_start(ch) => {
return get_identifier(stream, pos, start_pos, c);
}
(ch, _) => { (ch, _) => {
return Some(( return Some((
Token::LexError(Box::new(LERR::UnexpectedInput(ch.to_string()))), Token::LexError(Box::new(LERR::UnexpectedInput(ch.to_string()))),
@ -1409,6 +1412,15 @@ pub fn is_valid_identifier(name: impl Iterator<Item = char>) -> bool {
first_alphabetic first_alphabetic
} }
fn is_id_first_alphabetic(x: char) -> bool {
unicode_xid::UnicodeXID::is_xid_start(x)
}
fn is_id_continue(x: char) -> bool {
unicode_xid::UnicodeXID::is_xid_continue(x)
}
/*
fn is_id_first_alphabetic(x: char) -> bool { fn is_id_first_alphabetic(x: char) -> bool {
x.is_ascii_alphabetic() x.is_ascii_alphabetic()
} }
@ -1416,6 +1428,7 @@ fn is_id_first_alphabetic(x: char) -> bool {
fn is_id_continue(x: char) -> bool { fn is_id_continue(x: char) -> bool {
x.is_ascii_alphanumeric() || x == '_' x.is_ascii_alphanumeric() || x == '_'
} }
*/
/// A type that implements the `InputStream` trait. /// A type that implements the `InputStream` trait.
/// Multiple character streams are jointed together to form one single stream. /// Multiple character streams are jointed together to form one single stream.