add unicode-xid

Cargo.toml

@@ -18,6 +18,7 @@ categories = [ "no-std", "embedded", "parser-implementations" ]
 
 [dependencies]
 num-traits = { version = "0.2.11", default-features = false }
+unicode-xid = "0.2.1"
 
 [features]
 #default = ["unchecked", "sync", "no_optimize", "no_float", "only_i32", "no_index", "no_object", "no_function", "no_module"]

src/token.rs

@@ -1338,6 +1338,9 @@ fn get_next_token_inner(
             ('\0', _) => unreachable!(),
 
             (ch, _) if ch.is_whitespace() => (),
+            (ch, _) if unicode_xid::UnicodeXID::is_xid_start(ch) => {
+                return get_identifier(stream, pos, start_pos, c);
+            }
             (ch, _) => {
                 return Some((
                     Token::LexError(Box::new(LERR::UnexpectedInput(ch.to_string()))),
@@ -1409,6 +1412,15 @@ pub fn is_valid_identifier(name: impl Iterator<Item = char>) -> bool {
     first_alphabetic
 }
 
+fn is_id_first_alphabetic(x: char) -> bool {
+    unicode_xid::UnicodeXID::is_xid_start(x)
+}
+
+fn is_id_continue(x: char) -> bool {
+    unicode_xid::UnicodeXID::is_xid_continue(x)
+}
+
+/*
 fn is_id_first_alphabetic(x: char) -> bool {
     x.is_ascii_alphabetic()
 }
@@ -1416,6 +1428,7 @@ fn is_id_first_alphabetic(x: char) -> bool {
 fn is_id_continue(x: char) -> bool {
     x.is_ascii_alphanumeric() || x == '_'
 }
+*/
 
 /// A type that implements the `InputStream` trait.
 /// Multiple character streams are jointed together to form one single stream.