add unicode_xid, test

Cargo.toml

@@ -18,7 +18,6 @@ categories = [ "no-std", "embedded", "parser-implementations" ]
 
 [dependencies]
 num-traits = { version = "0.2.11", default-features = false }
-unicode-xid = "0.2.1"
 
 [features]
 #default = ["unchecked", "sync", "no_optimize", "no_float", "only_i32", "no_index", "no_object", "no_function", "no_module"]
@@ -35,6 +34,7 @@ no_object = []      # no custom objects
 no_function = []    # no script-defined functions
 no_module = []      # no modules
 internals = []      # expose internal data structures
+unicode-xid-ident = ["unicode-xid"]  # allow unicode-xid for identifiers.
 
 # compiling for no-std
 no_std = [ "num-traits/libm", "hashbrown", "core-error", "libm", "ahash" ]
@@ -74,6 +74,11 @@ default_features = false
 features = ["derive", "alloc"]
 optional = true
 
+[dependencies.unicode-xid]
+version = "0.2.1"
+default_features = false
+optional = true
+
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 instant= { version = "0.1.4", features = ["wasm-bindgen"] } # WASM implementation of std::time::Instant
 

src/token.rs

@@ -1338,6 +1338,7 @@ fn get_next_token_inner(
             ('\0', _) => unreachable!(),
 
             (ch, _) if ch.is_whitespace() => (),
+            #[cfg(feature = "unicode-xid-ident")]
             (ch, _) if unicode_xid::UnicodeXID::is_xid_start(ch) => {
                 return get_identifier(stream, pos, start_pos, c);
             }
@@ -1412,23 +1413,26 @@ pub fn is_valid_identifier(name: impl Iterator<Item = char>) -> bool {
     first_alphabetic
 }
 
+#[cfg(feature = "unicode-xid-ident")]
 fn is_id_first_alphabetic(x: char) -> bool {
     unicode_xid::UnicodeXID::is_xid_start(x)
 }
 
+#[cfg(feature = "unicode-xid-ident")]
 fn is_id_continue(x: char) -> bool {
     unicode_xid::UnicodeXID::is_xid_continue(x)
 }
 
-/*
+#[cfg(not(feature = "unicode-xid-ident"))]
+
 fn is_id_first_alphabetic(x: char) -> bool {
     x.is_ascii_alphabetic()
 }
 
+#[cfg(not(feature = "unicode-xid-ident"))]
 fn is_id_continue(x: char) -> bool {
     x.is_ascii_alphanumeric() || x == '_'
 }
-*/
 
 /// A type that implements the `InputStream` trait.
 /// Multiple character streams are jointed together to form one single stream.

tests/tokens.rs

@@ -51,3 +51,21 @@ fn test_tokens_custom_operator() -> Result<(), Box<EvalAltResult>> {
 
     Ok(())
 }
+
+#[test]
+fn test_tokens_unicode_xid_ident() -> Result<(), Box<EvalAltResult>> {
+    let engine = Engine::new();
+    let result = engine.eval::<INT>(
+        r"
+                fn すべての答え() { 42 }
+                すべての答え()
+            ",
+    );
+    #[cfg(feature = "unicode-xid-ident")]
+    assert_eq!(result?, 42);
+
+    #[cfg(not(feature = "unicode-xid-ident"))]
+    assert!(result.is_err());
+
+    Ok(())
+}