diff --git a/Cargo.toml b/Cargo.toml index f0fdaa33..66bdba28 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,6 @@ categories = [ "no-std", "embedded", "parser-implementations" ] [dependencies] num-traits = { version = "0.2.11", default-features = false } -unicode-xid = "0.2.1" [features] #default = ["unchecked", "sync", "no_optimize", "no_float", "only_i32", "no_index", "no_object", "no_function", "no_module"] @@ -35,6 +34,7 @@ no_object = [] # no custom objects no_function = [] # no script-defined functions no_module = [] # no modules internals = [] # expose internal data structures +unicode-xid-ident = ["unicode-xid"] # allow unicode-xid for identifiers. # compiling for no-std no_std = [ "num-traits/libm", "hashbrown", "core-error", "libm", "ahash" ] @@ -74,6 +74,11 @@ default_features = false features = ["derive", "alloc"] optional = true +[dependencies.unicode-xid] +version = "0.2.1" +default_features = false +optional = true + [target.'cfg(target_arch = "wasm32")'.dependencies] instant= { version = "0.1.4", features = ["wasm-bindgen"] } # WASM implementation of std::time::Instant diff --git a/src/token.rs b/src/token.rs index 6cb1a4d2..d2f016cb 100644 --- a/src/token.rs +++ b/src/token.rs @@ -1338,6 +1338,7 @@ fn get_next_token_inner( ('\0', _) => unreachable!(), (ch, _) if ch.is_whitespace() => (), + #[cfg(feature = "unicode-xid-ident")] (ch, _) if unicode_xid::UnicodeXID::is_xid_start(ch) => { return get_identifier(stream, pos, start_pos, c); } @@ -1412,23 +1413,26 @@ pub fn is_valid_identifier(name: impl Iterator) -> bool { first_alphabetic } +#[cfg(feature = "unicode-xid-ident")] fn is_id_first_alphabetic(x: char) -> bool { unicode_xid::UnicodeXID::is_xid_start(x) } +#[cfg(feature = "unicode-xid-ident")] fn is_id_continue(x: char) -> bool { unicode_xid::UnicodeXID::is_xid_continue(x) } -/* +#[cfg(not(feature = "unicode-xid-ident"))] + fn is_id_first_alphabetic(x: char) -> bool { x.is_ascii_alphabetic() } +#[cfg(not(feature = "unicode-xid-ident"))] fn is_id_continue(x: char) -> bool { x.is_ascii_alphanumeric() || x == '_' } -*/ /// A type that implements the `InputStream` trait. /// Multiple character streams are jointed together to form one single stream. diff --git a/tests/tokens.rs b/tests/tokens.rs index 523beab7..2523af7d 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -51,3 +51,21 @@ fn test_tokens_custom_operator() -> Result<(), Box> { Ok(()) } + +#[test] +fn test_tokens_unicode_xid_ident() -> Result<(), Box> { + let engine = Engine::new(); + let result = engine.eval::( + r" + fn すべての答え() { 42 } + すべての答え() + ", + ); + #[cfg(feature = "unicode-xid-ident")] + assert_eq!(result?, 42); + + #[cfg(not(feature = "unicode-xid-ident"))] + assert!(result.is_err()); + + Ok(()) +}