From ecc08271d97faa25aa53a037ab17ea16ae8c6f74 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Sun, 13 Dec 2020 14:31:24 +0800 Subject: [PATCH 1/5] Include actual tokens in custom syntax node. --- RELEASES.md | 11 +++++++++++ src/ast.rs | 23 +++++++++++++++++------ src/engine.rs | 3 --- src/parser.rs | 42 ++++++++++++++++++++++++++---------------- src/syntax.rs | 27 +++++++++++++++------------ tests/syntax.rs | 2 +- 6 files changed, 70 insertions(+), 38 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index 8d2baddb..fc7cedf4 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -4,6 +4,12 @@ Rhai Release Notes Version 0.19.8 ============== +This version makes it easier to generate documentation for a Rhai code base. + +Each function defined in an `AST` can optionally attach _doc-comments_ (which, as in Rust, +are comments prefixed by either `///` or `/**`). Doc-comments allow third-party tools to +automatically generate documentation for functions defined in a Rhai script. + Bug fixes --------- @@ -17,6 +23,11 @@ Breaking changes * The closure for `Engine::on_debug` now takes an additional `Position` parameter. * `AST::iter_functions` now returns `ScriptFnMetadata`. +New features +------------ + +* `AST::iter_functions` now returns `ScriptFnMetadata` which includes, among others, _doc-comments_ for functions prefixed by `///` or `/**`. + Enhancements ------------ diff --git a/src/ast.rs b/src/ast.rs index 0ecf5e76..1fc091fe 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -857,29 +857,40 @@ impl Stmt { /// This type is volatile and may change. #[derive(Clone)] pub struct CustomExpr { - /// List of keywords. - pub(crate) keywords: StaticVec, /// Implementation function. pub(crate) func: Shared, + /// List of keywords. + pub(crate) keywords: StaticVec, + /// List of tokens actually parsed. + pub(crate) tokens: Vec, } impl fmt::Debug for CustomExpr { #[inline(always)] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&self.keywords, f) + f.write_str("CustomExpr { keywords:")?; + fmt::Debug::fmt(&self.keywords, f)?; + f.write_str(", tokens:")?; + fmt::Debug::fmt(&self.tokens, f)?; + f.write_str("}") } } impl CustomExpr { + /// Get the implementation function for this custom syntax. + #[inline(always)] + pub fn func(&self) -> &FnCustomSyntaxEval { + self.func.as_ref() + } /// Get the keywords for this custom syntax. #[inline(always)] pub fn keywords(&self) -> &[Expr] { &self.keywords } - /// Get the implementation function for this custom syntax. + /// Get the actual parsed tokens for this custom syntax. #[inline(always)] - pub fn func(&self) -> &FnCustomSyntaxEval { - self.func.as_ref() + pub fn tokens(&self) -> &[ImmutableString] { + &self.tokens } } diff --git a/src/engine.rs b/src/engine.rs index 958cf1d9..0fad8de5 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -195,9 +195,6 @@ pub const FN_IDX_SET: &str = "index$set$"; pub const FN_ANONYMOUS: &str = "anon$"; #[cfg(any(not(feature = "no_index"), not(feature = "no_object")))] pub const OP_EQUALS: &str = "=="; -pub const MARKER_EXPR: &str = "$expr$"; -pub const MARKER_BLOCK: &str = "$block$"; -pub const MARKER_IDENT: &str = "$ident$"; /// A type specifying the method of chaining. #[cfg(any(not(feature = "no_index"), not(feature = "no_object")))] diff --git a/src/parser.rs b/src/parser.rs index d05e1042..e3b275da 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -4,7 +4,7 @@ use crate::ast::{ BinaryExpr, CustomExpr, Expr, FnCallExpr, Ident, IdentX, ReturnType, ScriptFnDef, Stmt, }; use crate::dynamic::{AccessMode, Union}; -use crate::engine::{KEYWORD_THIS, MARKER_BLOCK, MARKER_EXPR, MARKER_IDENT}; +use crate::engine::KEYWORD_THIS; use crate::module::NamespaceRef; use crate::optimize::optimize_into_ast; use crate::optimize::OptimizationLevel; @@ -20,7 +20,7 @@ use crate::stdlib::{ vec, vec::Vec, }; -use crate::syntax::CustomSyntax; +use crate::syntax::{CustomSyntax, MARKER_BLOCK, MARKER_EXPR, MARKER_IDENT}; use crate::token::{is_doc_comment, is_keyword_function, is_valid_identifier, Token, TokenStream}; use crate::utils::{get_hasher, StraightHasherBuilder}; use crate::{ @@ -1803,7 +1803,9 @@ fn parse_custom_syntax( syntax: &CustomSyntax, pos: Position, ) -> Result { - let mut exprs: StaticVec = Default::default(); + let mut keywords: StaticVec = Default::default(); + let mut segments: StaticVec<_> = Default::default(); + let mut tokens: Vec<_> = Default::default(); // Adjust the variables stack match syntax.scope_delta { @@ -1825,26 +1827,28 @@ fn parse_custom_syntax( let parse_func = &syntax.parse; - let mut segments: StaticVec<_> = Default::default(); - segments.push(key.to_string()); + segments.push(key.into()); + tokens.push(key.into()); loop { settings.pos = input.peek().unwrap().1; let settings = settings.level_up(); - let token = + let required_token = if let Some(seg) = parse_func(&segments).map_err(|err| err.0.into_err(settings.pos))? { seg } else { break; }; - match token.as_str() { + match required_token.as_str() { MARKER_IDENT => match input.next().unwrap() { (Token::Identifier(s), pos) => { - segments.push(s.clone()); - let var_name_def = IdentX::new(state.get_interned_string(s), pos); - exprs.push(Expr::Variable(Box::new((None, None, 0, var_name_def)))); + let ident = state.get_interned_string(s); + let var_name_def = IdentX::new(ident.clone(), pos); + keywords.push(Expr::Variable(Box::new((None, None, 0, var_name_def)))); + segments.push(ident); + tokens.push(state.get_interned_string(MARKER_IDENT)); } (Token::Reserved(s), pos) if is_valid_identifier(s.chars()) => { return Err(PERR::Reserved(s).into_err(pos)); @@ -1852,20 +1856,25 @@ fn parse_custom_syntax( (_, pos) => return Err(PERR::VariableExpected.into_err(pos)), }, MARKER_EXPR => { - exprs.push(parse_expr(input, state, lib, settings)?); - segments.push(MARKER_EXPR.into()); + keywords.push(parse_expr(input, state, lib, settings)?); + let keyword = state.get_interned_string(MARKER_EXPR); + segments.push(keyword.clone()); + tokens.push(keyword); } MARKER_BLOCK => match parse_block(input, state, lib, settings)? { Stmt::Block(statements, pos) => { - exprs.push(Expr::Stmt(Box::new(statements.into()), pos)); - segments.push(MARKER_BLOCK.into()); + keywords.push(Expr::Stmt(Box::new(statements.into()), pos)); + let keyword = state.get_interned_string(MARKER_BLOCK); + segments.push(keyword.clone()); + tokens.push(keyword); } _ => unreachable!(), }, s => match input.next().unwrap() { (Token::LexError(err), pos) => return Err(err.into_err(pos)), (t, _) if t.syntax().as_ref() == s => { - segments.push(t.syntax().into_owned()); + segments.push(required_token.clone()); + tokens.push(required_token.clone()); } (_, pos) => { return Err(PERR::MissingToken( @@ -1880,8 +1889,9 @@ fn parse_custom_syntax( Ok(Expr::Custom( Box::new(CustomExpr { - keywords: exprs, + keywords, func: syntax.func.clone(), + tokens, }), pos, )) diff --git a/src/syntax.rs b/src/syntax.rs index ce221146..a8befa39 100644 --- a/src/syntax.rs +++ b/src/syntax.rs @@ -1,19 +1,19 @@ //! Module implementing custom syntax for [`Engine`]. use crate::ast::Expr; -use crate::engine::{EvalContext, MARKER_BLOCK, MARKER_EXPR, MARKER_IDENT}; +use crate::engine::EvalContext; use crate::fn_native::SendSync; -use crate::stdlib::{ - boxed::Box, - format, - string::{String, ToString}, -}; +use crate::stdlib::{boxed::Box, format, string::ToString}; use crate::token::{is_valid_identifier, Token}; use crate::{ Dynamic, Engine, EvalAltResult, ImmutableString, LexError, ParseError, Position, Shared, StaticVec, }; +pub const MARKER_EXPR: &str = "$expr$"; +pub const MARKER_BLOCK: &str = "$block$"; +pub const MARKER_IDENT: &str = "$ident$"; + /// A general expression evaluation trait object. #[cfg(not(feature = "sync"))] pub type FnCustomSyntaxEval = @@ -25,11 +25,12 @@ pub type FnCustomSyntaxEval = /// A general expression parsing trait object. #[cfg(not(feature = "sync"))] -pub type FnCustomSyntaxParse = dyn Fn(&[String]) -> Result, ParseError>; +pub type FnCustomSyntaxParse = + dyn Fn(&[ImmutableString]) -> Result, ParseError>; /// A general expression parsing trait object. #[cfg(feature = "sync")] pub type FnCustomSyntaxParse = - dyn Fn(&[String]) -> Result, ParseError> + Send + Sync; + dyn Fn(&[ImmutableString]) -> Result, ParseError> + Send + Sync; /// An expression sub-tree in an [`AST`][crate::AST]. #[derive(Debug, Clone)] @@ -100,7 +101,7 @@ impl Engine { /// * `keywords` holds a slice of strings that define the custom syntax. /// * `new_vars` is the number of new variables declared by this custom syntax, or the number of variables removed (if negative). /// * `func` is the implementation function. - pub fn register_custom_syntax + ToString>( + pub fn register_custom_syntax + Into>( &mut self, keywords: impl AsRef<[S]>, new_vars: isize, @@ -110,7 +111,7 @@ impl Engine { ) -> Result<&mut Self, ParseError> { let keywords = keywords.as_ref(); - let mut segments: StaticVec<_> = Default::default(); + let mut segments: StaticVec = Default::default(); for s in keywords { let s = s.as_ref().trim(); @@ -122,7 +123,7 @@ impl Engine { let seg = match s { // Markers not in first position - MARKER_EXPR | MARKER_BLOCK | MARKER_IDENT if !segments.is_empty() => s.to_string(), + MARKER_IDENT | MARKER_EXPR | MARKER_BLOCK if !segments.is_empty() => s.into(), // Standard or reserved keyword/symbol not in first position s if !segments.is_empty() && Token::lookup_from_syntax(s).is_some() => { // Make it a custom keyword/symbol @@ -212,7 +213,9 @@ impl Engine { pub fn register_custom_syntax_raw( &mut self, key: impl Into, - parse: impl Fn(&[String]) -> Result, ParseError> + SendSync + 'static, + parse: impl Fn(&[ImmutableString]) -> Result, ParseError> + + SendSync + + 'static, new_vars: isize, func: impl Fn(&mut EvalContext, &[Expression]) -> Result> + SendSync diff --git a/tests/syntax.rs b/tests/syntax.rs index 526ebf92..04cfaabf 100644 --- a/tests/syntax.rs +++ b/tests/syntax.rs @@ -95,7 +95,7 @@ fn test_custom_syntax_raw() -> Result<(), Box> { "hello", |stream| match stream.len() { 0 => unreachable!(), - 1 => Ok(Some("$ident$".to_string())), + 1 => Ok(Some("$ident$".into())), 2 => match stream[1].as_str() { "world" | "kitty" => Ok(None), s => Err(ParseError( From 6f2fecb76b1a3016406ee7813609c4b2474f4d01 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Mon, 14 Dec 2020 15:15:05 +0800 Subject: [PATCH 2/5] Add type iterator docs. --- RELEASES.md | 1 + doc/src/SUMMARY.md | 1 + doc/src/about/non-design.md | 4 +-- doc/src/about/related.md | 6 +++-- doc/src/language/assignment-op.md | 12 ++++----- doc/src/language/for.md | 2 +- doc/src/language/iterator.md | 45 +++++++++++++++++++++++++++++++ doc/src/language/logic.md | 27 +++++++++++++------ doc/src/language/modules/index.md | 2 +- doc/src/language/try-catch.md | 2 +- doc/src/links.md | 2 ++ doc/src/plugins/module.md | 2 +- doc/src/rust/modules/create.md | 2 +- doc/src/rust/modules/index.md | 2 +- doc/src/start/install.md | 3 ++- src/engine_api.rs | 6 ++--- 16 files changed, 91 insertions(+), 28 deletions(-) create mode 100644 doc/src/language/iterator.md diff --git a/RELEASES.md b/RELEASES.md index fc7cedf4..32b16d16 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -15,6 +15,7 @@ Bug fixes * Constants are no longer propagated by the optimizer if shadowed by a non-constant variable. * Constants passed as the `this` parameter to Rhai functions now throws an error if assigned to. +* Generic type parameter of `Engine::register_iterator` is `IntoIterator` instead of `Iterator`. Breaking changes ---------------- diff --git a/doc/src/SUMMARY.md b/doc/src/SUMMARY.md index 270f6d1b..556a60fb 100644 --- a/doc/src/SUMMARY.md +++ b/doc/src/SUMMARY.md @@ -82,6 +82,7 @@ The Rhai Scripting Language 11. [Do Loop](language/do.md) 12. [Loop Statement](language/loop.md) 13. [For Loop](language/for.md) + 1. [Iterators for Custom Types](language/iterator.md) 14. [Return Values](language/return.md) 15. [Throw Exception on Error](language/throw.md) 16. [Catch Exceptions](language/try-catch.md) diff --git a/doc/src/about/non-design.md b/doc/src/about/non-design.md index a8b06ea8..4f01b14a 100644 --- a/doc/src/about/non-design.md +++ b/doc/src/about/non-design.md @@ -44,8 +44,8 @@ It doesn't attempt to be a new language. For example: * **No formal language grammar** - Rhai uses a hand-coded lexer, a hand-coded top-down recursive-descent parser for statements, and a hand-coded Pratt parser for expressions. - This lack of formalism allows the _parser_ itself to be exposed as a service in order to support - [disabling keywords/operators][disable keywords and operators], adding [custom operators], + This lack of formalism allows the _tokenizer_ and _parser_ themselves to be exposed as services in order + to support [disabling keywords/operators][disable keywords and operators], adding [custom operators], and defining [custom syntax]. diff --git a/doc/src/about/related.md b/doc/src/about/related.md index 596b54e7..9b55003b 100644 --- a/doc/src/about/related.md +++ b/doc/src/about/related.md @@ -4,8 +4,10 @@ Related Resources {{#include ../links.md}} -Other Online Resources for Rhai ------------------------------- +Online Resources for Rhai +------------------------- + +* [GitHub](https://github.com/jonathandturner/rhai) - Home repository * [`crates.io`](https://crates.io/crates/rhai) - Rhai crate diff --git a/doc/src/language/assignment-op.md b/doc/src/language/assignment-op.md index d604dac3..5cfd73c5 100644 --- a/doc/src/language/assignment-op.md +++ b/doc/src/language/assignment-op.md @@ -34,7 +34,10 @@ number ^= 0x00ff; // number = number ^ 0x00ff; The Flexible `+=` ---------------- -The `+=` operator can also be used to build [strings]: +The the `+` and `+=` operators are often [overloaded][function overloading] to perform +build-up operations for different data types. + +For example, it is used to build [strings]: ```rust let my_str = "abc"; @@ -44,7 +47,7 @@ my_str += 12345; my_str == "abcABC12345" ``` -It may also be used to concatenate [arrays]: +to concatenate [arrays]: ```rust let my_array = [1, 2, 3]; @@ -53,7 +56,7 @@ my_array += [4, 5]; my_array == [1, 2, 3, 4, 5]; ``` -or mix two [object maps] together: +and mix two [object maps] together: ```rust let my_obj = #{a:1, b:2}; @@ -61,6 +64,3 @@ my_obj += #{c:3, d:4, e:5}; my_obj.len() == 5; ``` - -In fact, the `+` and `+=` operators are usually [overloaded][function overloading] when -something is to be _added_ to an existing type. diff --git a/doc/src/language/for.md b/doc/src/language/for.md index 4985b7aa..6af5a422 100644 --- a/doc/src/language/for.md +++ b/doc/src/language/for.md @@ -3,7 +3,7 @@ {{#include ../links.md}} -Iterating through a range or an [array], or any type with a registered _type iterator_, +Iterating through a range or an [array], or any type with a registered [type iterator], is provided by the `for` ... `in` loop. Like C, `continue` can be used to skip to the next iteration, by-passing all following statements; diff --git a/doc/src/language/iterator.md b/doc/src/language/iterator.md new file mode 100644 index 00000000..e885efba --- /dev/null +++ b/doc/src/language/iterator.md @@ -0,0 +1,45 @@ +Iterators for Custom Types +========================== + +{{#include ../links.md}} + +If a [custom type] is iterable, the [`for`](for.md) loop can be used to iterate through +its items in sequence. + +In order to use a [`for`](for.md) statement, a _type iterator_ must be registered for +the [custom type] in question. + +`Engine::register_iterator` allows registration of a _type iterator_ for any type +that implements `IntoIterator`: + +```rust +// Custom type +#[derive(Debug, Clone)] +struct TestStruct { ... } + +// Implement 'IntoIterator' trait +impl IntoIterator for TestStruct { + type Item = ...; + type IntoIter = SomeIterType; + + fn into_iter(self) -> Self::IntoIter { + ... + } +} + +engine + .register_type_with_name::("TestStruct") + .register_fn("new_ts", || TestStruct { ... }) + .register_iterator::(); // register type iterator +``` + +With a type iterator registered, the [custom type] can be iterated through: + +```rust +let ts = new_ts(); + +// Use 'for' statement to loop through items in 'ts' +for item in ts { + ... +} +``` diff --git a/doc/src/language/logic.md b/doc/src/language/logic.md index 498cab0d..edb1ae41 100644 --- a/doc/src/language/logic.md +++ b/doc/src/language/logic.md @@ -6,6 +6,15 @@ Logic Operators Comparison Operators ------------------- +| Operator | Description | +| :------: | ------------------------- | +| `==` | equals to | +| `!=` | not equals to | +| `>` | greater than | +| `>=` | greater than or equals to | +| `<` | less than | +| `<=` | less than or equals to | + Comparing most values of the same data type work out-of-the-box for all [standard types] supported by the system. However, if using a [raw `Engine`] without loading any [packages], comparisons can only be made between a limited @@ -43,15 +52,15 @@ ts != 42; // true - types cannot be compared Boolean operators ----------------- -| Operator | Description | -| ----------------- | ------------------------------------- | -| `!` | boolean _Not_ | -| `&&` | boolean _And_ (short-circuits) | -| \|\| | boolean _Or_ (short-circuits) | -| `&` | boolean _And_ (doesn't short-circuit) | -| \| | boolean _Or_ (doesn't short-circuit) | +| Operator | Description | Short-Circuits? | +| :---------------: | ------------- | :-------------: | +| `!` (prefix) | boolean _NOT_ | no | +| `&&` | boolean _AND_ | yes | +| `&` | boolean _AND_ | no | +| \|\| | boolean _OR_ | yes | +| \| | boolean _OR_ | no | -Double boolean operators `&&` and `||` _short-circuit_, meaning that the second operand will not be evaluated +Double boolean operators `&&` and `||` _short-circuit_ - meaning that the second operand will not be evaluated if the first one already proves the condition wrong. Single boolean operators `&` and `|` always evaluate both operands. @@ -65,3 +74,5 @@ a() | b(); // both a() and b() are evaluated a() & b(); // both a() and b() are evaluated ``` + +All boolean operators are [built in][built-in operators] for the `bool` data type. diff --git a/doc/src/language/modules/index.md b/doc/src/language/modules/index.md index 14931c3a..dd8c928a 100644 --- a/doc/src/language/modules/index.md +++ b/doc/src/language/modules/index.md @@ -6,7 +6,7 @@ Modules Rhai allows organizing code (functions, both Rust-based or script-based, and variables) into _modules_. Modules can be disabled via the [`no_module`] feature. -A module is of the type `Module` and holds a collection of functions, variables, type iterators and sub-modules. +A module is of the type `Module` and holds a collection of functions, variables, [type iterators] and sub-modules. It may be created entirely from Rust functions, or it may encapsulate a Rhai script together with the functions and variables defined by that script. diff --git a/doc/src/language/try-catch.md b/doc/src/language/try-catch.md index c28236c5..73d4da2f 100644 --- a/doc/src/language/try-catch.md +++ b/doc/src/language/try-catch.md @@ -91,7 +91,7 @@ Many script-oriented exceptions can be caught via `try` ... `catch`: | [Array]/[string] indexing out-of-bounds | error message [string] | | Indexing with an inappropriate data type | error message [string] | | Error in a dot expression | error message [string] | -| `for` statement without an type iterator | error message [string] | +| `for` statement without a [type iterator] | error message [string] | | Error in an `in` expression | error message [string] | | Data race detected | error message [string] | diff --git a/doc/src/links.md b/doc/src/links.md index a63f1b26..115c8890 100644 --- a/doc/src/links.md +++ b/doc/src/links.md @@ -57,6 +57,8 @@ [custom types]: {{rootUrl}}/rust/custom.md [getters/setters]: {{rootUrl}}/rust/getters-setters.md [indexers]: {{rootUrl}}/rust/indexers.md +[type iterator]: {{rootUrl}}/language/iterator.md +[type iterators]: {{rootUrl}}/language/iterator.md [`instant::Instant`]: https://crates.io/crates/instant diff --git a/doc/src/plugins/module.md b/doc/src/plugins/module.md index a1e2a684..ac0c9f80 100644 --- a/doc/src/plugins/module.md +++ b/doc/src/plugins/module.md @@ -162,7 +162,7 @@ x == 43; ``` All functions (usually _methods_) defined in the module and marked with `#[rhai_fn(global)]`, -as well as all _type iterators_, are automatically exposed to the _global_ namespace, so +as well as all [type iterators], are automatically exposed to the _global_ namespace, so [iteration]({{rootUrl}}/language/for.md), [getters/setters] and [indexers] for [custom types] can work as expected. diff --git a/doc/src/rust/modules/create.md b/doc/src/rust/modules/create.md index 6e1a1479..4c279800 100644 --- a/doc/src/rust/modules/create.md +++ b/doc/src/rust/modules/create.md @@ -75,7 +75,7 @@ engine.eval::("calc::inc(41)")? == 42; // refer to the 'Calc' module `Module::set_fn_XXX_mut` can expose functions (usually _methods_) in the module to the _global_ namespace, so [getters/setters] and [indexers] for [custom types] can work as expected. -Type iterators, because of their special nature, are always exposed to the _global_ namespace. +[Type iterators], because of their special nature, are always exposed to the _global_ namespace. ```rust use rhai::{Engine, Module, FnNamespace}; diff --git a/doc/src/rust/modules/index.md b/doc/src/rust/modules/index.md index 14931c3a..dd8c928a 100644 --- a/doc/src/rust/modules/index.md +++ b/doc/src/rust/modules/index.md @@ -6,7 +6,7 @@ Modules Rhai allows organizing code (functions, both Rust-based or script-based, and variables) into _modules_. Modules can be disabled via the [`no_module`] feature. -A module is of the type `Module` and holds a collection of functions, variables, type iterators and sub-modules. +A module is of the type `Module` and holds a collection of functions, variables, [type iterators] and sub-modules. It may be created entirely from Rust functions, or it may encapsulate a Rhai script together with the functions and variables defined by that script. diff --git a/doc/src/start/install.md b/doc/src/start/install.md index a7dc944f..007f6d63 100644 --- a/doc/src/start/install.md +++ b/doc/src/start/install.md @@ -21,7 +21,8 @@ rhai = "*" ``` Crate versions are released on [`crates.io`](https:/crates.io/crates/rhai/) infrequently, -so to track the latest features, enhancements and bug fixes, pull directly from GitHub: +so to track the latest features, enhancements and bug fixes, pull directly from +[GitHub](https://github.com/jonathandturner/rhai): ```toml [dependencies] diff --git a/src/engine_api.rs b/src/engine_api.rs index 118c4dcc..fe6a9a48 100644 --- a/src/engine_api.rs +++ b/src/engine_api.rs @@ -158,13 +158,13 @@ impl Engine { self.type_names.insert(type_name::().into(), name.into()); self } - /// Register an iterator adapter for an iterable type with the [`Engine`]. + /// Register an type iterator for an iterable type with the [`Engine`]. /// This is an advanced feature. #[inline(always)] pub fn register_iterator(&mut self) -> &mut Self where - T: Variant + Clone + Iterator, - ::Item: Variant + Clone, + T: Variant + Clone + IntoIterator, + ::Item: Variant + Clone, { self.global_namespace.set_iterable::(); self From 5ea6efe6fda5ac2cc8192764c05e22a99b2fa005 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Mon, 14 Dec 2020 15:15:16 +0800 Subject: [PATCH 3/5] Add doc-comment docs. --- doc/src/language/comments.md | 46 ++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/doc/src/language/comments.md b/doc/src/language/comments.md index 4f01f680..eab5273c 100644 --- a/doc/src/language/comments.md +++ b/doc/src/language/comments.md @@ -3,13 +3,15 @@ Comments {{#include ../links.md}} -Comments are C-style, including '`/*` ... `*/`' pairs and '`//`' for comments to the end of the line. +Comments are C-style, including '`/*` ... `*/`' pairs for block comments +and '`//`' for comments to the end of the line. + Comments can be nested. ```rust let /* intruder comment */ name = "Bob"; -// This is a very important comment +// This is a very important one-line comment /* This comment spans multiple lines, so it @@ -20,3 +22,43 @@ let /* intruder comment */ name = "Bob"; /*/*/*/*/**/*/*/*/*/ */ ``` + + +Doc-Comments +------------ + +Similar to Rust, comments starting with `///` (three slashes) or `/**` (two asterisks) are +_doc-comments_. + +Doc-comments can only appear in front of [function] definitions, not any other elements: + +```rust +/// This is a valid one-line doc-comment +fn foo() {} + +/** This is a + ** valid block + ** doc-comment + **/ +fn bar(x) { + /// Syntax error - this doc-comment is invalid + x + 1 +} + +/** Syntax error - this doc-comment is invalid */ +let x = 42; + +/// Syntax error - this doc-comment is also invalid +{ + let x = 42; +} +``` + +Doc-comments are stored within the script's [`AST`] after compilation. + +The `AST::iter_functions` method provides a `ScriptFnMetadata` instance +for each function defined within the script, which includes doc-comments. + +Doc-comments never affect the evaluation of a script nor do they incur +significant performance overhead. However, third party tools can take advantage +of this information to auto-generate documentation for Rhai script functions. From 17310ef576e8e66c776c304f07d6af39208ee507 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Mon, 14 Dec 2020 23:05:13 +0800 Subject: [PATCH 4/5] Encapsulate structures. --- src/ast.rs | 24 +++--------------------- src/engine.rs | 14 ++++++++++++-- tests/syntax.rs | 4 ++-- tests/var_scope.rs | 2 +- 4 files changed, 18 insertions(+), 26 deletions(-) diff --git a/src/ast.rs b/src/ast.rs index 1fc091fe..e9823156 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -858,11 +858,11 @@ impl Stmt { #[derive(Clone)] pub struct CustomExpr { /// Implementation function. - pub(crate) func: Shared, + pub func: Shared, /// List of keywords. - pub(crate) keywords: StaticVec, + pub keywords: StaticVec, /// List of tokens actually parsed. - pub(crate) tokens: Vec, + pub tokens: Vec, } impl fmt::Debug for CustomExpr { @@ -876,24 +876,6 @@ impl fmt::Debug for CustomExpr { } } -impl CustomExpr { - /// Get the implementation function for this custom syntax. - #[inline(always)] - pub fn func(&self) -> &FnCustomSyntaxEval { - self.func.as_ref() - } - /// Get the keywords for this custom syntax. - #[inline(always)] - pub fn keywords(&self) -> &[Expr] { - &self.keywords - } - /// Get the actual parsed tokens for this custom syntax. - #[inline(always)] - pub fn tokens(&self) -> &[ImmutableString] { - &self.tokens - } -} - /// _(INTERNALS)_ A binary expression. /// Exported under the `internals` feature only. /// diff --git a/src/engine.rs b/src/engine.rs index 0fad8de5..0fa2ee9f 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -532,7 +532,7 @@ pub struct Limits { #[derive(Debug)] pub struct EvalContext<'e, 'x, 'px: 'x, 'a, 's, 'm, 'pm: 'm, 't, 'pt: 't> { pub(crate) engine: &'e Engine, - pub scope: &'x mut Scope<'px>, + pub(crate) scope: &'x mut Scope<'px>, pub(crate) mods: &'a mut Imports, pub(crate) state: &'s mut State, pub(crate) lib: &'m [&'pm Module], @@ -546,6 +546,16 @@ impl<'e, 'x, 'px, 'a, 's, 'm, 'pm, 't, 'pt> EvalContext<'e, 'x, 'px, 'a, 's, 'm, pub fn engine(&self) -> &'e Engine { self.engine } + /// The current [`Scope`]. + #[inline(always)] + pub fn scope(&self) -> &Scope<'px> { + self.scope + } + /// Mutable reference to the current [`Scope`]. + #[inline(always)] + pub fn scope_mut(&mut self) -> &mut &'x mut Scope<'px> { + &mut self.scope + } /// _(INTERNALS)_ The current set of modules imported via `import` statements. /// Available under the `internals` feature only. #[cfg(feature = "internals")] @@ -1822,7 +1832,7 @@ impl Engine { Expr::Custom(custom, _) => { let expressions = custom - .keywords() + .keywords .iter() .map(Into::into) .collect::>(); diff --git a/tests/syntax.rs b/tests/syntax.rs index 04cfaabf..2f35eca3 100644 --- a/tests/syntax.rs +++ b/tests/syntax.rs @@ -27,7 +27,7 @@ fn test_custom_syntax() -> Result<(), Box> { let stmt = inputs.get(1).unwrap(); let condition = inputs.get(2).unwrap(); - context.scope.push(var_name, 0 as INT); + context.scope_mut().push(var_name, 0 as INT); loop { context.eval_expression_tree(stmt)?; @@ -110,7 +110,7 @@ fn test_custom_syntax_raw() -> Result<(), Box> { }, 1, |context, inputs| { - context.scope.push("foo", 999 as INT); + context.scope_mut().push("foo", 999 as INT); Ok(match inputs[0].get_variable_name().unwrap() { "world" => 123 as INT, diff --git a/tests/var_scope.rs b/tests/var_scope.rs index 52fff87f..6c4c41aa 100644 --- a/tests/var_scope.rs +++ b/tests/var_scope.rs @@ -71,7 +71,7 @@ fn test_var_resolver() -> Result<(), Box> { } // Silently maps 'chameleon' into 'innocent'. "chameleon" => context - .scope + .scope() .get_value("innocent") .map(Some) .ok_or_else(|| { From f8c14ba1c4794d456c50eb6854c963f6b007c5cd Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Tue, 15 Dec 2020 19:23:30 +0800 Subject: [PATCH 5/5] Add look-ahead to custom syntax parser. --- RELEASES.md | 2 ++ src/parser.rs | 16 +++++++++------- src/syntax.rs | 8 ++++---- tests/syntax.rs | 2 +- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index 32b16d16..d442c231 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -23,6 +23,7 @@ Breaking changes * `Engine::on_progress` now takes `u64` instead of `&u64`. * The closure for `Engine::on_debug` now takes an additional `Position` parameter. * `AST::iter_functions` now returns `ScriptFnMetadata`. +* The parser function passed to `Engine::register_custom_syntax_raw` now takes an additional parameter containing the _look-ahead_ symbol. New features ------------ @@ -34,6 +35,7 @@ Enhancements * Capturing a constant variable in a closure is now supported, with no cloning. * Provides position info for `debug` statements. +* A _look-ahead_ symbol is provided to custom syntax parsers, which can be used to parse variable-length symbol streams. Version 0.19.7 diff --git a/src/parser.rs b/src/parser.rs index e3b275da..afd0dd5f 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1831,15 +1831,17 @@ fn parse_custom_syntax( tokens.push(key.into()); loop { - settings.pos = input.peek().unwrap().1; + let (fwd_token, fwd_pos) = input.peek().unwrap(); + settings.pos = *fwd_pos; let settings = settings.level_up(); - let required_token = - if let Some(seg) = parse_func(&segments).map_err(|err| err.0.into_err(settings.pos))? { - seg - } else { - break; - }; + let required_token = if let Some(seg) = parse_func(&segments, fwd_token.syntax().as_ref()) + .map_err(|err| err.0.into_err(settings.pos))? + { + seg + } else { + break; + }; match required_token.as_str() { MARKER_IDENT => match input.next().unwrap() { diff --git a/src/syntax.rs b/src/syntax.rs index a8befa39..4ceec8f9 100644 --- a/src/syntax.rs +++ b/src/syntax.rs @@ -26,11 +26,11 @@ pub type FnCustomSyntaxEval = /// A general expression parsing trait object. #[cfg(not(feature = "sync"))] pub type FnCustomSyntaxParse = - dyn Fn(&[ImmutableString]) -> Result, ParseError>; + dyn Fn(&[ImmutableString], &str) -> Result, ParseError>; /// A general expression parsing trait object. #[cfg(feature = "sync")] pub type FnCustomSyntaxParse = - dyn Fn(&[ImmutableString]) -> Result, ParseError> + Send + Sync; + dyn Fn(&[ImmutableString], &str) -> Result, ParseError> + Send + Sync; /// An expression sub-tree in an [`AST`][crate::AST]. #[derive(Debug, Clone)] @@ -185,7 +185,7 @@ impl Engine { self.register_custom_syntax_raw( key, // Construct the parsing function - move |stream| { + move |stream, _| { if stream.len() >= segments.len() { Ok(None) } else { @@ -213,7 +213,7 @@ impl Engine { pub fn register_custom_syntax_raw( &mut self, key: impl Into, - parse: impl Fn(&[ImmutableString]) -> Result, ParseError> + parse: impl Fn(&[ImmutableString], &str) -> Result, ParseError> + SendSync + 'static, new_vars: isize, diff --git a/tests/syntax.rs b/tests/syntax.rs index 2f35eca3..f2049e86 100644 --- a/tests/syntax.rs +++ b/tests/syntax.rs @@ -93,7 +93,7 @@ fn test_custom_syntax_raw() -> Result<(), Box> { engine.register_custom_syntax_raw( "hello", - |stream| match stream.len() { + |stream, _| match stream.len() { 0 => unreachable!(), 1 => Ok(Some("$ident$".into())), 2 => match stream[1].as_str() {