Include actual tokens in custom syntax node.

RELEASES.md

@@ -4,6 +4,12 @@ Rhai Release Notes
 Version 0.19.8
 ==============
 
+This version makes it easier to generate documentation for a Rhai code base.
+
+Each function defined in an `AST` can optionally attach _doc-comments_ (which, as in Rust,
+are comments prefixed by either `///` or `/**`).  Doc-comments allow third-party tools to
+automatically generate documentation for functions defined in a Rhai script.
+
 Bug fixes
 ---------
 
@@ -17,6 +23,11 @@ Breaking changes
 * The closure for `Engine::on_debug` now takes an additional `Position` parameter.
 * `AST::iter_functions` now returns `ScriptFnMetadata`.
 
+New features
+------------
+
+* `AST::iter_functions` now returns `ScriptFnMetadata` which includes, among others, _doc-comments_ for functions prefixed by `///` or `/**`.
+
 Enhancements
 ------------
 

src/ast.rs

@@ -857,29 +857,40 @@ impl Stmt {
 /// This type is volatile and may change.
 #[derive(Clone)]
 pub struct CustomExpr {
-    /// List of keywords.
-    pub(crate) keywords: StaticVec<Expr>,
     /// Implementation function.
     pub(crate) func: Shared<FnCustomSyntaxEval>,
+    /// List of keywords.
+    pub(crate) keywords: StaticVec<Expr>,
+    /// List of tokens actually parsed.
+    pub(crate) tokens: Vec<ImmutableString>,
 }
 
 impl fmt::Debug for CustomExpr {
     #[inline(always)]
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        fmt::Debug::fmt(&self.keywords, f)
+        f.write_str("CustomExpr { keywords:")?;
+        fmt::Debug::fmt(&self.keywords, f)?;
+        f.write_str(", tokens:")?;
+        fmt::Debug::fmt(&self.tokens, f)?;
+        f.write_str("}")
     }
 }
 
 impl CustomExpr {
+    /// Get the implementation function for this custom syntax.
+    #[inline(always)]
+    pub fn func(&self) -> &FnCustomSyntaxEval {
+        self.func.as_ref()
+    }
     /// Get the keywords for this custom syntax.
     #[inline(always)]
     pub fn keywords(&self) -> &[Expr] {
         &self.keywords
     }
-    /// Get the implementation function for this custom syntax.
+    /// Get the actual parsed tokens for this custom syntax.
     #[inline(always)]
-    pub fn func(&self) -> &FnCustomSyntaxEval {
+    pub fn tokens(&self) -> &[ImmutableString] {
-        self.func.as_ref()
+        &self.tokens
     }
 }
 

src/engine.rs

@@ -195,9 +195,6 @@ pub const FN_IDX_SET: &str = "index$set$";
 pub const FN_ANONYMOUS: &str = "anon$";
 #[cfg(any(not(feature = "no_index"), not(feature = "no_object")))]
 pub const OP_EQUALS: &str = "==";
-pub const MARKER_EXPR: &str = "$expr$";
-pub const MARKER_BLOCK: &str = "$block$";
-pub const MARKER_IDENT: &str = "$ident$";
 
 /// A type specifying the method of chaining.
 #[cfg(any(not(feature = "no_index"), not(feature = "no_object")))]

src/parser.rs

@@ -4,7 +4,7 @@ use crate::ast::{
     BinaryExpr, CustomExpr, Expr, FnCallExpr, Ident, IdentX, ReturnType, ScriptFnDef, Stmt,
 };
 use crate::dynamic::{AccessMode, Union};
-use crate::engine::{KEYWORD_THIS, MARKER_BLOCK, MARKER_EXPR, MARKER_IDENT};
+use crate::engine::KEYWORD_THIS;
 use crate::module::NamespaceRef;
 use crate::optimize::optimize_into_ast;
 use crate::optimize::OptimizationLevel;
@@ -20,7 +20,7 @@ use crate::stdlib::{
     vec,
     vec::Vec,
 };
-use crate::syntax::CustomSyntax;
+use crate::syntax::{CustomSyntax, MARKER_BLOCK, MARKER_EXPR, MARKER_IDENT};
 use crate::token::{is_doc_comment, is_keyword_function, is_valid_identifier, Token, TokenStream};
 use crate::utils::{get_hasher, StraightHasherBuilder};
 use crate::{
@@ -1803,7 +1803,9 @@ fn parse_custom_syntax(
     syntax: &CustomSyntax,
     pos: Position,
 ) -> Result<Expr, ParseError> {
-    let mut exprs: StaticVec<Expr> = Default::default();
+    let mut keywords: StaticVec<Expr> = Default::default();
+    let mut segments: StaticVec<_> = Default::default();
+    let mut tokens: Vec<_> = Default::default();
 
     // Adjust the variables stack
     match syntax.scope_delta {
@@ -1825,26 +1827,28 @@ fn parse_custom_syntax(
 
     let parse_func = &syntax.parse;
 
-    let mut segments: StaticVec<_> = Default::default();
+    segments.push(key.into());
-    segments.push(key.to_string());
+    tokens.push(key.into());
 
     loop {
         settings.pos = input.peek().unwrap().1;
         let settings = settings.level_up();
 
-        let token =
+        let required_token =
             if let Some(seg) = parse_func(&segments).map_err(|err| err.0.into_err(settings.pos))? {
                 seg
             } else {
                 break;
             };
 
-        match token.as_str() {
+        match required_token.as_str() {
             MARKER_IDENT => match input.next().unwrap() {
                 (Token::Identifier(s), pos) => {
-                    segments.push(s.clone());
+                    let ident = state.get_interned_string(s);
-                    let var_name_def = IdentX::new(state.get_interned_string(s), pos);
+                    let var_name_def = IdentX::new(ident.clone(), pos);
-                    exprs.push(Expr::Variable(Box::new((None, None, 0, var_name_def))));
+                    keywords.push(Expr::Variable(Box::new((None, None, 0, var_name_def))));
+                    segments.push(ident);
+                    tokens.push(state.get_interned_string(MARKER_IDENT));
                 }
                 (Token::Reserved(s), pos) if is_valid_identifier(s.chars()) => {
                     return Err(PERR::Reserved(s).into_err(pos));
@@ -1852,20 +1856,25 @@ fn parse_custom_syntax(
                 (_, pos) => return Err(PERR::VariableExpected.into_err(pos)),
             },
             MARKER_EXPR => {
-                exprs.push(parse_expr(input, state, lib, settings)?);
+                keywords.push(parse_expr(input, state, lib, settings)?);
-                segments.push(MARKER_EXPR.into());
+                let keyword = state.get_interned_string(MARKER_EXPR);
+                segments.push(keyword.clone());
+                tokens.push(keyword);
             }
             MARKER_BLOCK => match parse_block(input, state, lib, settings)? {
                 Stmt::Block(statements, pos) => {
-                    exprs.push(Expr::Stmt(Box::new(statements.into()), pos));
+                    keywords.push(Expr::Stmt(Box::new(statements.into()), pos));
-                    segments.push(MARKER_BLOCK.into());
+                    let keyword = state.get_interned_string(MARKER_BLOCK);
+                    segments.push(keyword.clone());
+                    tokens.push(keyword);
                 }
                 _ => unreachable!(),
             },
             s => match input.next().unwrap() {
                 (Token::LexError(err), pos) => return Err(err.into_err(pos)),
                 (t, _) if t.syntax().as_ref() == s => {
-                    segments.push(t.syntax().into_owned());
+                    segments.push(required_token.clone());
+                    tokens.push(required_token.clone());
                 }
                 (_, pos) => {
                     return Err(PERR::MissingToken(
@@ -1880,8 +1889,9 @@ fn parse_custom_syntax(
 
     Ok(Expr::Custom(
         Box::new(CustomExpr {
-            keywords: exprs,
+            keywords,
             func: syntax.func.clone(),
+            tokens,
         }),
         pos,
     ))

src/syntax.rs

@@ -1,19 +1,19 @@
 //! Module implementing custom syntax for [`Engine`].
 
 use crate::ast::Expr;
-use crate::engine::{EvalContext, MARKER_BLOCK, MARKER_EXPR, MARKER_IDENT};
+use crate::engine::EvalContext;
 use crate::fn_native::SendSync;
-use crate::stdlib::{
+use crate::stdlib::{boxed::Box, format, string::ToString};
-    boxed::Box,
-    format,
-    string::{String, ToString},
-};
 use crate::token::{is_valid_identifier, Token};
 use crate::{
     Dynamic, Engine, EvalAltResult, ImmutableString, LexError, ParseError, Position, Shared,
     StaticVec,
 };
 
+pub const MARKER_EXPR: &str = "$expr$";
+pub const MARKER_BLOCK: &str = "$block$";
+pub const MARKER_IDENT: &str = "$ident$";
+
 /// A general expression evaluation trait object.
 #[cfg(not(feature = "sync"))]
 pub type FnCustomSyntaxEval =
@@ -25,11 +25,12 @@ pub type FnCustomSyntaxEval =
 
 /// A general expression parsing trait object.
 #[cfg(not(feature = "sync"))]
-pub type FnCustomSyntaxParse = dyn Fn(&[String]) -> Result<Option<String>, ParseError>;
+pub type FnCustomSyntaxParse =
+    dyn Fn(&[ImmutableString]) -> Result<Option<ImmutableString>, ParseError>;
 /// A general expression parsing trait object.
 #[cfg(feature = "sync")]
 pub type FnCustomSyntaxParse =
-    dyn Fn(&[String]) -> Result<Option<String>, ParseError> + Send + Sync;
+    dyn Fn(&[ImmutableString]) -> Result<Option<ImmutableString>, ParseError> + Send + Sync;
 
 /// An expression sub-tree in an [`AST`][crate::AST].
 #[derive(Debug, Clone)]
@@ -100,7 +101,7 @@ impl Engine {
     /// * `keywords` holds a slice of strings that define the custom syntax.  
     /// * `new_vars` is the number of new variables declared by this custom syntax, or the number of variables removed (if negative).
     /// * `func` is the implementation function.
-    pub fn register_custom_syntax<S: AsRef<str> + ToString>(
+    pub fn register_custom_syntax<S: AsRef<str> + Into<ImmutableString>>(
         &mut self,
         keywords: impl AsRef<[S]>,
         new_vars: isize,
@@ -110,7 +111,7 @@ impl Engine {
     ) -> Result<&mut Self, ParseError> {
         let keywords = keywords.as_ref();
 
-        let mut segments: StaticVec<_> = Default::default();
+        let mut segments: StaticVec<ImmutableString> = Default::default();
 
         for s in keywords {
             let s = s.as_ref().trim();
@@ -122,7 +123,7 @@ impl Engine {
 
             let seg = match s {
                 // Markers not in first position
-                MARKER_EXPR | MARKER_BLOCK | MARKER_IDENT if !segments.is_empty() => s.to_string(),
+                MARKER_IDENT | MARKER_EXPR | MARKER_BLOCK if !segments.is_empty() => s.into(),
                 // Standard or reserved keyword/symbol not in first position
                 s if !segments.is_empty() && Token::lookup_from_syntax(s).is_some() => {
                     // Make it a custom keyword/symbol
@@ -212,7 +213,9 @@ impl Engine {
     pub fn register_custom_syntax_raw(
         &mut self,
         key: impl Into<ImmutableString>,
-        parse: impl Fn(&[String]) -> Result<Option<String>, ParseError> + SendSync + 'static,
+        parse: impl Fn(&[ImmutableString]) -> Result<Option<ImmutableString>, ParseError>
+            + SendSync
+            + 'static,
         new_vars: isize,
         func: impl Fn(&mut EvalContext, &[Expression]) -> Result<Dynamic, Box<EvalAltResult>>
             + SendSync

tests/syntax.rs

@@ -95,7 +95,7 @@ fn test_custom_syntax_raw() -> Result<(), Box<EvalAltResult>> {
         "hello",
         |stream| match stream.len() {
             0 => unreachable!(),
-            1 => Ok(Some("$ident$".to_string())),
+            1 => Ok(Some("$ident$".into())),
             2 => match stream[1].as_str() {
                 "world" | "kitty" => Ok(None),
                 s => Err(ParseError(