Add doc-comment to function metadata.

src/parse_error.rs

@@ -121,6 +121,10 @@ pub enum ParseErrorType {
     Reserved(String),
     /// Missing an expression. Wrapped value is the expression type.
     ExprExpected(String),
+    /// Defining a doc-comment in an appropriate place (e.g. not at global level).
+    ///
+    /// Never appears under the `no_function` feature.
+    WrongDocComment,
     /// Defining a function `fn` in an appropriate place (e.g. inside another function).
     ///
     /// Never appears under the `no_function` feature.
@@ -189,6 +193,7 @@ impl ParseErrorType {
             Self::FnMissingParams(_) => "Expecting parameters in function declaration",
             Self::FnDuplicatedParam(_,_) => "Duplicated parameters in function declaration",
             Self::FnMissingBody(_) => "Expecting body statement block for function declaration",
+            Self::WrongDocComment => "Doc-comment must be followed immediately by a function definition",
             Self::WrongFnDefinition => "Function definitions must be at global level and cannot be inside a block or another function",
             Self::WrongExport => "Export statement can only appear at global level",
             Self::AssignmentToConstant(_) => "Cannot assign to a constant value",
@@ -243,7 +248,9 @@ impl fmt::Display for ParseErrorType {
             Self::LiteralTooLarge(typ, max) => {
                 write!(f, "{} exceeds the maximum limit ({})", typ, max)
             }
+
             Self::Reserved(s) => write!(f, "'{}' is a reserved keyword", s),
+
             _ => f.write_str(self.desc()),
         }
     }

src/parser.rs

@@ -21,7 +21,7 @@ use crate::stdlib::{
     vec::Vec,
 };
 use crate::syntax::CustomSyntax;
-use crate::token::{is_keyword_function, is_valid_identifier, Token, TokenStream};
+use crate::token::{is_doc_comment, is_keyword_function, is_valid_identifier, Token, TokenStream};
 use crate::utils::{get_hasher, StraightHasherBuilder};
 use crate::{
     calc_script_fn_hash, Dynamic, Engine, ImmutableString, LexError, ParseError, ParseErrorType,
@@ -54,8 +54,6 @@ struct ParseState<'e> {
     /// Tracks a list of external variables (variables that are not explicitly declared in the scope).
     #[cfg(not(feature = "no_closure"))]
     externals: HashMap<ImmutableString, Position>,
-    /// Latest global comments block.
-    comments: Vec<String>,
     /// An indicator that disables variable capturing into externals one single time
     /// up until the nearest consumed Identifier token.
     /// If set to false the next call to `access_var` will not capture the variable.
@@ -100,7 +98,6 @@ impl<'e> ParseState<'e> {
             strings: HashMap::with_capacity(64),
             stack: Vec::with_capacity(16),
             entry_stack_len: 0,
-            comments: Default::default(),
             #[cfg(not(feature = "no_module"))]
             modules: Default::default(),
         }
@@ -2400,6 +2397,37 @@ fn parse_stmt(
 ) -> Result<Option<Stmt>, ParseError> {
     use AccessMode::{ReadOnly, ReadWrite};
 
+    let mut fn_comments: Vec<String> = Default::default();
+    let mut fn_comments_pos = Position::NONE;
+
+    // Handle doc-comment.
+    #[cfg(not(feature = "no_function"))]
+    while let (Token::Comment(ref comment), comment_pos) = input.peek().unwrap() {
+        if fn_comments_pos.is_none() {
+            fn_comments_pos = *comment_pos;
+        }
+
+        if !is_doc_comment(comment) {
+            unreachable!();
+        }
+
+        if !settings.is_global {
+            return Err(PERR::WrongDocComment.into_err(fn_comments_pos));
+        }
+
+        if let Token::Comment(comment) = input.next().unwrap().0 {
+            fn_comments.push(comment);
+
+            match input.peek().unwrap() {
+                (Token::Fn, _) | (Token::Private, _) => break,
+                (Token::Comment(_), _) => (),
+                _ => return Err(PERR::WrongDocComment.into_err(fn_comments_pos)),
+            }
+        } else {
+            unreachable!();
+        }
+    }
+
     let (token, token_pos) = match input.peek().unwrap() {
         (Token::EOF, pos) => return Ok(Some(Stmt::Noop(*pos))),
         x => x,
@@ -2452,8 +2480,6 @@ fn parse_stmt(
                         pos: pos,
                     };
 
-                    let fn_comments = state.comments.clone();
-
                     let func = parse_fn(input, &mut new_state, lib, access, settings, fn_comments)?;
 
                     // Qualifiers (none) + function name + number of arguments.

src/token.rs

@@ -355,6 +355,9 @@ impl Token {
             Custom(s) => s.clone().into(),
             LexError(err) => err.to_string().into(),
 
+            Comment(s) if is_doc_comment(s) => s[..3].to_string().into(),
+            Comment(s) => s[..2].to_string().into(),
+
             token => match token {
                 LeftBrace => "{",
                 RightBrace => "}",
@@ -917,36 +920,36 @@ fn eat_next(stream: &mut impl InputStream, pos: &mut Position) -> Option<char> {
 /// Scan for a block comment until the end.
 fn scan_comment(
     stream: &mut impl InputStream,
-    state: &mut TokenizeState,
+    mut level: usize,
     pos: &mut Position,
     comment: &mut String,
-) {
+) -> usize {
     while let Some(c) = stream.get_next() {
         pos.advance();
 
-        if state.include_comments {
+        if !comment.is_empty() {
             comment.push(c);
         }
 
         match c {
             '/' => {
                 if let Some(c2) = stream.get_next() {
-                    if state.include_comments {
+                    if !comment.is_empty() {
                         comment.push(c2);
                     }
                     if c2 == '*' {
-                        state.comment_level += 1;
+                        level += 1;
                     }
                 }
                 pos.advance();
             }
             '*' => {
                 if let Some(c2) = stream.get_next() {
-                    if state.include_comments {
+                    if !comment.is_empty() {
                         comment.push(c2);
                     }
                     if c2 == '/' {
-                        state.comment_level -= 1;
+                        level -= 1;
                     }
                 }
                 pos.advance();
@@ -955,10 +958,12 @@ fn scan_comment(
             _ => (),
         }
 
-        if state.comment_level == 0 {
+        if level == 0 {
             break;
         }
     }
+
+    level
 }
 
 /// _(INTERNALS)_ Get the next token from the `stream`.
@@ -1012,6 +1017,12 @@ fn is_binary_char(c: char) -> bool {
     }
 }
 
+/// Test if the comment block is a doc-comment.
+#[inline(always)]
+pub fn is_doc_comment(comment: &str) -> bool {
+    comment.starts_with("///") || comment.starts_with("/**")
+}
+
 /// Get the next token.
 fn get_next_token_inner(
     stream: &mut impl InputStream,
@@ -1022,9 +1033,9 @@ fn get_next_token_inner(
     if state.comment_level > 0 {
         let start_pos = *pos;
         let mut comment = String::new();
-        scan_comment(stream, state, pos, &mut comment);
+        state.comment_level = scan_comment(stream, state.comment_level, pos, &mut comment);
 
-        if state.include_comments {
+        if state.include_comments || is_doc_comment(&comment) {
             return Some((Token::Comment(comment), start_pos));
         }
     }
@@ -1271,10 +1282,10 @@ fn get_next_token_inner(
             ('/', '/') => {
                 eat_next(stream, pos);
 
-                let mut comment = if state.include_comments {
+                let mut comment = match stream.peek_next().unwrap() {
-                    "//".to_string()
+                    '/' => "///".to_string(),
-                } else {
+                    _ if state.include_comments => "//".to_string(),
-                    String::new()
+                    _ => String::new(),
                 };
 
                 while let Some(c) = stream.get_next() {
@@ -1283,13 +1294,13 @@ fn get_next_token_inner(
                         break;
                     }
 
-                    if state.include_comments {
+                    if !comment.is_empty() {
                         comment.push(c);
                     }
                     pos.advance();
                 }
 
-                if state.include_comments {
+                if state.include_comments || is_doc_comment(&comment) {
                     return Some((Token::Comment(comment), start_pos));
                 }
             }
@@ -1298,14 +1309,14 @@ fn get_next_token_inner(
 
                 eat_next(stream, pos);
 
-                let mut comment = if state.include_comments {
+                let mut comment = match stream.peek_next().unwrap() {
-                    "/*".to_string()
+                    '*' => "/**".to_string(),
-                } else {
+                    _ if state.include_comments => "/*".to_string(),
-                    String::new()
+                    _ => String::new(),
                 };
-                scan_comment(stream, state, pos, &mut comment);
+                state.comment_level = scan_comment(stream, state.comment_level, pos, &mut comment);
 
-                if state.include_comments {
+                if state.include_comments || is_doc_comment(&comment) {
                     return Some((Token::Comment(comment), start_pos));
                 }
             }