From ecc08271d97faa25aa53a037ab17ea16ae8c6f74 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Sun, 13 Dec 2020 14:31:24 +0800 Subject: [PATCH] Include actual tokens in custom syntax node. --- RELEASES.md | 11 +++++++++++ src/ast.rs | 23 +++++++++++++++++------ src/engine.rs | 3 --- src/parser.rs | 42 ++++++++++++++++++++++++++---------------- src/syntax.rs | 27 +++++++++++++++------------ tests/syntax.rs | 2 +- 6 files changed, 70 insertions(+), 38 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index 8d2baddb..fc7cedf4 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -4,6 +4,12 @@ Rhai Release Notes Version 0.19.8 ============== +This version makes it easier to generate documentation for a Rhai code base. + +Each function defined in an `AST` can optionally attach _doc-comments_ (which, as in Rust, +are comments prefixed by either `///` or `/**`). Doc-comments allow third-party tools to +automatically generate documentation for functions defined in a Rhai script. + Bug fixes --------- @@ -17,6 +23,11 @@ Breaking changes * The closure for `Engine::on_debug` now takes an additional `Position` parameter. * `AST::iter_functions` now returns `ScriptFnMetadata`. +New features +------------ + +* `AST::iter_functions` now returns `ScriptFnMetadata` which includes, among others, _doc-comments_ for functions prefixed by `///` or `/**`. + Enhancements ------------ diff --git a/src/ast.rs b/src/ast.rs index 0ecf5e76..1fc091fe 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -857,29 +857,40 @@ impl Stmt { /// This type is volatile and may change. #[derive(Clone)] pub struct CustomExpr { - /// List of keywords. - pub(crate) keywords: StaticVec, /// Implementation function. pub(crate) func: Shared, + /// List of keywords. + pub(crate) keywords: StaticVec, + /// List of tokens actually parsed. + pub(crate) tokens: Vec, } impl fmt::Debug for CustomExpr { #[inline(always)] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&self.keywords, f) + f.write_str("CustomExpr { keywords:")?; + fmt::Debug::fmt(&self.keywords, f)?; + f.write_str(", tokens:")?; + fmt::Debug::fmt(&self.tokens, f)?; + f.write_str("}") } } impl CustomExpr { + /// Get the implementation function for this custom syntax. + #[inline(always)] + pub fn func(&self) -> &FnCustomSyntaxEval { + self.func.as_ref() + } /// Get the keywords for this custom syntax. #[inline(always)] pub fn keywords(&self) -> &[Expr] { &self.keywords } - /// Get the implementation function for this custom syntax. + /// Get the actual parsed tokens for this custom syntax. #[inline(always)] - pub fn func(&self) -> &FnCustomSyntaxEval { - self.func.as_ref() + pub fn tokens(&self) -> &[ImmutableString] { + &self.tokens } } diff --git a/src/engine.rs b/src/engine.rs index 958cf1d9..0fad8de5 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -195,9 +195,6 @@ pub const FN_IDX_SET: &str = "index$set$"; pub const FN_ANONYMOUS: &str = "anon$"; #[cfg(any(not(feature = "no_index"), not(feature = "no_object")))] pub const OP_EQUALS: &str = "=="; -pub const MARKER_EXPR: &str = "$expr$"; -pub const MARKER_BLOCK: &str = "$block$"; -pub const MARKER_IDENT: &str = "$ident$"; /// A type specifying the method of chaining. #[cfg(any(not(feature = "no_index"), not(feature = "no_object")))] diff --git a/src/parser.rs b/src/parser.rs index d05e1042..e3b275da 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -4,7 +4,7 @@ use crate::ast::{ BinaryExpr, CustomExpr, Expr, FnCallExpr, Ident, IdentX, ReturnType, ScriptFnDef, Stmt, }; use crate::dynamic::{AccessMode, Union}; -use crate::engine::{KEYWORD_THIS, MARKER_BLOCK, MARKER_EXPR, MARKER_IDENT}; +use crate::engine::KEYWORD_THIS; use crate::module::NamespaceRef; use crate::optimize::optimize_into_ast; use crate::optimize::OptimizationLevel; @@ -20,7 +20,7 @@ use crate::stdlib::{ vec, vec::Vec, }; -use crate::syntax::CustomSyntax; +use crate::syntax::{CustomSyntax, MARKER_BLOCK, MARKER_EXPR, MARKER_IDENT}; use crate::token::{is_doc_comment, is_keyword_function, is_valid_identifier, Token, TokenStream}; use crate::utils::{get_hasher, StraightHasherBuilder}; use crate::{ @@ -1803,7 +1803,9 @@ fn parse_custom_syntax( syntax: &CustomSyntax, pos: Position, ) -> Result { - let mut exprs: StaticVec = Default::default(); + let mut keywords: StaticVec = Default::default(); + let mut segments: StaticVec<_> = Default::default(); + let mut tokens: Vec<_> = Default::default(); // Adjust the variables stack match syntax.scope_delta { @@ -1825,26 +1827,28 @@ fn parse_custom_syntax( let parse_func = &syntax.parse; - let mut segments: StaticVec<_> = Default::default(); - segments.push(key.to_string()); + segments.push(key.into()); + tokens.push(key.into()); loop { settings.pos = input.peek().unwrap().1; let settings = settings.level_up(); - let token = + let required_token = if let Some(seg) = parse_func(&segments).map_err(|err| err.0.into_err(settings.pos))? { seg } else { break; }; - match token.as_str() { + match required_token.as_str() { MARKER_IDENT => match input.next().unwrap() { (Token::Identifier(s), pos) => { - segments.push(s.clone()); - let var_name_def = IdentX::new(state.get_interned_string(s), pos); - exprs.push(Expr::Variable(Box::new((None, None, 0, var_name_def)))); + let ident = state.get_interned_string(s); + let var_name_def = IdentX::new(ident.clone(), pos); + keywords.push(Expr::Variable(Box::new((None, None, 0, var_name_def)))); + segments.push(ident); + tokens.push(state.get_interned_string(MARKER_IDENT)); } (Token::Reserved(s), pos) if is_valid_identifier(s.chars()) => { return Err(PERR::Reserved(s).into_err(pos)); @@ -1852,20 +1856,25 @@ fn parse_custom_syntax( (_, pos) => return Err(PERR::VariableExpected.into_err(pos)), }, MARKER_EXPR => { - exprs.push(parse_expr(input, state, lib, settings)?); - segments.push(MARKER_EXPR.into()); + keywords.push(parse_expr(input, state, lib, settings)?); + let keyword = state.get_interned_string(MARKER_EXPR); + segments.push(keyword.clone()); + tokens.push(keyword); } MARKER_BLOCK => match parse_block(input, state, lib, settings)? { Stmt::Block(statements, pos) => { - exprs.push(Expr::Stmt(Box::new(statements.into()), pos)); - segments.push(MARKER_BLOCK.into()); + keywords.push(Expr::Stmt(Box::new(statements.into()), pos)); + let keyword = state.get_interned_string(MARKER_BLOCK); + segments.push(keyword.clone()); + tokens.push(keyword); } _ => unreachable!(), }, s => match input.next().unwrap() { (Token::LexError(err), pos) => return Err(err.into_err(pos)), (t, _) if t.syntax().as_ref() == s => { - segments.push(t.syntax().into_owned()); + segments.push(required_token.clone()); + tokens.push(required_token.clone()); } (_, pos) => { return Err(PERR::MissingToken( @@ -1880,8 +1889,9 @@ fn parse_custom_syntax( Ok(Expr::Custom( Box::new(CustomExpr { - keywords: exprs, + keywords, func: syntax.func.clone(), + tokens, }), pos, )) diff --git a/src/syntax.rs b/src/syntax.rs index ce221146..a8befa39 100644 --- a/src/syntax.rs +++ b/src/syntax.rs @@ -1,19 +1,19 @@ //! Module implementing custom syntax for [`Engine`]. use crate::ast::Expr; -use crate::engine::{EvalContext, MARKER_BLOCK, MARKER_EXPR, MARKER_IDENT}; +use crate::engine::EvalContext; use crate::fn_native::SendSync; -use crate::stdlib::{ - boxed::Box, - format, - string::{String, ToString}, -}; +use crate::stdlib::{boxed::Box, format, string::ToString}; use crate::token::{is_valid_identifier, Token}; use crate::{ Dynamic, Engine, EvalAltResult, ImmutableString, LexError, ParseError, Position, Shared, StaticVec, }; +pub const MARKER_EXPR: &str = "$expr$"; +pub const MARKER_BLOCK: &str = "$block$"; +pub const MARKER_IDENT: &str = "$ident$"; + /// A general expression evaluation trait object. #[cfg(not(feature = "sync"))] pub type FnCustomSyntaxEval = @@ -25,11 +25,12 @@ pub type FnCustomSyntaxEval = /// A general expression parsing trait object. #[cfg(not(feature = "sync"))] -pub type FnCustomSyntaxParse = dyn Fn(&[String]) -> Result, ParseError>; +pub type FnCustomSyntaxParse = + dyn Fn(&[ImmutableString]) -> Result, ParseError>; /// A general expression parsing trait object. #[cfg(feature = "sync")] pub type FnCustomSyntaxParse = - dyn Fn(&[String]) -> Result, ParseError> + Send + Sync; + dyn Fn(&[ImmutableString]) -> Result, ParseError> + Send + Sync; /// An expression sub-tree in an [`AST`][crate::AST]. #[derive(Debug, Clone)] @@ -100,7 +101,7 @@ impl Engine { /// * `keywords` holds a slice of strings that define the custom syntax. /// * `new_vars` is the number of new variables declared by this custom syntax, or the number of variables removed (if negative). /// * `func` is the implementation function. - pub fn register_custom_syntax + ToString>( + pub fn register_custom_syntax + Into>( &mut self, keywords: impl AsRef<[S]>, new_vars: isize, @@ -110,7 +111,7 @@ impl Engine { ) -> Result<&mut Self, ParseError> { let keywords = keywords.as_ref(); - let mut segments: StaticVec<_> = Default::default(); + let mut segments: StaticVec = Default::default(); for s in keywords { let s = s.as_ref().trim(); @@ -122,7 +123,7 @@ impl Engine { let seg = match s { // Markers not in first position - MARKER_EXPR | MARKER_BLOCK | MARKER_IDENT if !segments.is_empty() => s.to_string(), + MARKER_IDENT | MARKER_EXPR | MARKER_BLOCK if !segments.is_empty() => s.into(), // Standard or reserved keyword/symbol not in first position s if !segments.is_empty() && Token::lookup_from_syntax(s).is_some() => { // Make it a custom keyword/symbol @@ -212,7 +213,9 @@ impl Engine { pub fn register_custom_syntax_raw( &mut self, key: impl Into, - parse: impl Fn(&[String]) -> Result, ParseError> + SendSync + 'static, + parse: impl Fn(&[ImmutableString]) -> Result, ParseError> + + SendSync + + 'static, new_vars: isize, func: impl Fn(&mut EvalContext, &[Expression]) -> Result> + SendSync diff --git a/tests/syntax.rs b/tests/syntax.rs index 526ebf92..04cfaabf 100644 --- a/tests/syntax.rs +++ b/tests/syntax.rs @@ -95,7 +95,7 @@ fn test_custom_syntax_raw() -> Result<(), Box> { "hello", |stream| match stream.len() { 0 => unreachable!(), - 1 => Ok(Some("$ident$".to_string())), + 1 => Ok(Some("$ident$".into())), 2 => match stream[1].as_str() { "world" | "kitty" => Ok(None), s => Err(ParseError(