kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Include actual tokens in custom syntax node.

Browse Source
This commit is contained in:
Stephen Chung 2020-12-13 14:31:24 +08:00
parent 87174de051
commit ecc08271d9
6 changed files with 70 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
RELEASES.md
View File

@ -4,6 +4,12 @@ Rhai Release Notes
Version 0.19.8
==============
This version makes it easier to generate documentation for a Rhai code base.
Each function defined in an `AST` can optionally attach _doc-comments_ (which, as in Rust,
are comments prefixed by either `///` or `/**`). Doc-comments allow third-party tools to
automatically generate documentation for functions defined in a Rhai script.
Bug fixes
---------
@ -17,6 +23,11 @@ Breaking changes
* The closure for `Engine::on_debug` now takes an additional `Position` parameter.
* `AST::iter_functions` now returns `ScriptFnMetadata`.
New features
------------
* `AST::iter_functions` now returns `ScriptFnMetadata` which includes, among others, _doc-comments_ for functions prefixed by `///` or `/**`.
Enhancements
------------

23
src/ast.rs
View File

@ -857,29 +857,40 @@ impl Stmt {
/// This type is volatile and may change.
#[derive(Clone)]
pub struct CustomExpr {
/// List of keywords.
pub(crate) keywords: StaticVec<Expr>,
/// Implementation function.
pub(crate) func: Shared<FnCustomSyntaxEval>,
/// List of keywords.
pub(crate) keywords: StaticVec<Expr>,
/// List of tokens actually parsed.
pub(crate) tokens: Vec<ImmutableString>,
}
impl fmt::Debug for CustomExpr {
#[inline(always)]
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Debug::fmt(&self.keywords, f)
f.write_str("CustomExpr { keywords:")?;
fmt::Debug::fmt(&self.keywords, f)?;
f.write_str(", tokens:")?;
fmt::Debug::fmt(&self.tokens, f)?;
f.write_str("}")
}
}
impl CustomExpr {
/// Get the implementation function for this custom syntax.
#[inline(always)]
pub fn func(&self) -> &FnCustomSyntaxEval {
self.func.as_ref()
}
/// Get the keywords for this custom syntax.
#[inline(always)]
pub fn keywords(&self) -> &[Expr] {
&self.keywords
}
/// Get the implementation function for this custom syntax.
/// Get the actual parsed tokens for this custom syntax.
#[inline(always)]
pub fn func(&self) -> &FnCustomSyntaxEval {
self.func.as_ref()
pub fn tokens(&self) -> &[ImmutableString] {
&self.tokens
}
}

3
src/engine.rs
View File

@ -195,9 +195,6 @@ pub const FN_IDX_SET: &str = "index$set$";
pub const FN_ANONYMOUS: &str = "anon$";
#[cfg(any(not(feature = "no_index"), not(feature = "no_object")))]
pub const OP_EQUALS: &str = "==";
pub const MARKER_EXPR: &str = "$expr$";
pub const MARKER_BLOCK: &str = "$block$";
pub const MARKER_IDENT: &str = "$ident$";
/// A type specifying the method of chaining.
#[cfg(any(not(feature = "no_index"), not(feature = "no_object")))]

42
src/parser.rs
View File

@ -4,7 +4,7 @@ use crate::ast::{
BinaryExpr, CustomExpr, Expr, FnCallExpr, Ident, IdentX, ReturnType, ScriptFnDef, Stmt,
};
use crate::dynamic::{AccessMode, Union};
use crate::engine::{KEYWORD_THIS, MARKER_BLOCK, MARKER_EXPR, MARKER_IDENT};
use crate::engine::KEYWORD_THIS;
use crate::module::NamespaceRef;
use crate::optimize::optimize_into_ast;
use crate::optimize::OptimizationLevel;
@ -20,7 +20,7 @@ use crate::stdlib::{
vec,
vec::Vec,
};
use crate::syntax::CustomSyntax;
use crate::syntax::{CustomSyntax, MARKER_BLOCK, MARKER_EXPR, MARKER_IDENT};
use crate::token::{is_doc_comment, is_keyword_function, is_valid_identifier, Token, TokenStream};
use crate::utils::{get_hasher, StraightHasherBuilder};
use crate::{
@ -1803,7 +1803,9 @@ fn parse_custom_syntax(
syntax: &CustomSyntax,
pos: Position,
) -> Result<Expr, ParseError> {
let mut exprs: StaticVec<Expr> = Default::default();
let mut keywords: StaticVec<Expr> = Default::default();
let mut segments: StaticVec<_> = Default::default();
let mut tokens: Vec<_> = Default::default();
// Adjust the variables stack
match syntax.scope_delta {
@ -1825,26 +1827,28 @@ fn parse_custom_syntax(
let parse_func = &syntax.parse;
let mut segments: StaticVec<_> = Default::default();
segments.push(key.to_string());
segments.push(key.into());
tokens.push(key.into());
loop {
settings.pos = input.peek().unwrap().1;
let settings = settings.level_up();
let token =
let required_token =
if let Some(seg) = parse_func(&segments).map_err(|err| err.0.into_err(settings.pos))? {
seg
} else {
break;
};
match token.as_str() {
match required_token.as_str() {
MARKER_IDENT => match input.next().unwrap() {
(Token::Identifier(s), pos) => {
segments.push(s.clone());
let var_name_def = IdentX::new(state.get_interned_string(s), pos);
exprs.push(Expr::Variable(Box::new((None, None, 0, var_name_def))));
let ident = state.get_interned_string(s);
let var_name_def = IdentX::new(ident.clone(), pos);
keywords.push(Expr::Variable(Box::new((None, None, 0, var_name_def))));
segments.push(ident);
tokens.push(state.get_interned_string(MARKER_IDENT));
}
(Token::Reserved(s), pos) if is_valid_identifier(s.chars()) => {
return Err(PERR::Reserved(s).into_err(pos));
@ -1852,20 +1856,25 @@ fn parse_custom_syntax(
(_, pos) => return Err(PERR::VariableExpected.into_err(pos)),
},
MARKER_EXPR => {
exprs.push(parse_expr(input, state, lib, settings)?);
segments.push(MARKER_EXPR.into());
keywords.push(parse_expr(input, state, lib, settings)?);
let keyword = state.get_interned_string(MARKER_EXPR);
segments.push(keyword.clone());
tokens.push(keyword);
}
MARKER_BLOCK => match parse_block(input, state, lib, settings)? {
Stmt::Block(statements, pos) => {
exprs.push(Expr::Stmt(Box::new(statements.into()), pos));
segments.push(MARKER_BLOCK.into());
keywords.push(Expr::Stmt(Box::new(statements.into()), pos));
let keyword = state.get_interned_string(MARKER_BLOCK);
segments.push(keyword.clone());
tokens.push(keyword);
}
_ => unreachable!(),
},
s => match input.next().unwrap() {
(Token::LexError(err), pos) => return Err(err.into_err(pos)),
(t, _) if t.syntax().as_ref() == s => {
segments.push(t.syntax().into_owned());
segments.push(required_token.clone());
tokens.push(required_token.clone());
}
(_, pos) => {
return Err(PERR::MissingToken(
@ -1880,8 +1889,9 @@ fn parse_custom_syntax(
Ok(Expr::Custom(
Box::new(CustomExpr {
keywords: exprs,
keywords,
func: syntax.func.clone(),
tokens,
}),
pos,
))

27
src/syntax.rs
View File

@ -1,19 +1,19 @@
//! Module implementing custom syntax for [`Engine`].
use crate::ast::Expr;
use crate::engine::{EvalContext, MARKER_BLOCK, MARKER_EXPR, MARKER_IDENT};
use crate::engine::EvalContext;
use crate::fn_native::SendSync;
use crate::stdlib::{
boxed::Box,
format,
string::{String, ToString},
};
use crate::stdlib::{boxed::Box, format, string::ToString};
use crate::token::{is_valid_identifier, Token};
use crate::{
Dynamic, Engine, EvalAltResult, ImmutableString, LexError, ParseError, Position, Shared,
StaticVec,
};
pub const MARKER_EXPR: &str = "$expr$";
pub const MARKER_BLOCK: &str = "$block$";
pub const MARKER_IDENT: &str = "$ident$";
/// A general expression evaluation trait object.
#[cfg(not(feature = "sync"))]
pub type FnCustomSyntaxEval =
@ -25,11 +25,12 @@ pub type FnCustomSyntaxEval =
/// A general expression parsing trait object.
#[cfg(not(feature = "sync"))]
pub type FnCustomSyntaxParse = dyn Fn(&[String]) -> Result<Option<String>, ParseError>;
pub type FnCustomSyntaxParse =
dyn Fn(&[ImmutableString]) -> Result<Option<ImmutableString>, ParseError>;
/// A general expression parsing trait object.
#[cfg(feature = "sync")]
pub type FnCustomSyntaxParse =
dyn Fn(&[String]) -> Result<Option<String>, ParseError> + Send + Sync;
dyn Fn(&[ImmutableString]) -> Result<Option<ImmutableString>, ParseError> + Send + Sync;
/// An expression sub-tree in an [`AST`][crate::AST].
#[derive(Debug, Clone)]
@ -100,7 +101,7 @@ impl Engine {
/// * `keywords` holds a slice of strings that define the custom syntax.
/// * `new_vars` is the number of new variables declared by this custom syntax, or the number of variables removed (if negative).
/// * `func` is the implementation function.
pub fn register_custom_syntax<S: AsRef<str> + ToString>(
pub fn register_custom_syntax<S: AsRef<str> + Into<ImmutableString>>(
&mut self,
keywords: impl AsRef<[S]>,
new_vars: isize,
@ -110,7 +111,7 @@ impl Engine {
) -> Result<&mut Self, ParseError> {
let keywords = keywords.as_ref();
let mut segments: StaticVec<_> = Default::default();
let mut segments: StaticVec<ImmutableString> = Default::default();
for s in keywords {
let s = s.as_ref().trim();
@ -122,7 +123,7 @@ impl Engine {
let seg = match s {
// Markers not in first position
MARKER_EXPR | MARKER_BLOCK | MARKER_IDENT if !segments.is_empty() => s.to_string(),
MARKER_IDENT | MARKER_EXPR | MARKER_BLOCK if !segments.is_empty() => s.into(),
// Standard or reserved keyword/symbol not in first position
s if !segments.is_empty() && Token::lookup_from_syntax(s).is_some() => {
// Make it a custom keyword/symbol
@ -212,7 +213,9 @@ impl Engine {
pub fn register_custom_syntax_raw(
&mut self,
key: impl Into<ImmutableString>,
parse: impl Fn(&[String]) -> Result<Option<String>, ParseError> + SendSync + 'static,
parse: impl Fn(&[ImmutableString]) -> Result<Option<ImmutableString>, ParseError>
+ SendSync
+ 'static,
new_vars: isize,
func: impl Fn(&mut EvalContext, &[Expression]) -> Result<Dynamic, Box<EvalAltResult>>
+ SendSync

2
tests/syntax.rs
View File

@ -95,7 +95,7 @@ fn test_custom_syntax_raw() -> Result<(), Box<EvalAltResult>> {
"hello",
|stream| match stream.len() {
0 => unreachable!(),
1 => Ok(Some("$ident$".to_string())),
1 => Ok(Some("$ident$".into())),
2 => match stream[1].as_str() {
"world" | "kitty" => Ok(None),
s => Err(ParseError(