From 719f0babbf94638c3dccf1547c577fb6f71d2fb5 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Fri, 18 Dec 2020 16:07:19 +0800 Subject: [PATCH] Fix comments parsing. --- RELEASES.md | 1 + src/token.rs | 73 +++++++++++++++++++++++++++-------------------- tests/comments.rs | 57 ++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 31 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index d442c231..bc08d9dc 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -16,6 +16,7 @@ Bug fixes * Constants are no longer propagated by the optimizer if shadowed by a non-constant variable. * Constants passed as the `this` parameter to Rhai functions now throws an error if assigned to. * Generic type parameter of `Engine::register_iterator` is `IntoIterator` instead of `Iterator`. +* Fixes parsing of block comments ending with `**/` or inner blocks starting with `//*`. Breaking changes ---------------- diff --git a/src/token.rs b/src/token.rs index f1c00d14..e6f3588a 100644 --- a/src/token.rs +++ b/src/token.rs @@ -918,41 +918,41 @@ fn eat_next(stream: &mut impl InputStream, pos: &mut Position) -> Option { } /// Scan for a block comment until the end. -fn scan_comment( +fn scan_block_comment( stream: &mut impl InputStream, mut level: usize, pos: &mut Position, - comment: &mut String, + comment: &mut Option, ) -> usize { while let Some(c) = stream.get_next() { pos.advance(); - if !comment.is_empty() { + if let Some(ref mut comment) = comment { comment.push(c); } match c { '/' => { - if let Some(c2) = stream.get_next() { - if !comment.is_empty() { - comment.push(c2); - } + if let Some(c2) = stream.peek_next() { if c2 == '*' { + eat_next(stream, pos); + if let Some(ref mut comment) = comment { + comment.push(c2); + } level += 1; } } - pos.advance(); } '*' => { - if let Some(c2) = stream.get_next() { - if !comment.is_empty() { - comment.push(c2); - } + if let Some(c2) = stream.peek_next() { if c2 == '/' { + eat_next(stream, pos); + if let Some(ref mut comment) = comment { + comment.push(c2); + } level -= 1; } } - pos.advance(); } '\n' => pos.new_line(), _ => (), @@ -1032,11 +1032,16 @@ fn get_next_token_inner( // Still inside a comment? if state.comment_level > 0 { let start_pos = *pos; - let mut comment = String::new(); - state.comment_level = scan_comment(stream, state.comment_level, pos, &mut comment); + let mut comment = if state.include_comments { + Some(String::new()) + } else { + None + }; - if state.include_comments || is_doc_comment(&comment) { - return Some((Token::Comment(comment), start_pos)); + state.comment_level = scan_block_comment(stream, state.comment_level, pos, &mut comment); + + if state.include_comments || is_doc_comment(comment.as_ref().unwrap()) { + return Some((Token::Comment(comment.unwrap()), start_pos)); } } @@ -1282,10 +1287,13 @@ fn get_next_token_inner( ('/', '/') => { eat_next(stream, pos); - let mut comment = match stream.peek_next().unwrap() { - '/' => "///".to_string(), - _ if state.include_comments => "//".to_string(), - _ => String::new(), + let mut comment = match stream.peek_next() { + Some('/') => { + eat_next(stream, pos); + Some("///".to_string()) + } + _ if state.include_comments => Some("//".to_string()), + _ => None, }; while let Some(c) = stream.get_next() { @@ -1293,30 +1301,33 @@ fn get_next_token_inner( pos.new_line(); break; } - - if !comment.is_empty() { + if let Some(ref mut comment) = comment { comment.push(c); } pos.advance(); } - if state.include_comments || is_doc_comment(&comment) { + if let Some(comment) = comment { return Some((Token::Comment(comment), start_pos)); } } ('/', '*') => { state.comment_level = 1; - eat_next(stream, pos); - let mut comment = match stream.peek_next().unwrap() { - '*' => "/**".to_string(), - _ if state.include_comments => "/*".to_string(), - _ => String::new(), + let mut comment = match stream.peek_next() { + Some('*') => { + eat_next(stream, pos); + Some("/**".to_string()) + } + _ if state.include_comments => Some("/*".to_string()), + _ => None, }; - state.comment_level = scan_comment(stream, state.comment_level, pos, &mut comment); - if state.include_comments || is_doc_comment(&comment) { + state.comment_level = + scan_block_comment(stream, state.comment_level, pos, &mut comment); + + if let Some(comment) = comment { return Some((Token::Comment(comment), start_pos)); } } diff --git a/tests/comments.rs b/tests/comments.rs index d7525960..17746a94 100644 --- a/tests/comments.rs +++ b/tests/comments.rs @@ -21,5 +21,62 @@ fn test_comments() -> Result<(), Box> { 42 ); + assert_eq!(engine.eval::<()>("/* Hello world */")?, ()); + + Ok(()) +} + +#[cfg(not(feature = "no_function"))] +#[test] +fn test_comments_doc() -> Result<(), Box> { + let engine = Engine::new(); + + let ast = engine.compile( + r" + /// Hello world + + + fn foo() {} + ", + )?; + + assert_eq!( + ast.iter_functions().next().unwrap().comments[0], + "/// Hello world" + ); + + assert!(engine + .compile( + r" + /// Hello world + let x = 42; + " + ) + .is_err()); + + let ast = engine.compile( + r" + /** Hello world + ** how are you? + **/ + + fn foo() {} + ", + )?; + + assert_eq!( + ast.iter_functions().next().unwrap().comments[0], + "/** Hello world\n ** how are you?\n **/" + ); + + assert!(engine + .compile( + r" + /** Hello world */ + let x = 42; + " + ) + .is_err()); + Ok(()) }