From c1c1e5f6842368a7255f0108abfceaa1399f97d6 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Sun, 11 Apr 2021 11:19:46 +0800 Subject: [PATCH 1/3] Use string start position as error position. --- src/token.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/token.rs b/src/token.rs index 197ed303..0c7208a0 100644 --- a/src/token.rs +++ b/src/token.rs @@ -910,7 +910,7 @@ pub fn parse_string_literal( None if !continuation && !verbatim => { pos.advance(); state.is_within_text_terminated_by = None; - return Err((LERR::UnterminatedString, *pos)); + return Err((LERR::UnterminatedString, start)); } None => { if verbatim || escape != "\\" { @@ -1019,11 +1019,11 @@ pub fn parse_string_literal( } // New-line cannot be escaped - // Cannot have new-lines inside non-multi-line string literals - '\n' if !escape.is_empty() || !verbatim => { + // Cannot have new-lines inside non-verbatim strings + '\n' if !verbatim || !escape.is_empty() => { pos.rewind(); state.is_within_text_terminated_by = None; - return Err((LERR::UnterminatedString, *pos)); + return Err((LERR::UnterminatedString, start)); } '\n' => { @@ -1942,8 +1942,9 @@ impl<'a> Iterator for TokenIterator<'a> { // {EOF} None => return None, // Unterminated string at EOF - Some((Token::StringConstant(_), _)) if self.state.is_within_text_terminated_by.is_some() => { - return Some((Token::LexError(LERR::UnterminatedString), self.pos)); + Some((Token::StringConstant(_), pos)) if self.state.is_within_text_terminated_by.is_some() => { + self.state.is_within_text_terminated_by = None; + return Some((Token::LexError(LERR::UnterminatedString), pos)); } // Reserved keyword/symbol Some((Token::Reserved(s), pos)) => (match From ccbbf3eb346c3d44033fce74c45fe1d99c2de6fc Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Sun, 11 Apr 2021 11:20:34 +0800 Subject: [PATCH 2/3] Enhance floating-point display. --- src/ast.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ast.rs b/src/ast.rs index d7b50786..074021bf 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -1496,10 +1496,10 @@ impl crate::stdlib::ops::DerefMut for FloatWrapper { } #[cfg(not(feature = "no_float"))] -impl fmt::Debug for FloatWrapper { +impl fmt::Debug for FloatWrapper { #[inline(always)] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.0, f) + fmt::Debug::fmt(&self.0, f) } } @@ -1641,10 +1641,10 @@ impl fmt::Debug for Expr { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::DynamicConstant(value, pos) => write!(f, "{:?} @ {:?}", value, pos), - Self::BoolConstant(value, pos) => write!(f, "{} @ {:?}", value, pos), - Self::IntegerConstant(value, pos) => write!(f, "{} @ {:?}", value, pos), + Self::BoolConstant(value, pos) => write!(f, "{:?} @ {:?}", value, pos), + Self::IntegerConstant(value, pos) => write!(f, "{:?} @ {:?}", value, pos), #[cfg(not(feature = "no_float"))] - Self::FloatConstant(value, pos) => write!(f, "{} @ {:?}", value, pos), + Self::FloatConstant(value, pos) => write!(f, "{:?} @ {:?}", value, pos), Self::CharConstant(value, pos) => write!(f, "{:?} @ {:?}", value, pos), Self::StringConstant(value, pos) => write!(f, "{:?} @ {:?}", value, pos), Self::FnPointer(value, pos) => write!(f, "Fn({:?}) @ {:?}", value, pos), From 44e2e6aeb4fdccf72fd38fa7d6bc2cbfdb8f08c5 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Sun, 11 Apr 2021 21:49:03 +0800 Subject: [PATCH 3/3] Fix string handling at EOF. --- src/token.rs | 59 ++++++++++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/src/token.rs b/src/token.rs index 0c7208a0..34c00034 100644 --- a/src/token.rs +++ b/src/token.rs @@ -833,8 +833,6 @@ pub struct TokenizeState { pub non_unary: bool, /// Is the tokenizer currently inside a block comment? pub comment_level: usize, - /// Return [`None`] at the end of the stream instead of [`Some(Token::EOF)`][Token::EOF]? - pub end_with_none: bool, /// Include comments? pub include_comments: bool, /// Disable doc-comments? @@ -882,7 +880,8 @@ pub trait InputStream { /// |`` `hello``_{EOF}_ |`StringConstant("hello")` |``Some('`')`` | /// |`` `hello``_{LF}{EOF}_ |`StringConstant("hello\n")` |``Some('`')`` | /// |`` `hello ${`` |`InterpolatedString("hello ")`
next token is `{`|`None` | -/// |`` } hello` `` |`StringConstant(" hello")` |``Some('`')`` | +/// |`` } hello` `` |`StringConstant(" hello")` |`None` | +/// |`} hello`_{EOF}_ |`StringConstant(" hello")` |``Some('`')`` | pub fn parse_string_literal( stream: &mut impl InputStream, state: &mut TokenizeState, @@ -902,23 +901,32 @@ pub fn parse_string_literal( state.is_within_text_terminated_by = Some(termination_char); loop { + assert!( + !verbatim || escape.is_empty(), + "verbatim strings should not have any escapes" + ); + let next_char = match stream.get_next() { Some(ch) => { pos.advance(); ch } - None if !continuation && !verbatim => { + None if verbatim => { + assert_eq!(escape, "", "verbatim strings should not have any escapes"); + pos.advance(); + break; + } + None if continuation && !escape.is_empty() => { + assert_eq!(escape, "\\", "unexpected escape {} at end of line", escape); + pos.advance(); + break; + } + None => { + result += &escape; pos.advance(); state.is_within_text_terminated_by = None; return Err((LERR::UnterminatedString, start)); } - None => { - if verbatim || escape != "\\" { - result += &escape; - } - pos.advance(); - break; - } }; // String interpolation? @@ -942,7 +950,7 @@ pub fn parse_string_literal( // \r - ignore if followed by \n '\r' if stream.peek_next().map(|ch| ch == '\n').unwrap_or(false) => {} // \... - '\\' if escape.is_empty() && !verbatim => { + '\\' if !verbatim && escape.is_empty() => { escape.push('\\'); } // \\ @@ -1011,26 +1019,28 @@ pub fn parse_string_literal( break; } + // Verbatim + '\n' if verbatim => { + assert_eq!(escape, "", "verbatim strings should not have any escapes"); + pos.new_line(); + result.push(next_char); + } + // Line continuation '\n' if continuation && !escape.is_empty() => { + assert_eq!(escape, "\\", "unexpected escape {} at end of line", escape); escape.clear(); pos.new_line(); skip_whitespace_until = start.position().unwrap() + 1; } - // New-line cannot be escaped - // Cannot have new-lines inside non-verbatim strings - '\n' if !verbatim || !escape.is_empty() => { + // Unterminated string + '\n' => { pos.rewind(); state.is_within_text_terminated_by = None; return Err((LERR::UnterminatedString, start)); } - '\n' => { - pos.new_line(); - result.push(next_char); - } - // Unknown escape sequence _ if !escape.is_empty() => { escape.push(next_char); @@ -1746,11 +1756,7 @@ fn get_next_token_inner( pos.advance(); - if state.end_with_none { - None - } else { - Some((Token::EOF, *pos)) - } + Some((Token::EOF, *pos)) } /// Get the next identifier. @@ -1941,7 +1947,7 @@ impl<'a> Iterator for TokenIterator<'a> { let (token, pos) = match get_next_token(&mut self.stream, &mut self.state, &mut self.pos) { // {EOF} None => return None, - // Unterminated string at EOF + // {EOF} after unterminated string Some((Token::StringConstant(_), pos)) if self.state.is_within_text_terminated_by.is_some() => { self.state.is_within_text_terminated_by = None; return Some((Token::LexError(LERR::UnterminatedString), pos)); @@ -2065,7 +2071,6 @@ impl Engine { max_string_size: None, non_unary: false, comment_level: 0, - end_with_none: false, include_comments: false, #[cfg(not(feature = "no_function"))] #[cfg(feature = "metadata")]