diff --git a/src/ast.rs b/src/ast.rs index d7b50786..074021bf 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -1496,10 +1496,10 @@ impl crate::stdlib::ops::DerefMut for FloatWrapper { } #[cfg(not(feature = "no_float"))] -impl fmt::Debug for FloatWrapper { +impl fmt::Debug for FloatWrapper { #[inline(always)] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.0, f) + fmt::Debug::fmt(&self.0, f) } } @@ -1641,10 +1641,10 @@ impl fmt::Debug for Expr { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::DynamicConstant(value, pos) => write!(f, "{:?} @ {:?}", value, pos), - Self::BoolConstant(value, pos) => write!(f, "{} @ {:?}", value, pos), - Self::IntegerConstant(value, pos) => write!(f, "{} @ {:?}", value, pos), + Self::BoolConstant(value, pos) => write!(f, "{:?} @ {:?}", value, pos), + Self::IntegerConstant(value, pos) => write!(f, "{:?} @ {:?}", value, pos), #[cfg(not(feature = "no_float"))] - Self::FloatConstant(value, pos) => write!(f, "{} @ {:?}", value, pos), + Self::FloatConstant(value, pos) => write!(f, "{:?} @ {:?}", value, pos), Self::CharConstant(value, pos) => write!(f, "{:?} @ {:?}", value, pos), Self::StringConstant(value, pos) => write!(f, "{:?} @ {:?}", value, pos), Self::FnPointer(value, pos) => write!(f, "Fn({:?}) @ {:?}", value, pos), diff --git a/src/token.rs b/src/token.rs index 197ed303..34c00034 100644 --- a/src/token.rs +++ b/src/token.rs @@ -833,8 +833,6 @@ pub struct TokenizeState { pub non_unary: bool, /// Is the tokenizer currently inside a block comment? pub comment_level: usize, - /// Return [`None`] at the end of the stream instead of [`Some(Token::EOF)`][Token::EOF]? - pub end_with_none: bool, /// Include comments? pub include_comments: bool, /// Disable doc-comments? @@ -882,7 +880,8 @@ pub trait InputStream { /// |`` `hello``_{EOF}_ |`StringConstant("hello")` |``Some('`')`` | /// |`` `hello``_{LF}{EOF}_ |`StringConstant("hello\n")` |``Some('`')`` | /// |`` `hello ${`` |`InterpolatedString("hello ")`
next token is `{`|`None` | -/// |`` } hello` `` |`StringConstant(" hello")` |``Some('`')`` | +/// |`` } hello` `` |`StringConstant(" hello")` |`None` | +/// |`} hello`_{EOF}_ |`StringConstant(" hello")` |``Some('`')`` | pub fn parse_string_literal( stream: &mut impl InputStream, state: &mut TokenizeState, @@ -902,23 +901,32 @@ pub fn parse_string_literal( state.is_within_text_terminated_by = Some(termination_char); loop { + assert!( + !verbatim || escape.is_empty(), + "verbatim strings should not have any escapes" + ); + let next_char = match stream.get_next() { Some(ch) => { pos.advance(); ch } - None if !continuation && !verbatim => { - pos.advance(); - state.is_within_text_terminated_by = None; - return Err((LERR::UnterminatedString, *pos)); - } - None => { - if verbatim || escape != "\\" { - result += &escape; - } + None if verbatim => { + assert_eq!(escape, "", "verbatim strings should not have any escapes"); pos.advance(); break; } + None if continuation && !escape.is_empty() => { + assert_eq!(escape, "\\", "unexpected escape {} at end of line", escape); + pos.advance(); + break; + } + None => { + result += &escape; + pos.advance(); + state.is_within_text_terminated_by = None; + return Err((LERR::UnterminatedString, start)); + } }; // String interpolation? @@ -942,7 +950,7 @@ pub fn parse_string_literal( // \r - ignore if followed by \n '\r' if stream.peek_next().map(|ch| ch == '\n').unwrap_or(false) => {} // \... - '\\' if escape.is_empty() && !verbatim => { + '\\' if !verbatim && escape.is_empty() => { escape.push('\\'); } // \\ @@ -1011,24 +1019,26 @@ pub fn parse_string_literal( break; } + // Verbatim + '\n' if verbatim => { + assert_eq!(escape, "", "verbatim strings should not have any escapes"); + pos.new_line(); + result.push(next_char); + } + // Line continuation '\n' if continuation && !escape.is_empty() => { + assert_eq!(escape, "\\", "unexpected escape {} at end of line", escape); escape.clear(); pos.new_line(); skip_whitespace_until = start.position().unwrap() + 1; } - // New-line cannot be escaped - // Cannot have new-lines inside non-multi-line string literals - '\n' if !escape.is_empty() || !verbatim => { + // Unterminated string + '\n' => { pos.rewind(); state.is_within_text_terminated_by = None; - return Err((LERR::UnterminatedString, *pos)); - } - - '\n' => { - pos.new_line(); - result.push(next_char); + return Err((LERR::UnterminatedString, start)); } // Unknown escape sequence @@ -1746,11 +1756,7 @@ fn get_next_token_inner( pos.advance(); - if state.end_with_none { - None - } else { - Some((Token::EOF, *pos)) - } + Some((Token::EOF, *pos)) } /// Get the next identifier. @@ -1941,9 +1947,10 @@ impl<'a> Iterator for TokenIterator<'a> { let (token, pos) = match get_next_token(&mut self.stream, &mut self.state, &mut self.pos) { // {EOF} None => return None, - // Unterminated string at EOF - Some((Token::StringConstant(_), _)) if self.state.is_within_text_terminated_by.is_some() => { - return Some((Token::LexError(LERR::UnterminatedString), self.pos)); + // {EOF} after unterminated string + Some((Token::StringConstant(_), pos)) if self.state.is_within_text_terminated_by.is_some() => { + self.state.is_within_text_terminated_by = None; + return Some((Token::LexError(LERR::UnterminatedString), pos)); } // Reserved keyword/symbol Some((Token::Reserved(s), pos)) => (match @@ -2064,7 +2071,6 @@ impl Engine { max_string_size: None, non_unary: false, comment_level: 0, - end_with_none: false, include_comments: false, #[cfg(not(feature = "no_function"))] #[cfg(feature = "metadata")]