From e36e490a30fe7e08e8064445d6822d9b0c31a4cc Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Tue, 30 Mar 2021 13:59:27 +0800 Subject: [PATCH] Support back-tick at EOL for literal strings. --- src/parser.rs | 4 ++-- src/token.rs | 37 +++++++++++++++++++++++++++++-------- tests/string.rs | 12 ++++++++++++ 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 49c1d57d..4460539d 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -715,13 +715,13 @@ fn parse_map_literal( return Err(PERR::Reserved(s).into_err(pos)); } (Token::LexError(err), pos) => return Err(err.into_err(pos)), - (_, pos) if map.is_empty() => { + (Token::EOF, pos) => { return Err( PERR::MissingToken(Token::RightBrace.into(), MISSING_RBRACE.into()) .into_err(pos), ); } - (Token::EOF, pos) => { + (_, pos) if map.is_empty() => { return Err( PERR::MissingToken(Token::RightBrace.into(), MISSING_RBRACE.into()) .into_err(pos), diff --git a/src/token.rs b/src/token.rs index c5498416..0aea77b5 100644 --- a/src/token.rs +++ b/src/token.rs @@ -1294,15 +1294,36 @@ fn get_next_token_inner( return get_identifier(stream, pos, start_pos, c); } - // " or ` - string literal - ('"', _) | ('`', _) => { - let multi_line = c == '`'; + // " - string literal + ('"', _) => { + return parse_string_literal(stream, state, pos, c, true, false).map_or_else( + |err| Some((Token::LexError(err.0), err.1)), + |out| Some((Token::StringConstant(out), start_pos)), + ); + } + // ` - string literal + ('`', _) => { + // Start from the next line if ` at the end of line + match stream.peek_next() { + // `\r - start from next line + Some('\r') => { + eat_next(stream, pos); + // `\r\n + if stream.peek_next().map(|ch| ch == '\n').unwrap_or(false) { + eat_next(stream, pos); + } + } + // `\n - start from next line + Some('\n') => { + eat_next(stream, pos); + } + _ => (), + } - return parse_string_literal(stream, state, pos, c, !multi_line, multi_line) - .map_or_else( - |err| Some((Token::LexError(err.0), err.1)), - |out| Some((Token::StringConstant(out), start_pos)), - ); + return parse_string_literal(stream, state, pos, c, false, true).map_or_else( + |err| Some((Token::LexError(err.0), err.1)), + |out| Some((Token::StringConstant(out), start_pos)), + ); } // ' - character literal diff --git a/tests/string.rs b/tests/string.rs index d6885a34..2e2d74a7 100644 --- a/tests/string.rs +++ b/tests/string.rs @@ -8,6 +8,10 @@ fn test_string() -> Result<(), Box> { engine.eval::(r#""Test string: \u2764""#)?, "Test string: ❤" ); + assert_eq!( + engine.eval::("\"Test\rstring: \\u2764\"")?, + "Test\rstring: ❤" + ); assert_eq!( engine.eval::(" \"Test string: \\u2764\\\n hello, world!\"")?, "Test string: ❤ hello, world!" @@ -16,6 +20,14 @@ fn test_string() -> Result<(), Box> { engine.eval::(" `Test string: \\u2764\nhello,\\nworld!`")?, "Test string: \\u2764\nhello,\\nworld!" ); + assert_eq!( + engine.eval::(" `\nTest string: \\u2764\nhello,\\nworld!`")?, + "Test string: \\u2764\nhello,\\nworld!" + ); + assert_eq!( + engine.eval::(" `\r\nTest string: \\u2764\nhello,\\nworld!`")?, + "Test string: \\u2764\nhello,\\nworld!" + ); assert_eq!( engine.eval::(r#""Test string: \x58""#)?, "Test string: X"