kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Use SmartString for comments.

Browse Source
This commit is contained in:
Stephen Chung 2022-03-25 08:52:53 +08:00
parent cad4c7a942
commit 77c06b94a2
1 changed files with 14 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/tokenizer.rs
View File

@ -5,7 +5,7 @@ use crate::engine::{
KEYWORD_FN_PTR_CURRY, KEYWORD_IS_DEF_VAR, KEYWORD_PRINT, KEYWORD_THIS, KEYWORD_TYPE_OF, KEYWORD_FN_PTR_CURRY, KEYWORD_IS_DEF_VAR, KEYWORD_PRINT, KEYWORD_THIS, KEYWORD_TYPE_OF,
}; };
use crate::func::native::OnParseTokenCallback; use crate::func::native::OnParseTokenCallback;
use crate::{Engine, LexError, SmartString, StaticVec, INT, UNSIGNED_INT}; use crate::{Engine, Identifier, LexError, SmartString, StaticVec, INT, UNSIGNED_INT};
#[cfg(feature = "no_std")] #[cfg(feature = "no_std")]
use std::prelude::v1::*; use std::prelude::v1::*;
use std::{ use std::{
@ -363,7 +363,7 @@ pub enum Token {
#[cfg(feature = "decimal")] #[cfg(feature = "decimal")]
DecimalConstant(rust_decimal::Decimal), DecimalConstant(rust_decimal::Decimal),
/// An identifier. /// An identifier.
Identifier(SmartString), Identifier(Identifier),
/// A character constant. /// A character constant.
CharConstant(char), CharConstant(char),
/// A string constant. /// A string constant.
@ -1113,8 +1113,8 @@ pub fn parse_string_literal(
allow_line_continuation: bool, allow_line_continuation: bool,
allow_interpolation: bool, allow_interpolation: bool,
) -> Result<(SmartString, bool, Position), (LexError, Position)> { ) -> Result<(SmartString, bool, Position), (LexError, Position)> {
let mut result = SmartString::new(); let mut result = SmartString::new_const();
let mut escape = SmartString::new(); let mut escape = SmartString::new_const();
let start = *pos; let start = *pos;
let mut first_char = Position::NONE; let mut first_char = Position::NONE;
@ -1323,7 +1323,7 @@ fn scan_block_comment(
stream: &mut impl InputStream, stream: &mut impl InputStream,
level: usize, level: usize,
pos: &mut Position, pos: &mut Position,
comment: Option<&mut String>, comment: Option<&mut SmartString>,
) -> usize { ) -> usize {
let mut level = level; let mut level = level;
let mut comment = comment; let mut comment = comment;
@ -1418,7 +1418,7 @@ fn get_next_token_inner(
if state.comment_level > 0 { if state.comment_level > 0 {
let start_pos = *pos; let start_pos = *pos;
let mut comment = if state.include_comments { let mut comment = if state.include_comments {
Some(String::new()) Some(SmartString::new_const())
} else { } else {
None None
}; };
@ -1796,7 +1796,7 @@ fn get_next_token_inner(
('/', '/') => { ('/', '/') => {
eat_next(stream, pos); eat_next(stream, pos);
let mut comment = match stream.peek_next() { let mut comment: Option<SmartString> = match stream.peek_next() {
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
#[cfg(feature = "metadata")] #[cfg(feature = "metadata")]
Some('/') => { Some('/') => {
@ -1805,10 +1805,10 @@ fn get_next_token_inner(
// Long streams of `///...` are not doc-comments // Long streams of `///...` are not doc-comments
match stream.peek_next() { match stream.peek_next() {
Some('/') => None, Some('/') => None,
_ => Some("///".to_string()), _ => Some("///".into()),
} }
} }
_ if state.include_comments => Some("//".to_string()), _ if state.include_comments => Some("//".into()),
_ => None, _ => None,
}; };
@ -1832,14 +1832,14 @@ fn get_next_token_inner(
} }
if let Some(comment) = comment { if let Some(comment) = comment {
return Some((Token::Comment(comment.into()), start_pos)); return Some((Token::Comment(comment), start_pos));
} }
} }
('/', '*') => { ('/', '*') => {
state.comment_level = 1; state.comment_level = 1;
eat_next(stream, pos); eat_next(stream, pos);
let mut comment = match stream.peek_next() { let mut comment: Option<SmartString> = match stream.peek_next() {
#[cfg(not(feature = "no_function"))] #[cfg(not(feature = "no_function"))]
#[cfg(feature = "metadata")] #[cfg(feature = "metadata")]
Some('*') => { Some('*') => {
@ -1848,10 +1848,10 @@ fn get_next_token_inner(
// Long streams of `/****...` are not doc-comments // Long streams of `/****...` are not doc-comments
match stream.peek_next() { match stream.peek_next() {
Some('*') => None, Some('*') => None,
_ => Some("/**".to_string()), _ => Some("/**".into()),
} }
} }
_ if state.include_comments => Some("/*".to_string()), _ if state.include_comments => Some("/*".into()),
_ => None, _ => None,
}; };
@ -1859,7 +1859,7 @@ fn get_next_token_inner(
scan_block_comment(stream, state.comment_level, pos, comment.as_mut()); scan_block_comment(stream, state.comment_level, pos, comment.as_mut());
if let Some(comment) = comment { if let Some(comment) = comment {
return Some((Token::Comment(comment.into()), start_pos)); return Some((Token::Comment(comment), start_pos));
} }
} }