kjuulh/rhai
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

Explanation of parse_string_literal return values.

Browse Source
This commit is contained in:
Stephen Chung 2021-04-13 14:38:04 +08:00
parent ca2ec0d4d5
commit 5a0199f44f
1 changed files with 22 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/token.rs
View File

@ -866,14 +866,10 @@ pub trait InputStream {
/// Returns the parsed string and a boolean indicating whether the string is
/// terminated by an interpolation `${`.
///
/// # Volatile API
///
/// This function is volatile and may change.
///
/// # Returns
///
/// |Type |Return Value |`state.is_within_text_terminated_by`|
/// |---------------------------------|----------------------------|------------------------------------|
/// |---------------------------------|:--------------------------:|:----------------------------------:|
/// |`"hello"` |`StringConstant("hello")` |`None` |
/// |`"hello`_{LF}_ or _{EOF}_ |`LexError` |`None` |
/// |`"hello\`_{EOF}_ or _{LF}{EOF}_ |`StringConstant("hello")` |`Some('"')` |
@ -882,6 +878,22 @@ pub trait InputStream {
/// |`` `hello ${`` |`InterpolatedString("hello ")`<br/>next token is `{`|`None` |
/// |`` } hello` `` |`StringConstant(" hello")` |`None` |
/// |`} hello`_{EOF}_ |`StringConstant(" hello")` |``Some('`')`` |
///
/// This function does not throw a `LexError` for the following conditions:
///
/// * Unterminated literal string at _{EOF}_
///
/// * Unterminated normal string with continuation at _{EOF}_
///
/// This is to facilitate using this function to parse a script line-by-line, where the end of the
/// line (i.e. _{EOF}_) is not necessarily the end of the script.
///
/// Any time a [`StringConstant`][`Token::StringConstant`] is returned with
/// `state.is_within_text_terminated_by` set to `Some(_)` is one of the above conditions.
///
/// # Volatile API
///
/// This function is volatile and may change.
pub fn parse_string_literal(
stream: &mut impl InputStream,
state: &mut TokenizeState,
@ -1947,7 +1959,11 @@ impl<'a> Iterator for TokenIterator<'a> {
let (token, pos) = match get_next_token(&mut self.stream, &mut self.state, &mut self.pos) {
// {EOF}
None => return None,
// {EOF} after unterminated string
// {EOF} after unterminated string.
// The only case where `TokenizeState.is_within_text_terminated_by` is set is when
// a verbatim string or a string with continuation encounters {EOF}.
// This is necessary to handle such cases for line-by-line parsing, but for an entire
// script it is a syntax error.
Some((Token::StringConstant(_), pos)) if self.state.is_within_text_terminated_by.is_some() => {
self.state.is_within_text_terminated_by = None;
return Some((Token::LexError(LERR::UnterminatedString), pos));