diff --git a/examples/repl.rs b/examples/repl.rs index b6a0c9c4..0b20988a 100644 --- a/examples/repl.rs +++ b/examples/repl.rs @@ -14,7 +14,6 @@ fn print_error(input: &str, err: EvalAltResult) { let line_no = if lines.len() > 1 { match err.position() { p if p.is_none() => "".to_string(), - p if p.is_eof() => format!("{}: ", lines.len()), p => format!("{}: ", p.line().unwrap()), } } else { @@ -25,15 +24,7 @@ fn print_error(input: &str, err: EvalAltResult) { let pos = err.position(); let pos_text = format!(" ({})", pos); - let pos = if pos.is_eof() { - let last = lines[lines.len() - 1]; - Position::new(lines.len(), last.len() + 1) - } else { - pos - }; - match pos { - p if p.is_eof() => panic!("should not be EOF"), p if p.is_none() => { // No position println!("{}", err); diff --git a/examples/rhai_runner.rs b/examples/rhai_runner.rs index 9a5cfe59..c2edf508 100644 --- a/examples/rhai_runner.rs +++ b/examples/rhai_runner.rs @@ -23,15 +23,9 @@ fn eprint_error(input: &str, err: EvalAltResult) { let lines: Vec<_> = input.split('\n').collect(); // Print error - let pos = if err.position().is_eof() { - let last = lines[lines.len() - 1]; - Position::new(lines.len(), last.len() + 1) - } else { - err.position() - }; + let pos = err.position(); match pos { - p if p.is_eof() => panic!("should not be EOF"), p if p.is_none() => { // No position eprintln!("{}", err); diff --git a/src/error.rs b/src/error.rs index 077ebd22..b3e6be13 100644 --- a/src/error.rs +++ b/src/error.rs @@ -110,11 +110,6 @@ impl ParseErrorType { pub(crate) fn into_err(self, pos: Position) -> ParseError { ParseError(self, pos) } - - /// Make a `ParseError` using the current type and EOF position. - pub(crate) fn into_err_eof(self) -> ParseError { - ParseError(self, Position::eof()) - } } /// Error when parsing a script. @@ -209,13 +204,11 @@ impl fmt::Display for ParseError { _ => write!(f, "{}", self.desc())?, } - if !self.1.is_eof() { - write!(f, " ({})", self.1) - } else if !self.1.is_none() { + if !self.1.is_none() { // Do not write any position if None Ok(()) } else { - write!(f, " at the end of the script but there is no more input") + write!(f, " ({})", self.1) } } } diff --git a/src/parser.rs b/src/parser.rs index 68fda4ef..b873e076 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -523,15 +523,12 @@ fn eat_token(input: &mut Peekable, token: Token) -> Position { /// Match a particular token, consuming it if matched. fn match_token(input: &mut Peekable, token: Token) -> Result { - if let Some((t, _)) = input.peek() { - if *t == token { - eat_token(input, token); - Ok(true) - } else { - Ok(false) - } + let (t, _) = input.peek().unwrap(); + if *t == token { + eat_token(input, token); + Ok(true) } else { - Err(PERR::UnexpectedEOF.into_err_eof()) + Ok(false) } } @@ -541,21 +538,21 @@ fn parse_paren_expr<'a>( begin: Position, allow_stmt_expr: bool, ) -> Result { - const MISSING_RPAREN: &str = "for a matching ( in this expression"; - if match_token(input, Token::RightParen)? { return Ok(Expr::Unit(begin)); } let expr = parse_expr(input, allow_stmt_expr)?; - match input.next() { + match input.next().unwrap() { // ( xxx ) - Some((Token::RightParen, _)) => Ok(expr), + (Token::RightParen, _) => Ok(expr), // ( xxx ??? - Some((_, pos)) => Err(PERR::MissingToken(")".into(), MISSING_RPAREN.into()).into_err(pos)), - // ( xxx - None => Err(PERR::MissingToken(")".into(), MISSING_RPAREN.into()).into_err_eof()), + (_, pos) => Err(PERR::MissingToken( + ")".into(), + "for a matching ( in this expression".into(), + ) + .into_err(pos)), } } @@ -568,17 +565,17 @@ fn parse_call_expr<'a, S: Into> + Display>( ) -> Result { let mut args_expr_list = Vec::new(); - match input.peek() { + match input.peek().unwrap() { //id {EOF} - None => { + (Token::EOF, pos) => { return Err(PERR::MissingToken( ")".into(), format!("to close the arguments list of this function call '{}'", id), ) - .into_err_eof()) + .into_err(*pos)) } // id() - Some((Token::RightParen, _)) => { + (Token::RightParen, _) => { eat_token(input, Token::RightParen); return Ok(Expr::FunctionCall(id.into(), args_expr_list, None, begin)); } @@ -589,22 +586,22 @@ fn parse_call_expr<'a, S: Into> + Display>( loop { args_expr_list.push(parse_expr(input, allow_stmt_expr)?); - match input.peek() { - None => { + match input.peek().unwrap() { + (Token::EOF, pos) => { return Err(PERR::MissingToken( ")".into(), format!("to close the arguments list of this function call '{}'", id), ) - .into_err_eof()) + .into_err(*pos)) } - Some((Token::RightParen, _)) => { + (Token::RightParen, _) => { eat_token(input, Token::RightParen); return Ok(Expr::FunctionCall(id.into(), args_expr_list, None, begin)); } - Some((Token::Comma, _)) => { + (Token::Comma, _) => { eat_token(input, Token::Comma); } - Some((_, pos)) => { + (_, pos) => { return Err(PERR::MissingToken( ",".into(), format!("to separate the arguments to function call '{}'", id), @@ -727,17 +724,16 @@ fn parse_index_expr<'a>( } // Check if there is a closing bracket - const MISSING_RBRACKET: &str = "for a matching [ in this index expression"; - - match input.peek() { - Some((Token::RightBracket, _)) => { + match input.peek().unwrap() { + (Token::RightBracket, _) => { eat_token(input, Token::RightBracket); Ok(Expr::Index(lhs, Box::new(idx_expr), pos)) } - Some((_, pos)) => { - Err(PERR::MissingToken("]".into(), MISSING_RBRACKET.into()).into_err(*pos)) - } - None => Err(PERR::MissingToken("]".into(), MISSING_RBRACKET.into()).into_err_eof()), + (_, pos) => Err(PERR::MissingToken( + "]".into(), + "for a matching [ in this index expression".into(), + ) + .into_err(*pos)), } } @@ -750,28 +746,28 @@ fn parse_array_literal<'a>( let mut arr = Vec::new(); if !match_token(input, Token::RightBracket)? { - while input.peek().is_some() { + while !input.peek().unwrap().0.is_eof() { arr.push(parse_expr(input, allow_stmt_expr)?); - match input.peek() { - Some((Token::Comma, _)) => eat_token(input, Token::Comma), - Some((Token::RightBracket, _)) => { + match input.peek().unwrap() { + (Token::EOF, pos) => { + return Err( + PERR::MissingToken("]".into(), "to end this array literal".into()) + .into_err(*pos), + ) + } + (Token::Comma, _) => eat_token(input, Token::Comma), + (Token::RightBracket, _) => { eat_token(input, Token::RightBracket); break; } - Some((_, pos)) => { + (_, pos) => { return Err(PERR::MissingToken( ",".into(), "to separate the items of this array literal".into(), ) .into_err(*pos)) } - None => { - return Err( - PERR::MissingToken("]".into(), "to end this array literal".into()) - .into_err_eof(), - ) - } }; } } @@ -788,24 +784,24 @@ fn parse_map_literal<'a>( let mut map = Vec::new(); if !match_token(input, Token::RightBrace)? { - while input.peek().is_some() { + while !input.peek().unwrap().0.is_eof() { const MISSING_RBRACE: &str = "to end this object map literal"; - let (name, pos) = match input.next() { - Some((Token::Identifier(s), pos)) => (s, pos), - Some((Token::StringConst(s), pos)) => (s, pos), - Some((_, pos)) if map.is_empty() => { + let (name, pos) = match input.next().unwrap() { + (Token::Identifier(s), pos) => (s, pos), + (Token::StringConst(s), pos) => (s, pos), + (_, pos) if map.is_empty() => { return Err(PERR::MissingToken("}".into(), MISSING_RBRACE.into()).into_err(pos)) } - Some((_, pos)) => return Err(PERR::PropertyExpected.into_err(pos)), - None => { - return Err(PERR::MissingToken("}".into(), MISSING_RBRACE.into()).into_err_eof()) + (Token::EOF, pos) => { + return Err(PERR::MissingToken("}".into(), MISSING_RBRACE.into()).into_err(pos)) } + (_, pos) => return Err(PERR::PropertyExpected.into_err(pos)), }; - match input.next() { - Some((Token::Colon, _)) => (), - Some((_, pos)) => { + match input.next().unwrap() { + (Token::Colon, _) => (), + (_, pos) => { return Err(PERR::MissingToken( ":".into(), format!( @@ -815,43 +811,30 @@ fn parse_map_literal<'a>( ) .into_err(pos)) } - None => { - return Err(PERR::MissingToken( - ":".into(), - format!( - "to follow the property '{}' in this object map literal", - name - ), - ) - .into_err_eof()) - } }; let expr = parse_expr(input, allow_stmt_expr)?; map.push((name, expr, pos)); - match input.peek() { - Some((Token::Comma, _)) => { + match input.peek().unwrap() { + (Token::Comma, _) => { eat_token(input, Token::Comma); } - Some((Token::RightBrace, _)) => { + (Token::RightBrace, _) => { eat_token(input, Token::RightBrace); break; } - Some((Token::Identifier(_), pos)) => { + (Token::Identifier(_), pos) => { return Err(PERR::MissingToken( ",".into(), "to separate the items of this object map literal".into(), ) .into_err(*pos)) } - Some((_, pos)) => { + (_, pos) => { return Err(PERR::MissingToken("}".into(), MISSING_RBRACE.into()).into_err(*pos)) } - None => { - return Err(PERR::MissingToken("}".into(), MISSING_RBRACE.into()).into_err_eof()) - } } } } @@ -875,38 +858,40 @@ fn parse_primary<'a>( input: &mut Peekable>, allow_stmt_expr: bool, ) -> Result { - let token = match input.peek() { + let (token, pos) = match input.peek().unwrap() { // { - block statement as expression - Some((Token::LeftBrace, pos)) if allow_stmt_expr => { + (Token::LeftBrace, pos) if allow_stmt_expr => { let pos = *pos; return parse_block(input, false, allow_stmt_expr) .map(|block| Expr::Stmt(Box::new(block), pos)); } - Some(_) => input.next().expect("should be a token"), - None => return Err(PERR::UnexpectedEOF.into_err_eof()), + (Token::EOF, pos) => return Err(PERR::UnexpectedEOF.into_err(*pos)), + _ => input.next().unwrap(), }; let mut root_expr = match token { - (Token::IntegerConstant(x), pos) => Expr::IntegerConstant(x, pos), - (Token::FloatConstant(x), pos) => Expr::FloatConstant(x, pos), - (Token::CharConstant(c), pos) => Expr::CharConstant(c, pos), - (Token::StringConst(s), pos) => Expr::StringConstant(s.into(), pos), - (Token::Identifier(s), pos) => Expr::Variable(s.into(), pos), - (Token::LeftParen, pos) => parse_paren_expr(input, pos, allow_stmt_expr)?, + Token::IntegerConstant(x) => Expr::IntegerConstant(x, pos), + Token::FloatConstant(x) => Expr::FloatConstant(x, pos), + Token::CharConstant(c) => Expr::CharConstant(c, pos), + Token::StringConst(s) => Expr::StringConstant(s.into(), pos), + Token::Identifier(s) => Expr::Variable(s.into(), pos), + Token::LeftParen => parse_paren_expr(input, pos, allow_stmt_expr)?, #[cfg(not(feature = "no_index"))] - (Token::LeftBracket, pos) => parse_array_literal(input, pos, allow_stmt_expr)?, + Token::LeftBracket => parse_array_literal(input, pos, allow_stmt_expr)?, #[cfg(not(feature = "no_object"))] - (Token::MapStart, pos) => parse_map_literal(input, pos, allow_stmt_expr)?, - (Token::True, pos) => Expr::True(pos), - (Token::False, pos) => Expr::False(pos), - (Token::LexError(err), pos) => return Err(PERR::BadInput(err.to_string()).into_err(pos)), - (token, pos) => { + Token::MapStart => parse_map_literal(input, pos, allow_stmt_expr)?, + Token::True => Expr::True(pos), + Token::False => Expr::False(pos), + Token::LexError(err) => return Err(PERR::BadInput(err.to_string()).into_err(pos)), + token => { return Err(PERR::BadInput(format!("Unexpected '{}'", token.syntax())).into_err(pos)) } }; // Tail processing all possible postfix operators - while let Some((token, _)) = input.peek() { + loop { + let (token, _) = input.peek().unwrap(); + if !root_expr.is_valid_postfix(token) { break; } @@ -936,9 +921,9 @@ fn parse_unary<'a>( input: &mut Peekable>, allow_stmt_expr: bool, ) -> Result { - match input.peek() { + match input.peek().unwrap() { // If statement is allowed to act as expressions - Some((Token::If, pos)) => { + (Token::If, pos) => { let pos = *pos; Ok(Expr::Stmt( Box::new(parse_if(input, false, allow_stmt_expr)?), @@ -946,7 +931,7 @@ fn parse_unary<'a>( )) } // -expr - Some((Token::UnaryMinus, _)) => { + (Token::UnaryMinus, _) => { let pos = eat_token(input, Token::UnaryMinus); match parse_unary(input, allow_stmt_expr)? { @@ -978,12 +963,12 @@ fn parse_unary<'a>( } } // +expr - Some((Token::UnaryPlus, _)) => { + (Token::UnaryPlus, _) => { eat_token(input, Token::UnaryPlus); parse_unary(input, allow_stmt_expr) } // !expr - Some((Token::Bang, _)) => { + (Token::Bang, _) => { let pos = eat_token(input, Token::Bang); Ok(Expr::FunctionCall( "!".into(), @@ -992,10 +977,10 @@ fn parse_unary<'a>( pos, )) } - // All other tokens - Some(_) => parse_primary(input, allow_stmt_expr), // {EOF} - None => Err(PERR::UnexpectedEOF.into_err_eof()), + (Token::EOF, pos) => Err(PERR::UnexpectedEOF.into_err(*pos)), + // All other tokens + _ => parse_primary(input, allow_stmt_expr), } } @@ -1378,12 +1363,11 @@ fn ensure_not_statement_expr<'a>( input: &mut Peekable>, type_name: &str, ) -> Result<(), ParseError> { - match input - .peek() - .ok_or_else(|| PERR::ExprExpected(type_name.to_string()).into_err_eof())? - { + match input.peek().unwrap() { // Disallow statement expressions - (Token::LeftBrace, pos) => Err(PERR::ExprExpected(type_name.to_string()).into_err(*pos)), + (Token::LeftBrace, pos) | (Token::EOF, pos) => { + Err(PERR::ExprExpected(type_name.to_string()).into_err(*pos)) + } // No need to check for others at this time - leave it for the expr parser _ => Ok(()), } @@ -1462,27 +1446,26 @@ fn parse_for<'a>( eat_token(input, Token::For); // for name ... - let name = match input - .next() - .ok_or_else(|| PERR::VariableExpected.into_err_eof())? - { + let name = match input.next().unwrap() { // Variable name (Token::Identifier(s), _) => s, // Bad identifier (Token::LexError(err), pos) => return Err(PERR::BadInput(err.to_string()).into_err(pos)), + // EOF + (Token::EOF, pos) => return Err(PERR::VariableExpected.into_err(pos)), // Not a variable name (_, pos) => return Err(PERR::VariableExpected.into_err(pos)), }; // for name in ... - const ERROR_MSG: &str = "after the iteration variable"; - - match input.next() { - Some((Token::In, _)) => (), - Some((_, pos)) => { - return Err(PERR::MissingToken("in".into(), ERROR_MSG.into()).into_err(pos)) + match input.next().unwrap() { + (Token::In, _) => (), + (_, pos) => { + return Err( + PERR::MissingToken("in".into(), "after the iteration variable".into()) + .into_err(pos), + ) } - None => return Err(PERR::MissingToken("in".into(), ERROR_MSG.into()).into_err_eof()), } // for name in expr { body } @@ -1503,10 +1486,7 @@ fn parse_let<'a>( input.next(); // let name ... - let (name, pos) = match input - .next() - .ok_or_else(|| PERR::VariableExpected.into_err_eof())? - { + let (name, pos) = match input.next().unwrap() { (Token::Identifier(s), pos) => (s, pos), (Token::LexError(err), pos) => return Err(PERR::BadInput(err.to_string()).into_err(pos)), (_, pos) => return Err(PERR::VariableExpected.into_err(pos)), @@ -1542,10 +1522,7 @@ fn parse_block<'a>( allow_stmt_expr: bool, ) -> Result { // Must start with { - let pos = match input - .next() - .ok_or_else(|| PERR::UnexpectedEOF.into_err_eof())? - { + let pos = match input.next().unwrap() { (Token::LeftBrace, pos) => pos, (_, pos) => { return Err( @@ -1565,35 +1542,28 @@ fn parse_block<'a>( statements.push(stmt); - match input.peek() { + match input.peek().unwrap() { // { ... stmt } - Some((Token::RightBrace, _)) => { + (Token::RightBrace, _) => { eat_token(input, Token::RightBrace); break; } // { ... stmt; - Some((Token::SemiColon, _)) if need_semicolon => { + (Token::SemiColon, _) if need_semicolon => { eat_token(input, Token::SemiColon); } // { ... { stmt } ; - Some((Token::SemiColon, _)) if !need_semicolon => (), + (Token::SemiColon, _) if !need_semicolon => (), // { ... { stmt } ??? - Some((_, _)) if !need_semicolon => (), + (_, _) if !need_semicolon => (), // { ... stmt ??? - error - Some((_, pos)) => { + (_, pos) => { // Semicolons are not optional between statements return Err( PERR::MissingToken(";".into(), "to terminate this statement".into()) .into_err(*pos), ); } - // {EOF} - None => { - return Err( - PERR::MissingToken("}".into(), "to end this statement block".into()) - .into_err_eof(), - ) - } } } @@ -1614,37 +1584,37 @@ fn parse_stmt<'a>( breakable: bool, allow_stmt_expr: bool, ) -> Result { - let token = match input.peek() { - Some(token) => token, - None => return Ok(Stmt::Noop(Position::eof())), + let (token, pos) = match input.peek().unwrap() { + (Token::EOF, pos) => return Ok(Stmt::Noop(*pos)), + x => x, }; match token { // Semicolon - empty statement - (Token::SemiColon, pos) => Ok(Stmt::Noop(*pos)), + Token::SemiColon => Ok(Stmt::Noop(*pos)), - (Token::LeftBrace, _) => parse_block(input, breakable, allow_stmt_expr), + Token::LeftBrace => parse_block(input, breakable, allow_stmt_expr), // fn ... #[cfg(not(feature = "no_function"))] - (Token::Fn, pos) => Err(PERR::WrongFnDefinition.into_err(*pos)), + Token::Fn => Err(PERR::WrongFnDefinition.into_err(*pos)), - (Token::If, _) => parse_if(input, breakable, allow_stmt_expr), - (Token::While, _) => parse_while(input, allow_stmt_expr), - (Token::Loop, _) => parse_loop(input, allow_stmt_expr), - (Token::For, _) => parse_for(input, allow_stmt_expr), + Token::If => parse_if(input, breakable, allow_stmt_expr), + Token::While => parse_while(input, allow_stmt_expr), + Token::Loop => parse_loop(input, allow_stmt_expr), + Token::For => parse_for(input, allow_stmt_expr), - (Token::Continue, _) if breakable => { + Token::Continue if breakable => { let pos = eat_token(input, Token::Continue); Ok(Stmt::Continue(pos)) } - (Token::Break, _) if breakable => { + Token::Break if breakable => { let pos = eat_token(input, Token::Break); Ok(Stmt::Break(pos)) } - (Token::Continue, pos) | (Token::Break, pos) => Err(PERR::LoopBreak.into_err(*pos)), + Token::Continue | Token::Break => Err(PERR::LoopBreak.into_err(*pos)), - (Token::Return, pos) | (Token::Throw, pos) => { + Token::Return | Token::Throw => { let pos = *pos; let return_type = match input.next() { @@ -1653,13 +1623,13 @@ fn parse_stmt<'a>( _ => panic!("token should be return or throw"), }; - match input.peek() { + match input.peek().unwrap() { // `return`/`throw` at {EOF} - None => Ok(Stmt::ReturnWithVal(None, return_type, Position::eof())), + (Token::EOF, pos) => Ok(Stmt::ReturnWithVal(None, return_type, *pos)), // `return;` or `throw;` - Some((Token::SemiColon, _)) => Ok(Stmt::ReturnWithVal(None, return_type, pos)), + (Token::SemiColon, _) => Ok(Stmt::ReturnWithVal(None, return_type, pos)), // `return` or `throw` with expression - Some((_, _)) => { + (_, _) => { let expr = parse_expr(input, allow_stmt_expr)?; let pos = expr.position(); Ok(Stmt::ReturnWithVal(Some(Box::new(expr)), return_type, pos)) @@ -1667,8 +1637,8 @@ fn parse_stmt<'a>( } } - (Token::Let, _) => parse_let(input, ScopeEntryType::Normal, allow_stmt_expr), - (Token::Const, _) => parse_let(input, ScopeEntryType::Constant, allow_stmt_expr), + Token::Let => parse_let(input, ScopeEntryType::Normal, allow_stmt_expr), + Token::Const => parse_let(input, ScopeEntryType::Constant, allow_stmt_expr), _ => parse_expr_stmt(input, allow_stmt_expr), } @@ -1681,16 +1651,14 @@ fn parse_fn<'a>( ) -> Result { let pos = input.next().expect("should be fn").1; - let name = match input.next() { - Some((Token::Identifier(s), _)) => s, - Some((_, pos)) => return Err(PERR::FnMissingName.into_err(pos)), - None => return Err(PERR::FnMissingName.into_err_eof()), + let name = match input.next().unwrap() { + (Token::Identifier(s), _) => s, + (_, pos) => return Err(PERR::FnMissingName.into_err(pos)), }; - match input.peek() { - Some((Token::LeftParen, _)) => eat_token(input, Token::LeftParen), - Some((_, pos)) => return Err(PERR::FnMissingParams(name).into_err(*pos)), - None => return Err(PERR::FnMissingParams(name.clone()).into_err_eof()), + match input.peek().unwrap() { + (Token::LeftParen, _) => eat_token(input, Token::LeftParen), + (_, pos) => return Err(PERR::FnMissingParams(name).into_err(*pos)), }; let mut params = Vec::new(); @@ -1700,24 +1668,18 @@ fn parse_fn<'a>( let sep_err = format!("to separate the parameters of function '{}'", name); loop { - match input.next() { - Some((Token::Identifier(s), pos)) => params.push((s, pos)), - Some((_, pos)) => return Err(PERR::MissingToken(")".into(), end_err).into_err(pos)), - None => { - return Err(PERR::MissingToken(")".into(), end_err.to_string()).into_err_eof()) - } + match input.next().unwrap() { + (Token::Identifier(s), pos) => params.push((s, pos)), + (_, pos) => return Err(PERR::MissingToken(")".into(), end_err).into_err(pos)), } - match input.next() { - Some((Token::RightParen, _)) => break, - Some((Token::Comma, _)) => (), - Some((Token::Identifier(_), pos)) => { + match input.next().unwrap() { + (Token::RightParen, _) => break, + (Token::Comma, _) => (), + (Token::Identifier(_), pos) => { return Err(PERR::MissingToken(",".into(), sep_err).into_err(pos)) } - Some((_, pos)) => return Err(PERR::MissingToken(",".into(), sep_err).into_err(pos)), - None => { - return Err(PERR::MissingToken(")".into(), end_err.to_string()).into_err_eof()) - } + (_, pos) => return Err(PERR::MissingToken(",".into(), sep_err).into_err(pos)), } } } @@ -1738,10 +1700,9 @@ fn parse_fn<'a>( })?; // Parse function body - let body = match input.peek() { - Some((Token::LeftBrace, _)) => parse_block(input, false, allow_stmt_expr)?, - Some((_, pos)) => return Err(PERR::FnMissingBody(name).into_err(*pos)), - None => return Err(PERR::FnMissingBody(name).into_err_eof()), + let body = match input.peek().unwrap() { + (Token::LeftBrace, _) => parse_block(input, false, allow_stmt_expr)?, + (_, pos) => return Err(PERR::FnMissingBody(name).into_err(*pos)), }; let params = params.into_iter().map(|(p, _)| p).collect(); @@ -1762,9 +1723,12 @@ pub fn parse_global_expr<'a>( ) -> Result { let expr = parse_expr(input, false)?; - if let Some((token, pos)) = input.peek() { + match input.peek().unwrap() { + (Token::EOF, _) => (), // Return error if the expression doesn't end - return Err(PERR::BadInput(format!("Unexpected '{}'", token.syntax())).into_err(*pos)); + (token, pos) => { + return Err(PERR::BadInput(format!("Unexpected '{}'", token.syntax())).into_err(*pos)) + } } Ok( @@ -1786,7 +1750,7 @@ fn parse_global_level<'a>( let mut statements = Vec::::new(); let mut functions = HashMap::::new(); - while input.peek().is_some() { + while !input.peek().unwrap().0.is_eof() { // Collect all the function definitions #[cfg(not(feature = "no_function"))] { @@ -1804,19 +1768,19 @@ fn parse_global_level<'a>( statements.push(stmt); - match input.peek() { + match input.peek().unwrap() { // EOF - None => break, + (Token::EOF, _) => break, // stmt ; - Some((Token::SemiColon, _)) if need_semicolon => { + (Token::SemiColon, _) if need_semicolon => { eat_token(input, Token::SemiColon); } // stmt ; - Some((Token::SemiColon, _)) if !need_semicolon => (), + (Token::SemiColon, _) if !need_semicolon => (), // { stmt } ??? - Some((_, _)) if !need_semicolon => (), + (_, _) if !need_semicolon => (), // stmt ??? - error - Some((_, pos)) => { + (_, pos) => { // Semicolons are not optional between statements return Err( PERR::MissingToken(";".into(), "to terminate this statement".into()) diff --git a/src/token.rs b/src/token.rs index 55563d0f..fd46d10f 100644 --- a/src/token.rs +++ b/src/token.rs @@ -22,9 +22,9 @@ type LERR = LexError; /// A location (line number + character position) in the input script. #[derive(Eq, PartialEq, Ord, PartialOrd, Hash, Clone, Copy)] pub struct Position { - /// Line number - 0 = none, MAX = EOF + /// Line number - 0 = none line: usize, - /// Character position - 0 = BOL, MAX = EOF + /// Character position - 0 = BOL pos: usize, } @@ -43,9 +43,9 @@ impl Position { } } - /// Get the line number (1-based), or `None` if no position or EOF. + /// Get the line number (1-based), or `None` if no position. pub fn line(&self) -> Option { - if self.is_none() || self.is_eof() { + if self.is_none() { None } else { Some(self.line) @@ -54,7 +54,7 @@ impl Position { /// Get the character position (1-based), or `None` if at beginning of a line. pub fn position(&self) -> Option { - if self.is_none() || self.is_eof() || self.pos == 0 { + if self.is_none() || self.pos == 0 { None } else { Some(self.pos) @@ -88,23 +88,10 @@ impl Position { Self { line: 0, pos: 0 } } - /// Create a `Position` at EOF. - pub(crate) fn eof() -> Self { - Self { - line: usize::MAX, - pos: usize::MAX, - } - } - /// Is there no `Position`? pub fn is_none(&self) -> bool { self.line == 0 && self.pos == 0 } - - /// Is the `Position` at EOF? - pub fn is_eof(&self) -> bool { - self.line == usize::MAX && self.pos == usize::MAX - } } impl Default for Position { @@ -115,9 +102,7 @@ impl Default for Position { impl fmt::Display for Position { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.is_eof() { - write!(f, "EOF") - } else if self.is_none() { + if self.is_none() { write!(f, "none") } else { write!(f, "line {}, position {}", self.line, self.pos) @@ -127,11 +112,7 @@ impl fmt::Display for Position { impl fmt::Debug for Position { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.is_eof() { - write!(f, "(EOF)") - } else { - write!(f, "({}:{})", self.line, self.pos) - } + write!(f, "({}:{})", self.line, self.pos) } } @@ -206,6 +187,7 @@ pub enum Token { ModuloAssign, PowerOfAssign, LexError(Box), + EOF, } impl Token { @@ -284,12 +266,23 @@ impl Token { PowerOfAssign => "~=", For => "for", In => "in", + EOF => "{EOF}", _ => panic!("operator should be match in outer scope"), }) .into(), } } + // Is this token EOF? + pub fn is_eof(&self) -> bool { + use Token::*; + + match self { + EOF => true, + _ => false, + } + } + // If another operator is after these, it's probably an unary operator // (not sure about fn name). pub fn is_next_unary(&self) -> bool { @@ -420,10 +413,13 @@ impl<'a> TokenIterator<'a> { fn get_next(&mut self) -> Option { loop { if self.streams.is_empty() { + // No more streams return None; } else if let Some(ch) = self.streams[0].next() { + // Next character in current stream return Some(ch); } else { + // Jump to the next stream let _ = self.streams.remove(0); } } @@ -432,10 +428,13 @@ impl<'a> TokenIterator<'a> { fn peek_next(&mut self) -> Option { loop { if self.streams.is_empty() { + // No more streams return None; } else if let Some(ch) = self.streams[0].peek() { + // Next character in current stream return Some(*ch); } else { + // Jump to the next stream let _ = self.streams.remove(0); } } @@ -466,10 +465,13 @@ impl<'a> TokenIterator<'a> { let mut escape = String::with_capacity(12); loop { - let next_char = self.get_next(); + let next_char = self + .get_next() + .ok_or((LERR::UnterminatedString, self.pos))?; + self.advance(); - match next_char.ok_or((LERR::UnterminatedString, Position::eof()))? { + match next_char { // \... '\\' if escape.is_empty() => { escape.push('\\'); @@ -956,7 +958,8 @@ impl<'a> TokenIterator<'a> { } } - None + self.advance(); + Some((Token::EOF, self.pos)) } }