From e4e471fd208d1a631ca05b8473623af63d7dbe55 Mon Sep 17 00:00:00 2001 From: Stephen Chung Date: Mon, 24 Feb 2020 23:25:52 +0800 Subject: [PATCH] Add line/position in compilation errors; Add compile to AST; FIx infinite loop in close parens. --- src/engine.rs | 220 ++++++---- src/lib.rs | 1 + src/parser.rs | 967 +++++++++++++++++++++++++---------------- tests/decrement.rs | 2 +- tests/mismatched_op.rs | 2 +- 5 files changed, 715 insertions(+), 477 deletions(-) diff --git a/src/engine.rs b/src/engine.rs index 165f6d22..9e942346 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -9,13 +9,14 @@ use std::{convert::TryInto, sync::Arc}; use crate::any::{Any, AnyExt}; use crate::call::FunArgs; use crate::fn_register::{RegisterBoxFn, RegisterFn}; -use crate::parser::{lex, parse, Expr, FnDef, Stmt}; +use crate::parser::{lex, parse, Expr, FnDef, ParseError, Stmt, AST}; use fmt::{Debug, Display}; type Array = Vec>; -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum EvalAltResult { + ErrorParseError(ParseError), ErrorFunctionNotFound(String), ErrorFunctionArgMismatch, ErrorArrayOutOfBounds(usize, i64), @@ -27,20 +28,20 @@ pub enum EvalAltResult { ErrorAssignmentToUnknownLHS, ErrorMismatchOutputType(String), ErrorCantOpenScriptFile(String), - InternalErrorMalformedDotExpression, + ErrorMalformedDotExpression, LoopBreak, Return(Box), } impl EvalAltResult { fn as_str(&self) -> Option<&str> { - match *self { - EvalAltResult::ErrorCantOpenScriptFile(ref s) => Some(s.as_str()), - EvalAltResult::ErrorVariableNotFound(ref s) => Some(s.as_str()), - EvalAltResult::ErrorFunctionNotFound(ref s) => Some(s.as_str()), - EvalAltResult::ErrorMismatchOutputType(ref s) => Some(s.as_str()), - _ => None, - } + Some(match *self { + EvalAltResult::ErrorCantOpenScriptFile(ref s) + | EvalAltResult::ErrorVariableNotFound(ref s) + | EvalAltResult::ErrorFunctionNotFound(ref s) + | EvalAltResult::ErrorMismatchOutputType(ref s) => s, + _ => return None, + }) } } @@ -49,6 +50,7 @@ impl PartialEq for EvalAltResult { use EvalAltResult::*; match (self, other) { + (&ErrorParseError(ref a), &ErrorParseError(ref b)) => a == b, (&ErrorFunctionNotFound(ref a), &ErrorFunctionNotFound(ref b)) => a == b, (&ErrorFunctionArgMismatch, &ErrorFunctionArgMismatch) => true, (&ErrorIndexMismatch, &ErrorIndexMismatch) => true, @@ -62,7 +64,7 @@ impl PartialEq for EvalAltResult { (&ErrorAssignmentToUnknownLHS, &ErrorAssignmentToUnknownLHS) => true, (&ErrorMismatchOutputType(ref a), &ErrorMismatchOutputType(ref b)) => a == b, (&ErrorCantOpenScriptFile(ref a), &ErrorCantOpenScriptFile(ref b)) => a == b, - (&InternalErrorMalformedDotExpression, &InternalErrorMalformedDotExpression) => true, + (&ErrorMalformedDotExpression, &ErrorMalformedDotExpression) => true, (&LoopBreak, &LoopBreak) => true, _ => false, } @@ -72,6 +74,7 @@ impl PartialEq for EvalAltResult { impl Error for EvalAltResult { fn description(&self) -> &str { match *self { + EvalAltResult::ErrorParseError(ref p) => p.description(), EvalAltResult::ErrorFunctionNotFound(_) => "Function not found", EvalAltResult::ErrorFunctionArgMismatch => "Function argument types do not match", EvalAltResult::ErrorIndexMismatch => "Array access expects integer index", @@ -85,15 +88,13 @@ impl Error for EvalAltResult { EvalAltResult::ErrorForMismatch => "For loops expect array", EvalAltResult::ErrorVariableNotFound(_) => "Variable not found", EvalAltResult::ErrorAssignmentToUnknownLHS => { - "Assignment to an unsupported left-hand side" + "Assignment to an unsupported left-hand side expression" } - EvalAltResult::ErrorMismatchOutputType(_) => "Cast of output failed", + EvalAltResult::ErrorMismatchOutputType(_) => "Output type is incorrect", EvalAltResult::ErrorCantOpenScriptFile(_) => "Cannot open script file", - EvalAltResult::InternalErrorMalformedDotExpression => { - "[Internal error] Unexpected expression in dot expression" - } - EvalAltResult::LoopBreak => "Loop broken before completion (not an error)", - EvalAltResult::Return(_) => "Function returned value (not an error)", + EvalAltResult::ErrorMalformedDotExpression => "Malformed dot expression", + EvalAltResult::LoopBreak => "[Not Error] Breaks out of loop", + EvalAltResult::Return(_) => "[Not Error] Function returns value", } } @@ -108,6 +109,7 @@ impl fmt::Display for EvalAltResult { write!(f, "{}: {}", self.description(), s) } else { match self { + EvalAltResult::ErrorParseError(ref p) => write!(f, "Syntax error: {}", p), EvalAltResult::ErrorArrayOutOfBounds(_, index) if *index < 0 => { write!(f, "{}: {} < 0", self.description(), index) } @@ -224,7 +226,11 @@ impl Engine { .iter() .map(|x| (*(&**x).box_clone()).type_name()) .collect::>(); - EvalAltResult::ErrorFunctionNotFound(format!("{} ({})", ident, typenames.join(","))) + EvalAltResult::ErrorFunctionNotFound(format!( + "{} ({})", + ident, + typenames.join(", ") + )) }) .and_then(move |f| match **f { FnIntExt::Ext(ref f) => f(args), @@ -308,7 +314,7 @@ impl Engine { use std::iter::once; match *dot_rhs { - Expr::FnCall(ref fn_name, ref args) => { + Expr::FunctionCall(ref fn_name, ref args) => { let mut args: Array = args .iter() .map(|arg| self.eval_expr(scope, arg)) @@ -351,9 +357,9 @@ impl Engine { self.call_fn_raw(get_fn_name, vec![this_ptr]) .and_then(|mut v| self.get_dot_val_helper(scope, v.as_mut(), inner_rhs)) } - _ => Err(EvalAltResult::InternalErrorMalformedDotExpression), + _ => Err(EvalAltResult::ErrorMalformedDotExpression), }, - _ => Err(EvalAltResult::InternalErrorMalformedDotExpression), + _ => Err(EvalAltResult::ErrorMalformedDotExpression), } } @@ -429,7 +435,7 @@ impl Engine { value } - _ => Err(EvalAltResult::InternalErrorMalformedDotExpression), + _ => Err(EvalAltResult::ErrorMalformedDotExpression), } } @@ -458,9 +464,9 @@ impl Engine { self.call_fn_raw(set_fn_name, vec![this_ptr, v.as_mut()]) }) } - _ => Err(EvalAltResult::InternalErrorMalformedDotExpression), + _ => Err(EvalAltResult::ErrorMalformedDotExpression), }, - _ => Err(EvalAltResult::InternalErrorMalformedDotExpression), + _ => Err(EvalAltResult::ErrorMalformedDotExpression), } } @@ -492,16 +498,16 @@ impl Engine { value } - _ => Err(EvalAltResult::InternalErrorMalformedDotExpression), + _ => Err(EvalAltResult::ErrorMalformedDotExpression), } } fn eval_expr(&self, scope: &mut Scope, expr: &Expr) -> Result, EvalAltResult> { match *expr { - Expr::IntConst(i) => Ok(Box::new(i)), - Expr::FloatConst(i) => Ok(Box::new(i)), - Expr::StringConst(ref s) => Ok(Box::new(s.clone())), - Expr::CharConst(ref c) => Ok(Box::new(*c)), + Expr::IntegerConstant(i) => Ok(Box::new(i)), + Expr::FloatConstant(i) => Ok(Box::new(i)), + Expr::StringConstant(ref s) => Ok(Box::new(s.clone())), + Expr::CharConstant(ref c) => Ok(Box::new(*c)), Expr::Identifier(ref id) => { for &mut (ref name, ref mut val) in &mut scope.iter_mut().rev() { if *id == *name { @@ -575,7 +581,7 @@ impl Engine { Ok(Box::new(arr)) } - Expr::FnCall(ref fn_name, ref args) => self.call_fn_raw( + Expr::FunctionCall(ref fn_name, ref args) => self.call_fn_raw( fn_name.to_owned(), args.iter() .map(|ex| self.eval_expr(scope, ex)) @@ -687,7 +693,7 @@ impl Engine { let result = self.eval_expr(scope, a)?; Err(EvalAltResult::Return(result)) } - Stmt::Var(ref name, ref init) => { + Stmt::Let(ref name, ref init) => { match *init { Some(ref v) => { let i = self.eval_expr(scope, v)?; @@ -700,6 +706,34 @@ impl Engine { } } + /// Compile a string into an AST + pub fn compile(input: &str) -> Result { + let tokens = lex(input); + + let mut peekables = tokens.peekable(); + let tree = parse(&mut peekables); + + tree + } + + /// Compile a file into an AST + pub fn compile_file(fname: &str) -> Result { + use std::fs::File; + use std::io::prelude::*; + + if let Ok(mut f) = File::open(fname) { + let mut contents = String::new(); + + if f.read_to_string(&mut contents).is_ok() { + Self::compile(&contents).map_err(|err| EvalAltResult::ErrorParseError(err)) + } else { + Err(EvalAltResult::ErrorCantOpenScriptFile(fname.to_owned())) + } + } else { + Err(EvalAltResult::ErrorCantOpenScriptFile(fname.to_owned())) + } + } + /// Evaluate a file pub fn eval_file(&mut self, fname: &str) -> Result { use std::fs::File; @@ -721,52 +755,58 @@ impl Engine { /// Evaluate a string pub fn eval(&mut self, input: &str) -> Result { let mut scope = Scope::new(); - self.eval_with_scope(&mut scope, input) } - /// Evaluate with own scope + /// Evaluate a string with own scope pub fn eval_with_scope( &mut self, scope: &mut Scope, input: &str, ) -> Result { - let tokens = lex(input); + let ast = Self::compile(input).map_err(|err| EvalAltResult::ErrorParseError(err))?; + self.eval_ast_with_scope(scope, &ast) + } - let mut peekables = tokens.peekable(); - let tree = parse(&mut peekables); + /// Evaluate an AST + pub fn eval_ast(&mut self, ast: &AST) -> Result { + let mut scope = Scope::new(); + self.eval_ast_with_scope(&mut scope, ast) + } - match tree { - Ok((ref os, ref fns)) => { - let mut x: Result, EvalAltResult> = Ok(Box::new(())); + /// Evaluate an AST with own scope + pub fn eval_ast_with_scope( + &mut self, + scope: &mut Scope, + ast: &AST, + ) -> Result { + let AST(os, fns) = ast; + let mut x: Result, EvalAltResult> = Ok(Box::new(())); - for f in fns { - let name = f.name.clone(); - let local_f = f.clone(); + for f in fns { + let name = f.name.clone(); + let local_f = f.clone(); - let spec = FnSpec { - ident: name, - args: None, - }; + let spec = FnSpec { + ident: name, + args: None, + }; - self.fns.insert(spec, Arc::new(FnIntExt::Int(local_f))); - } + self.fns.insert(spec, Arc::new(FnIntExt::Int(local_f))); + } - for o in os { - x = match self.eval_stmt(scope, o) { - Ok(v) => Ok(v), - Err(e) => return Err(e), - } - } - - let x = x?; - - match x.downcast::() { - Ok(out) => Ok(*out), - Err(a) => Err(EvalAltResult::ErrorMismatchOutputType((*a).type_name())), - } + for o in os { + x = match self.eval_stmt(scope, o) { + Ok(v) => Ok(v), + Err(e) => return Err(e), } - Err(_) => Err(EvalAltResult::ErrorFunctionArgMismatch), + } + + let x = x?; + + match x.downcast::() { + Ok(out) => Ok(*out), + Err(a) => Err(EvalAltResult::ErrorMismatchOutputType((*a).type_name())), } } @@ -815,7 +855,7 @@ impl Engine { let tree = parse(&mut peekables); match tree { - Ok((ref os, ref fns)) => { + Ok(AST(ref os, ref fns)) => { for f in fns { if f.params.len() > 6 { return Ok(()); @@ -878,7 +918,7 @@ impl Engine { ) } - macro_rules! reg_func2 { + macro_rules! reg_func2x { ($engine:expr, $x:expr, $op:expr, $v:ty, $r:ty, $( $y:ty ),*) => ( $( $engine.register_fn($x, $op as fn(x: $v, y: $y)->$r); @@ -886,7 +926,7 @@ impl Engine { ) } - macro_rules! reg_func2b { + macro_rules! reg_func2y { ($engine:expr, $x:expr, $op:expr, $v:ty, $r:ty, $( $y:ty ),*) => ( $( $engine.register_fn($x, $op as fn(y: $y, x: $v)->$r); @@ -1018,54 +1058,42 @@ impl Engine { reg_func1!(engine, "print", print, (), i32, i64, u32, u64); reg_func1!(engine, "print", print, (), f32, f64, bool, String); reg_func1!(engine, "print", print_debug, (), Array); + engine.register_fn("print", |_: ()| println!()); reg_func1!(engine, "debug", print_debug, (), i32, i64, u32, u64); reg_func1!(engine, "debug", print_debug, (), f32, f64, bool, String); - reg_func1!(engine, "debug", print_debug, (), Array); + reg_func1!(engine, "debug", print_debug, (), Array, ()); // Register array functions fn push(list: &mut Array, item: T) { list.push(Box::new(item)); } - fn pop(list: &mut Array) -> Box { - list.pop().unwrap() - } - fn shift(list: &mut Array) -> Box { - list.remove(0) - } - fn len(list: &mut Array) -> i64 { + + reg_func2x!(engine, "push", push, &mut Array, (), i32, i64, u32, u64); + reg_func2x!(engine, "push", push, &mut Array, (), f32, f64, bool); + reg_func2x!(engine, "push", push, &mut Array, (), String, Array, ()); + + engine.register_box_fn("pop", |list: &mut Array| list.pop().unwrap()); + engine.register_box_fn("shift", |list: &mut Array| list.remove(0)); + engine.register_fn("len", |list: &mut Array| -> i64 { list.len().try_into().unwrap() - } - - reg_func2!(engine, "push", push, &mut Array, (), i32, i64, u32, u64); - reg_func2!(engine, "push", push, &mut Array, (), f32, f64, bool); - reg_func2!(engine, "push", push, &mut Array, (), String, Array); - - engine.register_box_fn("pop", pop); - engine.register_box_fn("shift", shift); - engine.register_fn("len", len); + }); // Register string concatenate functions fn prepend(x: T, y: String) -> String { format!("{}{}", x, y) } - fn append(x: &mut String, y: T) -> String { + fn append(x: String, y: T) -> String { format!("{}{}", x, y) } - fn prepend_array(x: Array, y: String) -> String { - format!("{:?}{}", x, y) - } - fn append_array(x: &mut String, y: Array) -> String { - format!("{}{:?}", x, y) - } - reg_func2!(engine, "+", append, &mut String, String, i32, i64); - reg_func2!(engine, "+", append, &mut String, String, u32, u64); - reg_func2!(engine, "+", append, &mut String, String, f32, f64, bool); - engine.register_fn("+", append_array); + reg_func2x!(engine, "+", append, String, String, i32, i64, u32, u64, f32, f64, bool); + engine.register_fn("+", |x: String, y: Array| format!("{}{:?}", x, y)); + engine.register_fn("+", |x: String, _: ()| format!("{}", x)); - reg_func2b!(engine, "+", prepend, String, String, i32, i64, u32, u64, f32, f64, bool); - engine.register_fn("+", prepend_array); + reg_func2y!(engine, "+", prepend, String, String, i32, i64, u32, u64, f32, f64, bool); + engine.register_fn("+", |x: Array, y: String| format!("{:?}{}", x, y)); + engine.register_fn("+", |_: (), y: String| format!("{}", y)); // Register array iterator engine.register_iterator::(|a| { diff --git a/src/lib.rs b/src/lib.rs index 4c621796..c002e814 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -48,3 +48,4 @@ mod parser; pub use any::Any; pub use engine::{Engine, EvalAltResult, Scope}; pub use fn_register::{RegisterBoxFn, RegisterFn}; +pub use parser::{ParseError, ParseErrorType, AST}; diff --git a/src/parser.rs b/src/parser.rs index f524e8f0..e8cfa364 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -4,64 +4,86 @@ use std::fmt; use std::iter::Peekable; use std::str::Chars; -#[derive(Debug, Clone)] +#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy)] pub enum LexError { - UnexpectedChar, + UnexpectedChar(char), MalformedEscapeSequence, MalformedNumber, MalformedChar, Nothing, } +type LERR = LexError; + impl Error for LexError { fn description(&self) -> &str { match *self { - LexError::UnexpectedChar => "Unexpected character in input", - LexError::MalformedEscapeSequence => "Unexpected values in escape sequence", - LexError::MalformedNumber => "Unexpected characters in number", - LexError::MalformedChar => "Char constant not a single character", - LexError::Nothing => "This error is for internal use only", + LERR::UnexpectedChar(_) => "Unexpected character", + LERR::MalformedEscapeSequence => "Unexpected values in escape sequence", + LERR::MalformedNumber => "Unexpected characters in number", + LERR::MalformedChar => "Char constant not a single character", + LERR::Nothing => "This error is for internal use only", } } } impl fmt::Display for LexError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.description()) + match self { + LERR::UnexpectedChar(c) => write!(f, "Unexpected '{}'", c), + _ => write!(f, "{}", self.description()), + } } } -#[derive(Debug)] -pub enum ParseError { - BadInput, +#[derive(Debug, PartialEq, Clone)] +pub enum ParseErrorType { + BadInput(String), InputPastEndOfFile, UnknownOperator, - MissingRParen, - MissingLCurly, - MissingRCurly, - MissingRSquare, + MissingRightParen, + MissingLeftBrace, + MissingRightBrace, + MissingRightBracket, MalformedCallExpr, MalformedIndexExpr, - VarExpectsIdentifier, - FnMissingName, + VarExpectsIdentifier(Token), + FnMissingName(Token), FnMissingParams, } +type PERR = ParseErrorType; + +#[derive(Debug, PartialEq, Clone)] +pub struct ParseError(ParseErrorType, usize, usize); + +impl ParseError { + pub fn error_type(&self) -> &ParseErrorType { + &self.0 + } + pub fn line(&self) -> usize { + self.1 + } + pub fn position(&self) -> usize { + self.2 + } +} + impl Error for ParseError { fn description(&self) -> &str { - match *self { - ParseError::BadInput => "Unparseable characters in the input stream", - ParseError::InputPastEndOfFile => "Input past end of file", - ParseError::UnknownOperator => "Unknown operator", - ParseError::MissingRParen => "Expected ')'", - ParseError::MissingLCurly => "Expected '{'", - ParseError::MissingRCurly => "Expected '}'", - ParseError::MissingRSquare => "Expected ']'", - ParseError::MalformedCallExpr => "Call contains bad expression", - ParseError::MalformedIndexExpr => "Indexing expression missing correct index", - ParseError::VarExpectsIdentifier => "'var' expects the name of a variable", - ParseError::FnMissingName => "Function declaration is missing name", - ParseError::FnMissingParams => "Function declaration is missing parameters", + match self.0 { + PERR::BadInput(ref p) => p, + PERR::InputPastEndOfFile => "Script is incomplete", + PERR::UnknownOperator => "Unknown operator", + PERR::MissingRightParen => "Expecting ')'", + PERR::MissingLeftBrace => "Expecting '{'", + PERR::MissingRightBrace => "Expecting '}'", + PERR::MissingRightBracket => "Expecting ']'", + PERR::MalformedCallExpr => "Invalid expression in function call arguments", + PERR::MalformedIndexExpr => "Invalid index in indexing expression", + PERR::VarExpectsIdentifier(_) => "Expecting name of a variable", + PERR::FnMissingName(_) => "Expecting name in function declaration", + PERR::FnMissingParams => "Expecting parameters in function declaration", } } @@ -72,25 +94,47 @@ impl Error for ParseError { impl fmt::Display for ParseError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.description()) + match self.0 { + PERR::BadInput(ref s) => { + write!(f, "{}", s)?; + } + PERR::VarExpectsIdentifier(ref token) | PERR::FnMissingName(ref token) => match token { + Token::None => write!(f, "{}", self.description())?, + _ => write!( + f, + "{} (but gets {:?} token instead)", + self.description(), + token + )?, + }, + _ => write!(f, "{}", self.description())?, + } + + if self.line() > 0 { + write!(f, " at line {}, position {}", self.line(), self.position()) + } else { + write!(f, " but script is incomplete") + } } } -#[derive(Debug, Clone)] +pub struct AST(pub(crate) Vec, pub(crate) Vec); + +#[derive(Debug, PartialEq, Clone)] pub struct FnDef { pub name: String, pub params: Vec, pub body: Box, } -#[derive(Debug, Clone)] +#[derive(Debug, PartialEq, Clone)] pub enum Stmt { If(Box, Box), IfElse(Box, Box, Box), While(Box, Box), Loop(Box), For(String, Box, Box), - Var(String, Option>), + Let(String, Option>), Block(Vec), Expr(Box), Break, @@ -98,14 +142,14 @@ pub enum Stmt { ReturnWithVal(Box), } -#[derive(Debug, Clone)] +#[derive(Debug, PartialEq, Clone)] pub enum Expr { - IntConst(i64), - FloatConst(f64), + IntegerConstant(i64), + FloatConstant(f64), Identifier(String), - CharConst(char), - StringConst(String), - FnCall(String, Vec), + CharConstant(char), + StringConstant(String), + FunctionCall(String, Vec), Assignment(Box, Box), Dot(Box, Box), Index(String, Box), @@ -115,33 +159,34 @@ pub enum Expr { Unit, } -#[derive(Debug, Clone)] +#[derive(Debug, PartialEq, Clone)] pub enum Token { - IntConst(i64), - FloatConst(f64), + None, + IntegerConstant(i64), + FloatConstant(f64), Identifier(String), - CharConst(char), + CharConstant(char), StringConst(String), - LCurly, - RCurly, - LParen, - RParen, - LSquare, - RSquare, + LeftBrace, + RightBrace, + LeftParen, + RightParen, + LeftBracket, + RightBracket, Plus, UnaryPlus, Minus, UnaryMinus, Multiply, Divide, - Semicolon, + SemiColon, Colon, Comma, Period, Equals, True, False, - Var, + Let, If, Else, While, @@ -149,10 +194,10 @@ pub enum Token { LessThan, GreaterThan, Bang, - LessThanEqual, - GreaterThanEqual, - EqualTo, - NotEqualTo, + LessThanEqualsTo, + GreaterThanEqualsTo, + EqualsTo, + NotEqualsTo, Pipe, Or, Ampersand, @@ -188,12 +233,12 @@ impl Token { use self::Token::*; match *self { - LCurly | // (+expr) - is unary - // RCurly | {expr} - expr not unary & is closing - LParen | // {-expr} - is unary - // RParen | (expr) - expr not unary & is closing - LSquare | // [-expr] - is unary - // RSquare | [expr] - expr not unary & is closing + LeftBrace | // (+expr) - is unary + // RightBrace | {expr} - expr not unary & is closing + LeftParen | // {-expr} - is unary + // RightParen | (expr) - expr not unary & is closing + LeftBracket | // [-expr] - is unary + // RightBracket | [expr] - expr not unary & is closing Plus | UnaryPlus | Minus | @@ -207,10 +252,10 @@ impl Token { LessThan | GreaterThan | Bang | - LessThanEqual | - GreaterThanEqual | - EqualTo | - NotEqualTo | + LessThanEqualsTo | + GreaterThanEqualsTo | + EqualsTo | + NotEqualsTo | Pipe | Or | Ampersand | @@ -244,9 +289,9 @@ impl Token { use self::Token::*; match *self { - RCurly | - RParen | - RSquare | + RightBrace | + RightParen | + RightBracket | Plus | Minus | Multiply | @@ -256,10 +301,10 @@ impl Token { Equals | LessThan | GreaterThan | - LessThanEqual | - GreaterThanEqual | - EqualTo | - NotEqualTo | + LessThanEqualsTo | + GreaterThanEqualsTo | + EqualsTo | + NotEqualsTo | Pipe | Or | Ampersand | @@ -282,15 +327,33 @@ impl Token { pub struct TokenIterator<'a> { last: Token, + line: usize, + pos: usize, char_stream: Peekable>, } impl<'a> TokenIterator<'a> { - pub fn parse_string_const(&mut self, enclosing_char: char) -> Result { + fn advance(&mut self) { + self.pos += 1; + } + fn advance_line(&mut self) { + self.line += 1; + self.pos = 0; + } + + pub fn parse_string_const( + &mut self, + enclosing_char: char, + ) -> Result { let mut result = Vec::new(); let mut escape = false; while let Some(nxt) = self.char_stream.next() { + self.advance(); + if nxt == '\n' { + self.advance_line(); + } + match nxt { '\\' if !escape => escape = true, '\\' if escape => { @@ -318,17 +381,18 @@ impl<'a> TokenIterator<'a> { out_val *= 16; out_val += d1; } else { - return Err(LexError::MalformedEscapeSequence); + return Err((LERR::MalformedEscapeSequence, self.line, self.pos)); } } else { - return Err(LexError::MalformedEscapeSequence); + return Err((LERR::MalformedEscapeSequence, self.line, self.pos)); } + self.advance(); } if let Some(r) = char::from_u32(out_val) { result.push(r); } else { - return Err(LexError::MalformedEscapeSequence); + return Err((LERR::MalformedEscapeSequence, self.line, self.pos)); } } 'u' if escape => { @@ -340,17 +404,18 @@ impl<'a> TokenIterator<'a> { out_val *= 16; out_val += d1; } else { - return Err(LexError::MalformedEscapeSequence); + return Err((LERR::MalformedEscapeSequence, self.line, self.pos)); } } else { - return Err(LexError::MalformedEscapeSequence); + return Err((LERR::MalformedEscapeSequence, self.line, self.pos)); } + self.advance(); } if let Some(r) = char::from_u32(out_val) { result.push(r); } else { - return Err(LexError::MalformedEscapeSequence); + return Err((LERR::MalformedEscapeSequence, self.line, self.pos)); } } 'U' if escape => { @@ -362,25 +427,26 @@ impl<'a> TokenIterator<'a> { out_val *= 16; out_val += d1; } else { - return Err(LexError::MalformedEscapeSequence); + return Err((LERR::MalformedEscapeSequence, self.line, self.pos)); } } else { - return Err(LexError::MalformedEscapeSequence); + return Err((LERR::MalformedEscapeSequence, self.line, self.pos)); } + self.advance(); } if let Some(r) = char::from_u32(out_val) { result.push(r); } else { - return Err(LexError::MalformedEscapeSequence); + return Err((LERR::MalformedEscapeSequence, self.line, self.pos)); } } x if enclosing_char == x && escape => result.push(x), x if enclosing_char == x && !escape => break, - _ if escape => return Err(LexError::MalformedEscapeSequence), - _ => { + _ if escape => return Err((LERR::MalformedEscapeSequence, self.line, self.pos)), + x => { escape = false; - result.push(nxt); + result.push(x); } } } @@ -389,9 +455,15 @@ impl<'a> TokenIterator<'a> { Ok(out) } - fn inner_next(&mut self) -> Option { + fn inner_next(&mut self) -> Option<(Token, usize, usize)> { while let Some(c) = self.char_stream.next() { + self.advance(); + + let line = self.line; + let pos = self.pos; + match c { + '\n' => self.advance_line(), '0'..='9' => { let mut result = Vec::new(); let mut radix_base: Option = None; @@ -402,15 +474,18 @@ impl<'a> TokenIterator<'a> { '0'..='9' => { result.push(nxt); self.char_stream.next(); + self.advance(); } '.' => { result.push(nxt); self.char_stream.next(); + self.advance(); while let Some(&nxt_float) = self.char_stream.peek() { match nxt_float { '0'..='9' => { result.push(nxt_float); self.char_stream.next(); + self.advance(); } _ => break, } @@ -419,11 +494,13 @@ impl<'a> TokenIterator<'a> { 'x' | 'X' => { result.push(nxt); self.char_stream.next(); + self.advance(); while let Some(&nxt_hex) = self.char_stream.peek() { match nxt_hex { '0'..='9' | 'a'..='f' | 'A'..='F' => { result.push(nxt_hex); self.char_stream.next(); + self.advance(); } _ => break, } @@ -433,11 +510,13 @@ impl<'a> TokenIterator<'a> { 'o' | 'O' => { result.push(nxt); self.char_stream.next(); + self.advance(); while let Some(&nxt_oct) = self.char_stream.peek() { match nxt_oct { '0'..='8' => { result.push(nxt_oct); self.char_stream.next(); + self.advance(); } _ => break, } @@ -447,11 +526,13 @@ impl<'a> TokenIterator<'a> { 'b' | 'B' => { result.push(nxt); self.char_stream.next(); + self.advance(); while let Some(&nxt_bin) = self.char_stream.peek() { match nxt_bin { '0' | '1' | '_' => { result.push(nxt_bin); self.char_stream.next(); + self.advance(); } _ => break, } @@ -470,18 +551,23 @@ impl<'a> TokenIterator<'a> { .filter(|c| c != &'_') .collect(); if let Ok(val) = i64::from_str_radix(&out, radix) { - return Some(Token::IntConst(val)); + return Some((Token::IntegerConstant(val), line, pos)); } } let out: String = result.iter().cloned().collect(); - if let Ok(val) = out.parse::() { - return Some(Token::IntConst(val)); - } else if let Ok(val) = out.parse::() { - return Some(Token::FloatConst(val)); - } - return Some(Token::LexErr(LexError::MalformedNumber)); + return Some(( + if let Ok(val) = out.parse::() { + Token::IntegerConstant(val) + } else if let Ok(val) = out.parse::() { + Token::FloatConstant(val) + } else { + Token::LexErr(LERR::MalformedNumber) + }, + line, + pos, + )); } 'A'..='Z' | 'a'..='z' | '_' => { let mut result = Vec::new(); @@ -492,107 +578,144 @@ impl<'a> TokenIterator<'a> { x if x.is_alphanumeric() || x == '_' => { result.push(x); self.char_stream.next(); + self.advance(); } _ => break, } } let out: String = result.iter().cloned().collect(); - match out.as_ref() { - "true" => return Some(Token::True), - "false" => return Some(Token::False), - "let" => return Some(Token::Var), - "if" => return Some(Token::If), - "else" => return Some(Token::Else), - "while" => return Some(Token::While), - "loop" => return Some(Token::Loop), - "break" => return Some(Token::Break), - "return" => return Some(Token::Return), - "fn" => return Some(Token::Fn), - "for" => return Some(Token::For), - "in" => return Some(Token::In), - x => return Some(Token::Identifier(x.to_string())), + + return Some(( + match out.as_ref() { + "true" => Token::True, + "false" => Token::False, + "let" => Token::Let, + "if" => Token::If, + "else" => Token::Else, + "while" => Token::While, + "loop" => Token::Loop, + "break" => Token::Break, + "return" => Token::Return, + "fn" => Token::Fn, + "for" => Token::For, + "in" => Token::In, + x => Token::Identifier(x.to_string()), + }, + line, + pos, + )); + } + '"' => { + return match self.parse_string_const('"') { + Ok(out) => Some((Token::StringConst(out), line, pos)), + Err(e) => Some((Token::LexErr(e.0), e.1, e.2)), } } - '"' => match self.parse_string_const('"') { - Ok(out) => return Some(Token::StringConst(out)), - Err(e) => return Some(Token::LexErr(e)), - }, '\'' => match self.parse_string_const('\'') { Ok(result) => { let mut chars = result.chars(); - if let Some(out) = chars.next() { - println!("result: {}", result); - if chars.count() != 0 { - return Some(Token::LexErr(LexError::MalformedChar)); - } - return Some(Token::CharConst(out)); - } else { - return Some(Token::LexErr(LexError::MalformedChar)); - } + return Some(( + if let Some(out) = chars.next() { + if chars.count() != 0 { + Token::LexErr(LERR::MalformedChar) + } else { + Token::CharConstant(out) + } + } else { + Token::LexErr(LERR::MalformedChar) + }, + line, + pos, + )); } - Err(e) => return Some(Token::LexErr(e)), + Err(e) => return Some((Token::LexErr(e.0), e.1, e.2)), }, - '{' => return Some(Token::LCurly), - '}' => return Some(Token::RCurly), - '(' => return Some(Token::LParen), - ')' => return Some(Token::RParen), - '[' => return Some(Token::LSquare), - ']' => return Some(Token::RSquare), + '{' => return Some((Token::LeftBrace, line, pos)), + '}' => return Some((Token::RightBrace, line, pos)), + '(' => return Some((Token::LeftParen, line, pos)), + ')' => return Some((Token::RightParen, line, pos)), + '[' => return Some((Token::LeftBracket, line, pos)), + ']' => return Some((Token::RightBracket, line, pos)), '+' => { - return match self.char_stream.peek() { - Some(&'=') => { - self.char_stream.next(); - Some(Token::PlusAssign) - } - _ if self.last.is_next_unary() => Some(Token::UnaryPlus), - _ => Some(Token::Plus), - } + return Some(( + match self.char_stream.peek() { + Some(&'=') => { + self.char_stream.next(); + self.advance(); + Token::PlusAssign + } + _ if self.last.is_next_unary() => Token::UnaryPlus, + _ => Token::Plus, + }, + line, + pos, + )) } '-' => { - return match self.char_stream.peek() { - Some(&'=') => { - self.char_stream.next(); - Some(Token::MinusAssign) - } - _ if self.last.is_next_unary() => Some(Token::UnaryMinus), - _ => Some(Token::Minus), - } + return Some(( + match self.char_stream.peek() { + Some(&'=') => { + self.char_stream.next(); + self.advance(); + Token::MinusAssign + } + _ if self.last.is_next_unary() => Token::UnaryMinus, + _ => Token::Minus, + }, + line, + pos, + )) } '*' => { - return match self.char_stream.peek() { - Some(&'=') => { - self.char_stream.next(); - Some(Token::MultiplyAssign) - } - _ => Some(Token::Multiply), - } + return Some(( + match self.char_stream.peek() { + Some(&'=') => { + self.char_stream.next(); + self.advance(); + Token::MultiplyAssign + } + _ => Token::Multiply, + }, + line, + pos, + )) } '/' => match self.char_stream.peek() { Some(&'/') => { self.char_stream.next(); + self.advance(); while let Some(c) = self.char_stream.next() { if c == '\n' { + self.advance_line(); break; + } else { + self.advance(); } } } Some(&'*') => { let mut level = 1; self.char_stream.next(); + self.advance(); while let Some(c) = self.char_stream.next() { + self.advance(); + match c { '/' => { if let Some('*') = self.char_stream.next() { level += 1; } + self.advance(); } '*' => { if let Some('/') = self.char_stream.next() { level -= 1; } + self.advance(); } + '\n' => self.advance_line(), _ => (), } @@ -603,113 +726,173 @@ impl<'a> TokenIterator<'a> { } Some(&'=') => { self.char_stream.next(); - return Some(Token::DivideAssign); + self.advance(); + return Some((Token::DivideAssign, line, pos)); } - _ => return Some(Token::Divide), + _ => return Some((Token::Divide, line, pos)), }, - ';' => return Some(Token::Semicolon), - ':' => return Some(Token::Colon), - ',' => return Some(Token::Comma), - '.' => return Some(Token::Period), + ';' => return Some((Token::SemiColon, line, pos)), + ':' => return Some((Token::Colon, line, pos)), + ',' => return Some((Token::Comma, line, pos)), + '.' => return Some((Token::Period, line, pos)), '=' => match self.char_stream.peek() { Some(&'=') => { self.char_stream.next(); - return Some(Token::EqualTo); + self.advance(); + return Some((Token::EqualsTo, line, pos)); } - _ => return Some(Token::Equals), + _ => return Some((Token::Equals, line, pos)), }, '<' => match self.char_stream.peek() { Some(&'=') => { self.char_stream.next(); - return Some(Token::LessThanEqual); + self.advance(); + return Some((Token::LessThanEqualsTo, line, pos)); } Some(&'<') => { self.char_stream.next(); + self.advance(); return match self.char_stream.peek() { Some(&'=') => { self.char_stream.next(); - Some(Token::LeftShiftAssign) + self.advance(); + Some((Token::LeftShiftAssign, line, pos)) } _ => { self.char_stream.next(); - Some(Token::LeftShift) + self.advance(); + Some((Token::LeftShift, line, pos)) } }; } - _ => return Some(Token::LessThan), + _ => return Some((Token::LessThan, line, pos)), }, - '>' => match self.char_stream.peek() { - Some(&'=') => { - self.char_stream.next(); - return Some(Token::GreaterThanEqual); - } - Some(&'>') => { - self.char_stream.next(); - return match self.char_stream.peek() { + '>' => { + return Some(( + match self.char_stream.peek() { Some(&'=') => { self.char_stream.next(); - Some(Token::RightShiftAssign) + self.advance(); + Token::GreaterThanEqualsTo } - _ => { + Some(&'>') => { self.char_stream.next(); - Some(Token::RightShift) + self.advance(); + match self.char_stream.peek() { + Some(&'=') => { + self.char_stream.next(); + self.advance(); + Token::RightShiftAssign + } + _ => { + self.char_stream.next(); + self.advance(); + Token::RightShift + } + } } - }; - } - _ => return Some(Token::GreaterThan), - }, - '!' => match self.char_stream.peek() { - Some(&'=') => { - self.char_stream.next(); - return Some(Token::NotEqualTo); - } - _ => return Some(Token::Bang), - }, - '|' => match self.char_stream.peek() { - Some(&'|') => { - self.char_stream.next(); - return Some(Token::Or); - } - Some(&'=') => { - self.char_stream.next(); - return Some(Token::OrAssign); - } - _ => return Some(Token::Pipe), - }, - '&' => match self.char_stream.peek() { - Some(&'&') => { - self.char_stream.next(); - return Some(Token::And); - } - Some(&'=') => { - self.char_stream.next(); - return Some(Token::AndAssign); - } - _ => return Some(Token::Ampersand), - }, - '^' => match self.char_stream.peek() { - Some(&'=') => { - self.char_stream.next(); - return Some(Token::XOrAssign); - } - _ => return Some(Token::XOr), - }, - '%' => match self.char_stream.peek() { - Some(&'=') => { - self.char_stream.next(); - return Some(Token::ModuloAssign); - } - _ => return Some(Token::Modulo), - }, - '~' => match self.char_stream.peek() { - Some(&'=') => { - self.char_stream.next(); - return Some(Token::PowerOfAssign); - } - _ => return Some(Token::PowerOf), - }, - _x if _x.is_whitespace() => (), - _ => return Some(Token::LexErr(LexError::UnexpectedChar)), + _ => Token::GreaterThan, + }, + line, + pos, + )) + } + '!' => { + return Some(( + match self.char_stream.peek() { + Some(&'=') => { + self.char_stream.next(); + self.advance(); + Token::NotEqualsTo + } + _ => Token::Bang, + }, + line, + pos, + )) + } + '|' => { + return Some(( + match self.char_stream.peek() { + Some(&'|') => { + self.char_stream.next(); + self.advance(); + Token::Or + } + Some(&'=') => { + self.char_stream.next(); + self.advance(); + Token::OrAssign + } + _ => Token::Pipe, + }, + line, + pos, + )) + } + '&' => { + return Some(( + match self.char_stream.peek() { + Some(&'&') => { + self.char_stream.next(); + self.advance(); + Token::And + } + Some(&'=') => { + self.char_stream.next(); + self.advance(); + Token::AndAssign + } + _ => Token::Ampersand, + }, + line, + pos, + )) + } + '^' => { + return Some(( + match self.char_stream.peek() { + Some(&'=') => { + self.char_stream.next(); + self.advance(); + Token::XOrAssign + } + _ => Token::XOr, + }, + line, + pos, + )) + } + '%' => { + return Some(( + match self.char_stream.peek() { + Some(&'=') => { + self.char_stream.next(); + self.advance(); + Token::ModuloAssign + } + _ => Token::Modulo, + }, + line, + pos, + )) + } + '~' => { + return Some(( + match self.char_stream.peek() { + Some(&'=') => { + self.char_stream.next(); + self.advance(); + Token::PowerOfAssign + } + _ => Token::PowerOf, + }, + line, + pos, + )) + } + x if x.is_whitespace() => (), + x => return Some((Token::LexErr(LERR::UnexpectedChar(x)), line, pos)), } } @@ -718,21 +901,22 @@ impl<'a> TokenIterator<'a> { } impl<'a> Iterator for TokenIterator<'a> { - type Item = Token; + type Item = (Token, usize, usize); // TODO - perhaps this could be optimized? fn next(&mut self) -> Option { - self.last = match self.inner_next() { - Some(c) => c, - None => return None, - }; - Some(self.last.clone()) + self.inner_next().map(|x| { + self.last = x.0.clone(); + x + }) } } pub fn lex(input: &str) -> TokenIterator<'_> { TokenIterator { - last: Token::LexErr(LexError::Nothing), + last: Token::LexErr(LERR::Nothing), + line: 1, + pos: 0, char_stream: input.chars().peekable(), } } @@ -754,11 +938,11 @@ fn get_precedence(token: &Token) -> i32 { Token::Or | Token::XOr | Token::Pipe => 11, Token::And | Token::Ampersand => 12, Token::LessThan - | Token::LessThanEqual + | Token::LessThanEqualsTo | Token::GreaterThan - | Token::GreaterThanEqual - | Token::EqualTo - | Token::NotEqualTo => 15, + | Token::GreaterThanEqualsTo + | Token::EqualsTo + | Token::NotEqualsTo => 15, Token::Plus | Token::Minus => 20, Token::Divide | Token::Multiply | Token::PowerOf => 40, Token::LeftShift | Token::RightShift => 50, @@ -769,11 +953,19 @@ fn get_precedence(token: &Token) -> i32 { } fn parse_paren_expr<'a>(input: &mut Peekable>) -> Result { + match input.peek() { + Some((Token::RightParen, _, _)) => { + input.next(); + return Ok(Expr::Unit); + } + _ => (), + } + let expr = parse_expr(input)?; match input.next() { - Some(Token::RParen) => Ok(expr), - _ => Err(ParseError::MissingRParen), + Some((Token::RightParen, _, _)) => Ok(expr), + _ => Err(ParseError(PERR::MissingRightParen, 0, 0)), } } @@ -783,25 +975,28 @@ fn parse_call_expr<'a>( ) -> Result { let mut args = Vec::new(); - if let Some(&Token::RParen) = input.peek() { + if let Some(&(Token::RightParen, _, _)) = input.peek() { input.next(); - return Ok(Expr::FnCall(id, args)); + return Ok(Expr::FunctionCall(id, args)); } loop { - if let Ok(arg) = parse_expr(input) { - args.push(arg); - } else { - return Err(ParseError::MalformedCallExpr); + match parse_expr(input) { + Ok(arg) => args.push(arg), + Err(mut err) => { + err.0 = PERR::MalformedCallExpr; + return Err(err); + } } match input.peek() { - Some(&Token::RParen) => { + Some(&(Token::RightParen, _, _)) => { input.next(); - return Ok(Expr::FnCall(id, args)); + return Ok(Expr::FunctionCall(id, args)); } - Some(&Token::Comma) => (), - _ => return Err(ParseError::MalformedCallExpr), + Some(&(Token::Comma, _, _)) => (), + Some(&(_, line, pos)) => return Err(ParseError(PERR::MalformedCallExpr, line, pos)), + None => return Err(ParseError(PERR::MalformedCallExpr, 0, 0)), } input.next(); @@ -812,16 +1007,19 @@ fn parse_index_expr<'a>( id: String, input: &mut Peekable>, ) -> Result { - if let Ok(idx) = parse_expr(input) { - match input.peek() { - Some(&Token::RSquare) => { + match parse_expr(input) { + Ok(idx) => match input.peek() { + Some(&(Token::RightBracket, _, _)) => { input.next(); return Ok(Expr::Index(id, Box::new(idx))); } - _ => return Err(ParseError::MalformedIndexExpr), + Some(&(_, line, pos)) => return Err(ParseError(PERR::MalformedIndexExpr, line, pos)), + None => return Err(ParseError(PERR::MalformedIndexExpr, 0, 0)), + }, + Err(mut err) => { + err.0 = PERR::MalformedIndexExpr; + return Err(err); } - } else { - return Err(ParseError::MalformedIndexExpr); } } @@ -830,11 +1028,11 @@ fn parse_ident_expr<'a>( input: &mut Peekable>, ) -> Result { match input.peek() { - Some(&Token::LParen) => { + Some(&(Token::LeftParen, _, _)) => { input.next(); parse_call_expr(id, input) } - Some(&Token::LSquare) => { + Some(&(Token::LeftBracket, _, _)) => { input.next(); parse_index_expr(id, input) } @@ -846,68 +1044,69 @@ fn parse_array_expr<'a>(input: &mut Peekable>) -> Result true, + Some(&(Token::RightBracket, _, _)) => true, _ => false, }; if !skip_contents { while let Some(_) = input.peek() { arr.push(parse_expr(input)?); - if let Some(&Token::Comma) = input.peek() { + if let Some(&(Token::Comma, _, _)) = input.peek() { input.next(); } - if let Some(&Token::RSquare) = input.peek() { + if let Some(&(Token::RightBracket, _, _)) = input.peek() { break; } } } match input.peek() { - Some(&Token::RSquare) => { + Some(&(Token::RightBracket, _, _)) => { input.next(); Ok(Expr::Array(arr)) } - _ => Err(ParseError::MissingRSquare), + Some(&(_, line, pos)) => Err(ParseError(PERR::MissingRightBracket, line, pos)), + None => Err(ParseError(PERR::MissingRightBracket, 0, 0)), } } fn parse_primary<'a>(input: &mut Peekable>) -> Result { - if let Some(token) = input.next() { - match token { - Token::IntConst(ref x) => Ok(Expr::IntConst(*x)), - Token::FloatConst(ref x) => Ok(Expr::FloatConst(*x)), - Token::StringConst(ref s) => Ok(Expr::StringConst(s.clone())), - Token::CharConst(ref c) => Ok(Expr::CharConst(*c)), + match input.next() { + Some((token, line, pos)) => match token { + Token::IntegerConstant(ref x) => Ok(Expr::IntegerConstant(*x)), + Token::FloatConstant(ref x) => Ok(Expr::FloatConstant(*x)), + Token::StringConst(ref s) => Ok(Expr::StringConstant(s.clone())), + Token::CharConstant(ref c) => Ok(Expr::CharConstant(*c)), Token::Identifier(ref s) => parse_ident_expr(s.clone(), input), - Token::LParen => parse_paren_expr(input), - Token::LSquare => parse_array_expr(input), + Token::LeftParen => parse_paren_expr(input), + Token::LeftBracket => parse_array_expr(input), Token::True => Ok(Expr::True), Token::False => Ok(Expr::False), - Token::LexErr(le) => { - println!("Error: {}", le); - Err(ParseError::BadInput) - } - _ => { - println!("Can't parse: {:?}", token); - Err(ParseError::BadInput) - } - } - } else { - Err(ParseError::InputPastEndOfFile) + Token::LexErr(le) => Err(ParseError(PERR::BadInput(le.to_string()), line, pos)), + _ => Err(ParseError( + PERR::BadInput(format!("Unexpected {:?} token", token)), + line, + pos, + )), + }, + None => Err(ParseError(PERR::InputPastEndOfFile, 0, 0)), } } fn parse_unary<'a>(input: &mut Peekable>) -> Result { - let tok = match input.peek() { - Some(tok) => tok.clone(), - None => return Err(ParseError::InputPastEndOfFile), + let token = match input.peek() { + Some((tok, _, _)) => tok.clone(), + None => return Err(ParseError(PERR::InputPastEndOfFile, 0, 0)), }; - match tok { + match token { Token::UnaryMinus => { input.next(); - Ok(Expr::FnCall("-".to_string(), vec![parse_primary(input)?])) + Ok(Expr::FunctionCall( + "-".to_string(), + vec![parse_primary(input)?], + )) } Token::UnaryPlus => { input.next(); @@ -915,7 +1114,10 @@ fn parse_unary<'a>(input: &mut Peekable>) -> Result { input.next(); - Ok(Expr::FnCall("!".to_string(), vec![parse_primary(input)?])) + Ok(Expr::FunctionCall( + "!".to_string(), + vec![parse_primary(input)?], + )) } _ => parse_primary(input), } @@ -931,7 +1133,7 @@ fn parse_binop<'a>( loop { let mut curr_prec = -1; - if let Some(curr_op) = input.peek() { + if let Some(&(ref curr_op, _, _)) = input.peek() { curr_prec = get_precedence(curr_op); } @@ -939,12 +1141,12 @@ fn parse_binop<'a>( return Ok(lhs_curr); } - if let Some(op_token) = input.next() { + if let Some((op_token, line, pos)) = input.next() { let mut rhs = parse_unary(input)?; let mut next_prec = -1; - if let Some(next_op) = input.peek() { + if let Some(&(ref next_op, _, _)) = input.peek() { next_prec = get_precedence(next_op); } @@ -956,119 +1158,117 @@ fn parse_binop<'a>( } lhs_curr = match op_token { - Token::Plus => Expr::FnCall("+".to_string(), vec![lhs_curr, rhs]), - Token::Minus => Expr::FnCall("-".to_string(), vec![lhs_curr, rhs]), - Token::Multiply => Expr::FnCall("*".to_string(), vec![lhs_curr, rhs]), - Token::Divide => Expr::FnCall("/".to_string(), vec![lhs_curr, rhs]), + Token::Plus => Expr::FunctionCall("+".to_string(), vec![lhs_curr, rhs]), + Token::Minus => Expr::FunctionCall("-".to_string(), vec![lhs_curr, rhs]), + Token::Multiply => Expr::FunctionCall("*".to_string(), vec![lhs_curr, rhs]), + Token::Divide => Expr::FunctionCall("/".to_string(), vec![lhs_curr, rhs]), Token::Equals => Expr::Assignment(Box::new(lhs_curr), Box::new(rhs)), Token::PlusAssign => { let lhs_copy = lhs_curr.clone(); Expr::Assignment( Box::new(lhs_curr), - Box::new(Expr::FnCall("+".to_string(), vec![lhs_copy, rhs])), + Box::new(Expr::FunctionCall("+".to_string(), vec![lhs_copy, rhs])), ) } Token::MinusAssign => { let lhs_copy = lhs_curr.clone(); Expr::Assignment( Box::new(lhs_curr), - Box::new(Expr::FnCall("-".to_string(), vec![lhs_copy, rhs])), + Box::new(Expr::FunctionCall("-".to_string(), vec![lhs_copy, rhs])), ) } Token::Period => Expr::Dot(Box::new(lhs_curr), Box::new(rhs)), - Token::EqualTo => Expr::FnCall("==".to_string(), vec![lhs_curr, rhs]), - Token::NotEqualTo => Expr::FnCall("!=".to_string(), vec![lhs_curr, rhs]), - Token::LessThan => Expr::FnCall("<".to_string(), vec![lhs_curr, rhs]), - Token::LessThanEqual => Expr::FnCall("<=".to_string(), vec![lhs_curr, rhs]), - Token::GreaterThan => Expr::FnCall(">".to_string(), vec![lhs_curr, rhs]), - Token::GreaterThanEqual => Expr::FnCall(">=".to_string(), vec![lhs_curr, rhs]), - Token::Or => Expr::FnCall("||".to_string(), vec![lhs_curr, rhs]), - Token::And => Expr::FnCall("&&".to_string(), vec![lhs_curr, rhs]), - Token::XOr => Expr::FnCall("^".to_string(), vec![lhs_curr, rhs]), + Token::EqualsTo => Expr::FunctionCall("==".to_string(), vec![lhs_curr, rhs]), + Token::NotEqualsTo => Expr::FunctionCall("!=".to_string(), vec![lhs_curr, rhs]), + Token::LessThan => Expr::FunctionCall("<".to_string(), vec![lhs_curr, rhs]), + Token::LessThanEqualsTo => { + Expr::FunctionCall("<=".to_string(), vec![lhs_curr, rhs]) + } + Token::GreaterThan => Expr::FunctionCall(">".to_string(), vec![lhs_curr, rhs]), + Token::GreaterThanEqualsTo => { + Expr::FunctionCall(">=".to_string(), vec![lhs_curr, rhs]) + } + Token::Or => Expr::FunctionCall("||".to_string(), vec![lhs_curr, rhs]), + Token::And => Expr::FunctionCall("&&".to_string(), vec![lhs_curr, rhs]), + Token::XOr => Expr::FunctionCall("^".to_string(), vec![lhs_curr, rhs]), Token::OrAssign => { let lhs_copy = lhs_curr.clone(); Expr::Assignment( Box::new(lhs_curr), - Box::new(Expr::FnCall("|".to_string(), vec![lhs_copy, rhs])), + Box::new(Expr::FunctionCall("|".to_string(), vec![lhs_copy, rhs])), ) } Token::AndAssign => { let lhs_copy = lhs_curr.clone(); Expr::Assignment( Box::new(lhs_curr), - Box::new(Expr::FnCall("&".to_string(), vec![lhs_copy, rhs])), + Box::new(Expr::FunctionCall("&".to_string(), vec![lhs_copy, rhs])), ) } Token::XOrAssign => { let lhs_copy = lhs_curr.clone(); Expr::Assignment( Box::new(lhs_curr), - Box::new(Expr::FnCall("^".to_string(), vec![lhs_copy, rhs])), + Box::new(Expr::FunctionCall("^".to_string(), vec![lhs_copy, rhs])), ) } Token::MultiplyAssign => { let lhs_copy = lhs_curr.clone(); Expr::Assignment( Box::new(lhs_curr), - Box::new(Expr::FnCall("*".to_string(), vec![lhs_copy, rhs])), + Box::new(Expr::FunctionCall("*".to_string(), vec![lhs_copy, rhs])), ) } Token::DivideAssign => { let lhs_copy = lhs_curr.clone(); Expr::Assignment( Box::new(lhs_curr), - Box::new(Expr::FnCall("/".to_string(), vec![lhs_copy, rhs])), + Box::new(Expr::FunctionCall("/".to_string(), vec![lhs_copy, rhs])), ) } - Token::Pipe => Expr::FnCall("|".to_string(), vec![lhs_curr, rhs]), - Token::LeftShift => Expr::FnCall("<<".to_string(), vec![lhs_curr, rhs]), - Token::RightShift => Expr::FnCall(">>".to_string(), vec![lhs_curr, rhs]), + Token::Pipe => Expr::FunctionCall("|".to_string(), vec![lhs_curr, rhs]), + Token::LeftShift => Expr::FunctionCall("<<".to_string(), vec![lhs_curr, rhs]), + Token::RightShift => Expr::FunctionCall(">>".to_string(), vec![lhs_curr, rhs]), Token::LeftShiftAssign => { let lhs_copy = lhs_curr.clone(); Expr::Assignment( Box::new(lhs_curr), - Box::new(Expr::FnCall("<<".to_string(), vec![lhs_copy, rhs])), + Box::new(Expr::FunctionCall("<<".to_string(), vec![lhs_copy, rhs])), ) } Token::RightShiftAssign => { let lhs_copy = lhs_curr.clone(); Expr::Assignment( Box::new(lhs_curr), - Box::new(Expr::FnCall(">>".to_string(), vec![lhs_copy, rhs])), + Box::new(Expr::FunctionCall(">>".to_string(), vec![lhs_copy, rhs])), ) } - Token::Ampersand => Expr::FnCall("&".to_string(), vec![lhs_curr, rhs]), - Token::Modulo => Expr::FnCall("%".to_string(), vec![lhs_curr, rhs]), + Token::Ampersand => Expr::FunctionCall("&".to_string(), vec![lhs_curr, rhs]), + Token::Modulo => Expr::FunctionCall("%".to_string(), vec![lhs_curr, rhs]), Token::ModuloAssign => { let lhs_copy = lhs_curr.clone(); Expr::Assignment( Box::new(lhs_curr), - Box::new(Expr::FnCall("%".to_string(), vec![lhs_copy, rhs])), + Box::new(Expr::FunctionCall("%".to_string(), vec![lhs_copy, rhs])), ) } - Token::PowerOf => Expr::FnCall("~".to_string(), vec![lhs_curr, rhs]), + Token::PowerOf => Expr::FunctionCall("~".to_string(), vec![lhs_curr, rhs]), Token::PowerOfAssign => { let lhs_copy = lhs_curr.clone(); Expr::Assignment( Box::new(lhs_curr), - Box::new(Expr::FnCall("~".to_string(), vec![lhs_copy, rhs])), + Box::new(Expr::FunctionCall("~".to_string(), vec![lhs_copy, rhs])), ) } - _ => return Err(ParseError::UnknownOperator), + _ => return Err(ParseError(PERR::UnknownOperator, line, pos)), }; } } } fn parse_expr<'a>(input: &mut Peekable>) -> Result { - match input.peek() { - Some(Token::RParen) => Ok(Expr::Unit), - _ => { - let lhs = parse_unary(input)?; - - parse_binop(input, 0, lhs) - } - } + let lhs = parse_unary(input)?; + parse_binop(input, 0, lhs) } fn parse_if<'a>(input: &mut Peekable>) -> Result { @@ -1078,7 +1278,7 @@ fn parse_if<'a>(input: &mut Peekable>) -> Result { + Some(&(Token::Else, _, _)) => { input.next(); let else_body = parse_block(input)?; Ok(Stmt::IfElse( @@ -1112,13 +1312,19 @@ fn parse_for<'a>(input: &mut Peekable>) -> Result s.clone(), - _ => return Err(ParseError::VarExpectsIdentifier), + Some((Token::Identifier(ref s), _, _)) => s.clone(), + Some((token, line, pos)) => { + return Err(ParseError(PERR::VarExpectsIdentifier(token), line, pos)) + } + None => return Err(ParseError(PERR::VarExpectsIdentifier(Token::None), 0, 0)), }; match input.next() { - Some(Token::In) => {} - _ => return Err(ParseError::VarExpectsIdentifier), + Some((Token::In, _, _)) => {} + Some((token, line, pos)) => { + return Err(ParseError(PERR::VarExpectsIdentifier(token), line, pos)) + } + None => return Err(ParseError(PERR::VarExpectsIdentifier(Token::None), 0, 0)), } let expr = parse_expr(input)?; @@ -1132,24 +1338,28 @@ fn parse_var<'a>(input: &mut Peekable>) -> Result s.clone(), - _ => return Err(ParseError::VarExpectsIdentifier), + Some((Token::Identifier(ref s), _, _)) => s.clone(), + Some((token, line, pos)) => { + return Err(ParseError(PERR::VarExpectsIdentifier(token), line, pos)) + } + None => return Err(ParseError(PERR::VarExpectsIdentifier(Token::None), 0, 0)), }; match input.peek() { - Some(&Token::Equals) => { + Some(&(Token::Equals, _, _)) => { input.next(); let initializer = parse_expr(input)?; - Ok(Stmt::Var(name, Some(Box::new(initializer)))) + Ok(Stmt::Let(name, Some(Box::new(initializer)))) } - _ => Ok(Stmt::Var(name, None)), + _ => Ok(Stmt::Let(name, None)), } } fn parse_block<'a>(input: &mut Peekable>) -> Result { match input.peek() { - Some(&Token::LCurly) => (), - _ => return Err(ParseError::MissingLCurly), + Some(&(Token::LeftBrace, _, _)) => (), + Some(&(_, line, pos)) => return Err(ParseError(PERR::MissingLeftBrace, line, pos)), + None => return Err(ParseError(PERR::MissingLeftBrace, 0, 0)), } input.next(); @@ -1157,7 +1367,7 @@ fn parse_block<'a>(input: &mut Peekable>) -> Result true, + Some(&(Token::RightBrace, _, _)) => true, _ => false, }; @@ -1165,52 +1375,52 @@ fn parse_block<'a>(input: &mut Peekable>) -> Result { + Some(&(Token::RightBrace, _, _)) => { input.next(); Ok(Stmt::Block(stmts)) } - _ => Err(ParseError::MissingRCurly), + Some(&(_, line, pos)) => Err(ParseError(PERR::MissingRightBrace, line, pos)), + None => Err(ParseError(PERR::MissingRightBrace, 0, 0)), } } fn parse_expr_stmt<'a>(input: &mut Peekable>) -> Result { - let expr = parse_expr(input)?; - Ok(Stmt::Expr(Box::new(expr))) + Ok(Stmt::Expr(Box::new(parse_expr(input)?))) } fn parse_stmt<'a>(input: &mut Peekable>) -> Result { match input.peek() { - Some(&Token::If) => parse_if(input), - Some(&Token::While) => parse_while(input), - Some(&Token::Loop) => parse_loop(input), - Some(&Token::For) => parse_for(input), - Some(&Token::Break) => { + Some(&(Token::If, _, _)) => parse_if(input), + Some(&(Token::While, _, _)) => parse_while(input), + Some(&(Token::Loop, _, _)) => parse_loop(input), + Some(&(Token::For, _, _)) => parse_for(input), + Some(&(Token::Break, _, _)) => { input.next(); Ok(Stmt::Break) } - Some(&Token::Return) => { + Some(&(Token::Return, _, _)) => { input.next(); match input.peek() { - Some(&Token::Semicolon) => Ok(Stmt::Return), + Some(&(Token::SemiColon, _, _)) => Ok(Stmt::Return), _ => { let ret = parse_expr(input)?; Ok(Stmt::ReturnWithVal(Box::new(ret))) } } } - Some(&Token::LCurly) => parse_block(input), - Some(&Token::Var) => parse_var(input), + Some(&(Token::LeftBrace, _, _)) => parse_block(input), + Some(&(Token::Let, _, _)) => parse_var(input), _ => parse_expr_stmt(input), } } @@ -1219,21 +1429,23 @@ fn parse_fn<'a>(input: &mut Peekable>) -> Result s.clone(), - _ => return Err(ParseError::FnMissingName), + Some((Token::Identifier(ref s), _, _)) => s.clone(), + Some((token, line, pos)) => return Err(ParseError(PERR::FnMissingName(token), line, pos)), + None => return Err(ParseError(PERR::FnMissingName(Token::None), 0, 0)), }; match input.peek() { - Some(&Token::LParen) => { + Some(&(Token::LeftParen, _, _)) => { input.next(); } - _ => return Err(ParseError::FnMissingParams), + Some(&(_, line, pos)) => return Err(ParseError(PERR::FnMissingParams, line, pos)), + None => return Err(ParseError(PERR::FnMissingParams, 0, 0)), } let mut params = Vec::new(); let skip_params = match input.peek() { - Some(&Token::RParen) => { + Some(&(Token::RightParen, _, _)) => { input.next(); true } @@ -1243,12 +1455,13 @@ fn parse_fn<'a>(input: &mut Peekable>) -> Result break, - Some(Token::Comma) => (), - Some(Token::Identifier(ref s)) => { + Some((Token::RightParen, _, _)) => break, + Some((Token::Comma, _, _)) => (), + Some((Token::Identifier(ref s), _, _)) => { params.push(s.clone()); } - _ => return Err(ParseError::MalformedCallExpr), + Some((_, line, pos)) => return Err(ParseError(PERR::MalformedCallExpr, line, pos)), + None => return Err(ParseError(PERR::MalformedCallExpr, 0, 0)), } } } @@ -1262,28 +1475,24 @@ fn parse_fn<'a>(input: &mut Peekable>) -> Result( - input: &mut Peekable>, -) -> Result<(Vec, Vec), ParseError> { +fn parse_top_level<'a>(input: &mut Peekable>) -> Result { let mut stmts = Vec::new(); let mut fndefs = Vec::new(); while let Some(_) = input.peek() { match input.peek() { - Some(&Token::Fn) => fndefs.push(parse_fn(input)?), + Some(&(Token::Fn, _, _)) => fndefs.push(parse_fn(input)?), _ => stmts.push(parse_stmt(input)?), } - if let Some(&Token::Semicolon) = input.peek() { + if let Some(&(Token::SemiColon, _, _)) = input.peek() { input.next(); } } - Ok((stmts, fndefs)) + Ok(AST(stmts, fndefs)) } -pub fn parse<'a>( - input: &mut Peekable>, -) -> Result<(Vec, Vec), ParseError> { +pub fn parse<'a>(input: &mut Peekable>) -> Result { parse_top_level(input) } diff --git a/tests/decrement.rs b/tests/decrement.rs index 2cd8b10c..ab1dc6c8 100644 --- a/tests/decrement.rs +++ b/tests/decrement.rs @@ -10,7 +10,7 @@ fn test_decrement() { assert_eq!( engine.eval::("let s = \"test\"; s -= \"ing\"; s"), Err(EvalAltResult::ErrorFunctionNotFound( - "- (alloc::string::String,alloc::string::String)".to_string() + "- (alloc::string::String, alloc::string::String)".to_string() )) ); } diff --git a/tests/mismatched_op.rs b/tests/mismatched_op.rs index d0ea1826..944051d7 100644 --- a/tests/mismatched_op.rs +++ b/tests/mismatched_op.rs @@ -32,7 +32,7 @@ fn test_mismatched_op_custom_type() { assert_eq!( engine.eval::("60 + new_ts()"), Err(EvalAltResult::ErrorFunctionNotFound( - "+ (i64,mismatched_op::test_mismatched_op_custom_type::TestStruct)".into() + "+ (i64, mismatched_op::test_mismatched_op_custom_type::TestStruct)".into() )) ); }