Add reserved symbols.

This commit is contained in:
Stephen Chung 2020-07-08 12:09:18 +08:00
parent 150f02d8b7
commit d92a514f48
3 changed files with 118 additions and 74 deletions

View File

@ -110,5 +110,5 @@ The Rhai Scripting Language
7. [Eval Statement](language/eval.md) 7. [Eval Statement](language/eval.md)
9. [Appendix](appendix/index.md) 9. [Appendix](appendix/index.md)
1. [Keywords](appendix/keywords.md) 1. [Keywords](appendix/keywords.md)
2. [Operators](appendix/operators.md) 2. [Operators and Symbols](appendix/operators.md)
3. [Literals](appendix/literals.md) 3. [Literals](appendix/literals.md)

View File

@ -1,8 +1,12 @@
Operators Operators and Symbols
========= ====================
{{#include ../links.md}} {{#include ../links.md}}
Operators
---------
| Operator | Description | Binary? | Binding direction | | Operator | Description | Binary? | Binding direction |
| :---------------: | ------------------------------ | :-----: | :---------------: | | :---------------: | ------------------------------ | :-----: | :---------------: |
| `+` | Add | Yes | Left | | `+` | Add | Yes | Left |
@ -28,3 +32,21 @@ Operators
| `!` | Boolean _Not_ | No | Left | | `!` | Boolean _Not_ | No | Left |
| `[` .. `]` | Indexing | Yes | Right | | `[` .. `]` | Indexing | Yes | Right |
| `.` | Property access, Method call | Yes | Right | | `.` | Property access, Method call | Yes | Right |
Symbols
-------
| Symbol | Description |
| ------------ | ------------------------ |
| `:` | Property value separator |
| `::` | Module path separator |
| `=>` | _Reserved_ |
| `->` | _Reserved_ |
| `<-` | _Reserved_ |
| `===` | _Reserved_ |
| `!==` | _Reserved_ |
| `:=` | _Reserved_ |
| `::<` .. `>` | _Reserved_ |
| `@` | _Reserved_ |
| `(*` .. `*)` | _Reserved_ |

View File

@ -213,6 +213,7 @@ pub enum Token {
As, As,
LexError(Box<LexError>), LexError(Box<LexError>),
Comment(String), Comment(String),
Reserved(String),
Custom(String), Custom(String),
EOF, EOF,
} }
@ -229,6 +230,7 @@ impl Token {
StringConstant(_) => "string".into(), StringConstant(_) => "string".into(),
CharConstant(c) => c.to_string().into(), CharConstant(c) => c.to_string().into(),
Identifier(s) => s.clone().into(), Identifier(s) => s.clone().into(),
Reserved(s) => s.clone().into(),
Custom(s) => s.clone().into(), Custom(s) => s.clone().into(),
LexError(err) => err.to_string().into(), LexError(err) => err.to_string().into(),
@ -339,7 +341,6 @@ impl Token {
UnaryMinus | UnaryMinus |
Multiply | Multiply |
Divide | Divide |
Colon |
Comma | Comma |
Period | Period |
Equals | Equals |
@ -750,7 +751,9 @@ fn get_next_token_inner(
} }
} }
// 0x????, 0o????, 0b???? // 0x????, 0o????, 0b????
ch @ 'x' | ch @ 'X' | ch @ 'o' | ch @ 'O' | ch @ 'b' | ch @ 'B' if c == '0' => { ch @ 'x' | ch @ 'X' | ch @ 'o' | ch @ 'O' | ch @ 'b' | ch @ 'B'
if c == '0' =>
{
result.push(next_char); result.push(next_char);
eat_next(stream, pos); eat_next(stream, pos);
@ -889,20 +892,22 @@ fn get_next_token_inner(
} }
// " - string literal // " - string literal
('"', _) => return parse_string_literal(stream, state, pos, '"') ('"', _) => {
.map_or_else( return parse_string_literal(stream, state, pos, '"').map_or_else(
|err| Some((Token::LexError(Box::new(err.0)), err.1)), |err| Some((Token::LexError(Box::new(err.0)), err.1)),
|out| Some((Token::StringConstant(out), start_pos)), |out| Some((Token::StringConstant(out), start_pos)),
), )
}
// ' - character literal // ' - character literal
('\'', '\'') => return Some(( ('\'', '\'') => {
return Some((
Token::LexError(Box::new(LERR::MalformedChar("".to_string()))), Token::LexError(Box::new(LERR::MalformedChar("".to_string()))),
start_pos, start_pos,
)), ))
('\'', _) => return Some( }
parse_string_literal(stream, state, pos, '\'') ('\'', _) => {
.map_or_else( return Some(parse_string_literal(stream, state, pos, '\'').map_or_else(
|err| (Token::LexError(Box::new(err.0)), err.1), |err| (Token::LexError(Box::new(err.0)), err.1),
|result| { |result| {
let mut chars = result.chars(); let mut chars = result.chars();
@ -914,17 +919,21 @@ fn get_next_token_inner(
start_pos, start_pos,
) )
} else { } else {
(Token::CharConstant(first.expect("should be Some")), start_pos) (
Token::CharConstant(first.expect("should be Some")),
start_pos,
)
} }
}, },
), ))
), }
// Braces // Braces
('{', _) => return Some((Token::LeftBrace, start_pos)), ('{', _) => return Some((Token::LeftBrace, start_pos)),
('}', _) => return Some((Token::RightBrace, start_pos)), ('}', _) => return Some((Token::RightBrace, start_pos)),
// Parentheses // Parentheses
('(', '*') => return Some((Token::Reserved("(*".into()), start_pos)),
('(', _) => return Some((Token::LeftParen, start_pos)), ('(', _) => return Some((Token::LeftParen, start_pos)),
(')', _) => return Some((Token::RightParen, start_pos)), (')', _) => return Some((Token::RightParen, start_pos)),
@ -953,15 +962,11 @@ fn get_next_token_inner(
eat_next(stream, pos); eat_next(stream, pos);
return Some((Token::MinusAssign, start_pos)); return Some((Token::MinusAssign, start_pos));
} }
('-', '>') => return Some(( ('-', '>') => return Some((Token::Reserved("->".into()), start_pos)),
Token::LexError(Box::new(LERR::ImproperSymbol(
"'->' is not a valid symbol. This is not C or C++!".to_string(),
))),
start_pos,
)),
('-', _) if !state.non_unary => return Some((Token::UnaryMinus, start_pos)), ('-', _) if !state.non_unary => return Some((Token::UnaryMinus, start_pos)),
('-', _) => return Some((Token::Minus, start_pos)), ('-', _) => return Some((Token::Minus, start_pos)),
('*', ')') => return Some((Token::Reserved("*)".into()), start_pos)),
('*', '=') => { ('*', '=') => {
eat_next(stream, pos); eat_next(stream, pos);
return Some((Token::MultiplyAssign, start_pos)); return Some((Token::MultiplyAssign, start_pos));
@ -1026,49 +1031,31 @@ fn get_next_token_inner(
// Warn against `===` // Warn against `===`
if stream.peek_next() == Some('=') { if stream.peek_next() == Some('=') {
return Some(( return Some((Token::Reserved("===".into()), start_pos));
Token::LexError(Box::new(LERR::ImproperSymbol(
"'===' is not a valid operator. This is not JavaScript! Should it be '=='?"
.to_string(),
))),
start_pos,
));
} }
return Some((Token::EqualsTo, start_pos)); return Some((Token::EqualsTo, start_pos));
} }
('=', '>') => return Some(( ('=', '>') => return Some((Token::Reserved("=>".into()), start_pos)),
Token::LexError(Box::new(LERR::ImproperSymbol(
"'=>' is not a valid symbol. This is not Rust! Should it be '>='?"
.to_string(),
))),
start_pos,
)),
('=', _) => return Some((Token::Equals, start_pos)), ('=', _) => return Some((Token::Equals, start_pos)),
(':', ':') => { (':', ':') => {
eat_next(stream, pos); eat_next(stream, pos);
if stream.peek_next() == Some('<') {
return Some((Token::Reserved("::<".into()), start_pos));
}
return Some((Token::DoubleColon, start_pos)); return Some((Token::DoubleColon, start_pos));
} }
(':', '=') => return Some(( (':', '=') => return Some((Token::Reserved(":=".into()), start_pos)),
Token::LexError(Box::new(LERR::ImproperSymbol(
"':=' is not a valid assignment operator. This is not Pascal! Should it be simply '='?"
.to_string(),
))),
start_pos,
)),
(':', _) => return Some((Token::Colon, start_pos)), (':', _) => return Some((Token::Colon, start_pos)),
('<', '=') => { ('<', '=') => {
eat_next(stream, pos); eat_next(stream, pos);
return Some((Token::LessThanEqualsTo, start_pos)); return Some((Token::LessThanEqualsTo, start_pos));
} }
('<', '-') => return Some(( ('<', '-') => return Some((Token::Reserved("<-".into()), start_pos)),
Token::LexError(Box::new(LERR::ImproperSymbol(
"'<-' is not a valid symbol. Should it be '<='?".to_string(),
))),
start_pos,
)),
('<', '<') => { ('<', '<') => {
eat_next(stream, pos); eat_next(stream, pos);
@ -1106,15 +1093,8 @@ fn get_next_token_inner(
('!', '=') => { ('!', '=') => {
eat_next(stream, pos); eat_next(stream, pos);
// Warn against `!==`
if stream.peek_next() == Some('=') { if stream.peek_next() == Some('=') {
return Some(( return Some((Token::Reserved("!==".into()), start_pos));
Token::LexError(Box::new(LERR::ImproperSymbol(
"'!==' is not a valid operator. This is not JavaScript! Should it be '!='?"
.to_string(),
))),
start_pos,
));
} }
return Some((Token::NotEqualsTo, start_pos)); return Some((Token::NotEqualsTo, start_pos));
@ -1159,10 +1139,17 @@ fn get_next_token_inner(
} }
('~', _) => return Some((Token::PowerOf, start_pos)), ('~', _) => return Some((Token::PowerOf, start_pos)),
('@', _) => return Some((Token::Reserved("@".into()), start_pos)),
('\0', _) => unreachable!(), ('\0', _) => unreachable!(),
(ch, _) if ch.is_whitespace() => (), (ch, _) if ch.is_whitespace() => (),
(ch, _) => return Some((Token::LexError(Box::new(LERR::UnexpectedInput(ch.to_string()))), start_pos)), (ch, _) => {
return Some((
Token::LexError(Box::new(LERR::UnexpectedInput(ch.to_string()))),
start_pos,
))
}
} }
} }
@ -1237,6 +1224,41 @@ impl<'a> Iterator for TokenIterator<'a, '_> {
self.engine.custom_keywords.as_ref(), self.engine.custom_keywords.as_ref(),
) { ) {
(None, _, _) => None, (None, _, _) => None,
(Some((Token::Reserved(s), pos)), None, None) => return Some((match s.as_str() {
"===" => Token::LexError(Box::new(LERR::ImproperSymbol(
"'===' is not a valid operator. This is not JavaScript! Should it be '=='?"
.to_string(),
))),
"!==" => Token::LexError(Box::new(LERR::ImproperSymbol(
"'!==' is not a valid operator. This is not JavaScript! Should it be '!='?"
.to_string(),
))),
"->" => Token::LexError(Box::new(LERR::ImproperSymbol(
"'->' is not a valid symbol. This is not C or C++!".to_string(),
))),
"<-" => Token::LexError(Box::new(LERR::ImproperSymbol(
"'<-' is not a valid symbol. This is not Go! Should it be '<='?".to_string(),
))),
"=>" => Token::LexError(Box::new(LERR::ImproperSymbol(
"'=>' is not a valid symbol. This is not Rust! Should it be '>='?"
.to_string(),
))),
":=" => Token::LexError(Box::new(LERR::ImproperSymbol(
"':=' is not a valid assignment operator. This is not Go! Should it be simply '='?"
.to_string(),
))),
"::<" => Token::LexError(Box::new(LERR::ImproperSymbol(
"'::<>' is not a valid symbol. This is not Rust! Should it be '::'?"
.to_string(),
))),
"(*" | "*)" => Token::LexError(Box::new(LERR::ImproperSymbol(
"'(* .. *)' is not a valid comment style. This is not Pascal! Should it be '/* .. */'?"
.to_string(),
))),
token => Token::LexError(Box::new(LERR::ImproperSymbol(
format!("'{}' is not a valid symbol.", token)
))),
}, pos)),
(r @ Some(_), None, None) => r, (r @ Some(_), None, None) => r,
(Some((token, pos)), Some(disabled), _) (Some((token, pos)), Some(disabled), _)
if token.is_operator() && disabled.contains(token.syntax().as_ref()) => if token.is_operator() && disabled.contains(token.syntax().as_ref()) =>