diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs index 32515787..17356488 100644 --- a/src/tokenizer/mod.rs +++ b/src/tokenizer/mod.rs @@ -418,7 +418,7 @@ impl<'a> Tokenizer<'a> { '[' => self.gen_punct(Punct::OpenBracket), ']' => self.gen_punct(Punct::CloseBracket), ':' => self.gen_punct(Punct::Colon), - '?' => self.gen_punct(Punct::QuestionMark), + '?' => self.question_mark(), '#' => self.hash(), '~' => self.gen_punct(Punct::Tilde), '{' => self.open_curly(OpenCurlyKind::Block, Punct::OpenBrace), @@ -583,6 +583,10 @@ impl<'a> Tokenizer<'a> { trace!("ampersand ({}, {})", self.current_start, self.stream.idx); if self.look_ahead_byte_matches('&') { self.stream.skip_bytes(1); + if self.look_ahead_byte_matches('=') { + self.stream.skip_bytes(1); + return self.gen_punct(Punct::DoubleAmpersandEqual); + } self.gen_punct(Punct::DoubleAmpersand) } else if self.look_ahead_byte_matches('=') { self.stream.skip_bytes(1); @@ -597,6 +601,10 @@ impl<'a> Tokenizer<'a> { trace!("pipe ({}, {})", self.current_start, self.stream.idx); if self.look_ahead_byte_matches('|') { self.stream.skip_bytes(1); + if self.look_ahead_byte_matches('=') { + self.stream.skip_bytes(1); + return self.gen_punct(Punct::DoublePipeEqual); + } self.gen_punct(Punct::DoublePipe) } else if self.look_ahead_byte_matches('=') { self.stream.skip_bytes(1); @@ -680,6 +688,7 @@ impl<'a> Tokenizer<'a> { self.gen_punct(Punct::Caret) } } + /// a `#` could also be the start of a hash bang comment `#!` #[inline] fn hash(&mut self) -> Res { @@ -696,6 +705,29 @@ impl<'a> Tokenizer<'a> { self.gen_punct(Punct::Hash) } } + /// a `?` could also be the start of a ??. ??=, ?. operator + #[inline] + fn question_mark(&mut self) -> Res { + trace!( + "question_mark ({}, {})", + self.current_start, + self.stream.idx + ); + // hashbang comment can only appear at the start + if self.look_ahead_byte_matches('?') { + self.stream.skip_bytes(1); + if self.look_ahead_byte_matches('=') { + self.stream.skip_bytes(1); + return self.gen_punct(Punct::DoubleQuestionMarkEqual); + } + self.gen_punct(Punct::DoubleQuestionMark) + } else if self.look_ahead_byte_matches('.') { + self.stream.skip_bytes(1); + self.gen_punct(Punct::QuestionMarkDot) + } else { + self.gen_punct(Punct::QuestionMark) + } + } /// parse a number, this can include decimal or float literals /// like `0.01e1` or `10` as well as binary, octal or hex /// literals like `0b1`, `0o7`, or `0xf` and BigInt literals @@ -1405,13 +1437,13 @@ mod test { "-", "/", "*", "%", "&", "|", "^", ">>>=", //3 char "...", "===", "!==", ">>>", "<<=", ">>=", "**=", //2 char "&&", "||", "==", "!=", "+=", "-=", "*=", "/=", "++", "--", "<<", ">>", "&=", "|=", - "^=", "%=", "<=", ">=", "=>", "**", "@", + "^=", "%=", "<=", ">=", "=>", "**", "@", "??", "??=", "?.", "&&=", "||=", ]; for p in PUNCTS { let mut t = Tokenizer::new(p); let item = t.next(true).unwrap(); assert!(item.ty.is_punct()); - assert!(t.stream.at_end()); + assert!(t.stream.at_end(), "'{}' was not at the end", p); } } #[test] diff --git a/src/tokens/mod.rs b/src/tokens/mod.rs index 99bf919a..660ee45a 100644 --- a/src/tokens/mod.rs +++ b/src/tokens/mod.rs @@ -353,6 +353,7 @@ pub enum Punct { DoubleDash, DashEqual, DoubleAmpersand, + DoubleAmpersandEqual, DoubleAsterisk, DoubleAsteriskEqual, DoubleEqual, @@ -361,7 +362,10 @@ pub enum Punct { DoubleLessThan, DoubleLessThanEqual, DoublePipe, + DoublePipeEqual, DoublePlus, + DoubleQuestionMark, + DoubleQuestionMarkEqual, Ellipsis, Equal, EqualGreaterThan, @@ -383,6 +387,7 @@ pub enum Punct { Plus, PlusEqual, QuestionMark, + QuestionMarkDot, SemiColon, Tilde, TripleEqual, @@ -453,6 +458,11 @@ impl Punct { Punct::DoubleAsterisk => "**" == s, Punct::Hash => "#" == s, Punct::AtMark => "@" == s, + Punct::DoubleAmpersandEqual => s == "&&=", + Punct::DoublePipeEqual => s == "||=", + Punct::DoubleQuestionMark => s == "??", + Punct::DoubleQuestionMarkEqual => s == "??=", + Punct::QuestionMarkDot => s == "?.", } } } @@ -514,6 +524,11 @@ impl ToString for Punct { Punct::DoubleAsterisk => "**", Punct::Hash => "#", Punct::AtMark => "@", + Punct::DoubleAmpersandEqual => "&&=", + Punct::DoublePipeEqual => "||=", + Punct::DoubleQuestionMark => "??", + Punct::DoubleQuestionMarkEqual => "??=", + Punct::QuestionMarkDot => "?.", } .into() } diff --git a/tests/snippets/main.rs b/tests/snippets/main.rs index 08147864..591534cc 100644 --- a/tests/snippets/main.rs +++ b/tests/snippets/main.rs @@ -363,7 +363,8 @@ fn regex_pattern() { location, token: Token::RegEx(re2), .. - } = scanner.next().unwrap().unwrap() else { + } = scanner.next().unwrap().unwrap() + else { panic!("Expected regex"); }; assert_eq!(location.start.line, 1);