add 2020 operators

src/tokenizer/mod.rs

@@ -418,7 +418,7 @@ impl<'a> Tokenizer<'a> {
             '[' => self.gen_punct(Punct::OpenBracket),
             ']' => self.gen_punct(Punct::CloseBracket),
             ':' => self.gen_punct(Punct::Colon),
-            '?' => self.gen_punct(Punct::QuestionMark),
+            '?' => self.question_mark(),
             '#' => self.hash(),
             '~' => self.gen_punct(Punct::Tilde),
             '{' => self.open_curly(OpenCurlyKind::Block, Punct::OpenBrace),
@@ -583,6 +583,10 @@ impl<'a> Tokenizer<'a> {
         trace!("ampersand ({}, {})", self.current_start, self.stream.idx);
         if self.look_ahead_byte_matches('&') {
             self.stream.skip_bytes(1);
+            if self.look_ahead_byte_matches('=') {
+                self.stream.skip_bytes(1);
+                return self.gen_punct(Punct::DoubleAmpersandEqual);
+            }
             self.gen_punct(Punct::DoubleAmpersand)
         } else if self.look_ahead_byte_matches('=') {
             self.stream.skip_bytes(1);
@@ -597,6 +601,10 @@ impl<'a> Tokenizer<'a> {
         trace!("pipe ({}, {})", self.current_start, self.stream.idx);
         if self.look_ahead_byte_matches('|') {
             self.stream.skip_bytes(1);
+            if self.look_ahead_byte_matches('=') {
+                self.stream.skip_bytes(1);
+                return self.gen_punct(Punct::DoublePipeEqual);
+            }
             self.gen_punct(Punct::DoublePipe)
         } else if self.look_ahead_byte_matches('=') {
             self.stream.skip_bytes(1);
@@ -680,6 +688,7 @@ impl<'a> Tokenizer<'a> {
             self.gen_punct(Punct::Caret)
         }
     }
+
     /// a `#` could also be the start of a hash bang comment `#!`
     #[inline]
     fn hash(&mut self) -> Res<RawItem> {
@@ -696,6 +705,25 @@ impl<'a> Tokenizer<'a> {
             self.gen_punct(Punct::Hash)
         }
     }
+    /// a `?` could also be the start of a ??. ??=, ?. operator
+    #[inline]
+    fn question_mark(&mut self) -> Res<RawItem> {
+        trace!("question_mark ({}, {})", self.current_start, self.stream.idx);
+        // hashbang comment can only appear at the start
+        if self.look_ahead_byte_matches('?') {
+            self.stream.skip_bytes(1);
+            if self.look_ahead_byte_matches('=') {
+                self.stream.skip_bytes(1);
+                return self.gen_punct(Punct::DoubleQuestionMarkEqual);
+            }
+            self.gen_punct(Punct::DoubleQuestionMark)
+        } else if self.look_ahead_byte_matches('.') {
+            self.stream.skip_bytes(1);
+            self.gen_punct(Punct::QuestionMarkDot)
+        } else {
+            self.gen_punct(Punct::QuestionMark)
+        }
+    }
     /// parse a number, this can include decimal or float literals
     /// like `0.01e1` or `10` as well as binary, octal or hex
     /// literals like `0b1`, `0o7`, or `0xf` and BigInt literals
@@ -1405,13 +1433,13 @@ mod test {
             "-", "/", "*", "%", "&", "|", "^", ">>>=", //3 char
             "...", "===", "!==", ">>>", "<<=", ">>=", "**=", //2 char
             "&&", "||", "==", "!=", "+=", "-=", "*=", "/=", "++", "--", "<<", ">>", "&=", "|=",
-            "^=", "%=", "<=", ">=", "=>", "**", "@",
+            "^=", "%=", "<=", ">=", "=>", "**", "@", "??", "??=", "?.", "&&=", "||="
         ];
         for p in PUNCTS {
             let mut t = Tokenizer::new(p);
             let item = t.next(true).unwrap();
             assert!(item.ty.is_punct());
-            assert!(t.stream.at_end());
+            assert!(t.stream.at_end(), "'{}' was not at the end", p);
         }
     }
     #[test]

src/tokens/mod.rs

@@ -353,6 +353,7 @@ pub enum Punct {
     DoubleDash,
     DashEqual,
     DoubleAmpersand,
+    DoubleAmpersandEqual,
     DoubleAsterisk,
     DoubleAsteriskEqual,
     DoubleEqual,
@@ -361,7 +362,10 @@ pub enum Punct {
     DoubleLessThan,
     DoubleLessThanEqual,
     DoublePipe,
+    DoublePipeEqual,
     DoublePlus,
+    DoubleQuestionMark,
+    DoubleQuestionMarkEqual,
     Ellipsis,
     Equal,
     EqualGreaterThan,
@@ -383,6 +387,7 @@ pub enum Punct {
     Plus,
     PlusEqual,
     QuestionMark,
+    QuestionMarkDot,
     SemiColon,
     Tilde,
     TripleEqual,
@@ -453,6 +458,11 @@ impl Punct {
             Punct::DoubleAsterisk => "**" == s,
             Punct::Hash => "#" == s,
             Punct::AtMark => "@" == s,
+            Punct::DoubleAmpersandEqual => s == "&&=",
+            Punct::DoublePipeEqual => s == "||=",
+            Punct::DoubleQuestionMark => s == "??",
+            Punct::DoubleQuestionMarkEqual => s == "??=",
+            Punct::QuestionMarkDot => s == "?.",
         }
     }
 }
@@ -514,6 +524,11 @@ impl ToString for Punct {
             Punct::DoubleAsterisk => "**",
             Punct::Hash => "#",
             Punct::AtMark => "@",
+            Punct::DoubleAmpersandEqual => "&&=",
+            Punct::DoublePipeEqual => "||=",
+            Punct::DoubleQuestionMark => "??",
+            Punct::DoubleQuestionMarkEqual => "??=",
+            Punct::QuestionMarkDot => "?.",
         }
         .into()
     }