parser tests work

[rust.git] / crates / parser / src / tokens.rs

diff --git a/crates/parser/src/tokens.rs b/crates/parser/src/tokens.rs
index 495d9713ea9d6e3a08b7086ef0ffa001bb3654f9..4f10956070f7b3a764d0536afd34679a2e471a60 100644 (file)
--- a/crates/parser/src/tokens.rs
+++ b/crates/parser/src/tokens.rs
@@ -1,8 +1,19 @@
-use crate::{SyntaxKind, Token};
+use crate::SyntaxKind;
 
 #[allow(non_camel_case_types)]
 type bits = u64;
 
+/// `Token` abstracts the cursor of `TokenSource` operates on.
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+pub(crate) struct Token {
+    /// What is the current token?
+    pub(crate) kind: SyntaxKind,
+
+    /// Is the current token joined to the next one (`> >` vs `>>`).
+    pub(crate) is_jointed_to_next: bool,
+    pub(crate) contextual_kw: SyntaxKind,
+}
+
 /// Main input to the parser.
 ///
 /// A sequence of tokens represented internally as a struct of arrays.
@@ -49,13 +60,14 @@ pub fn len(&self) -> usize {
         self.kind.len()
     }
     pub(crate) fn get(&self, idx: usize) -> Token {
-        if idx > self.len() {
-            return self.eof();
+        if idx < self.len() {
+            let kind = self.kind[idx];
+            let is_jointed_to_next = self.get_joint(idx);
+            let contextual_kw = self.contextual_kw[idx];
+            Token { kind, is_jointed_to_next, contextual_kw }
+        } else {
+            self.eof()
         }
-        let kind = self.kind[idx];
-        let is_jointed_to_next = self.get_joint(idx);
-        let contextual_kw = self.contextual_kw[idx];
-        Token { kind, is_jointed_to_next, contextual_kw }
     }
 
     #[cold]