Declare other tokens used later in the reference grammar

There were some tokens used in the grammar but not declared. Antlr
doesn't really seem to care and happily uses them, but they appear in
RustLexer.tokens in a potentially-unexpected order.

diff --git a/src/grammar/RustLexer.g4 b/src/grammar/RustLexer.g4
index 5dcf0f6935f221b5293e53b751f7aba87f4afcc5..f062d33f25e25514423e15c72e7f68148f59adf4 100644 (file)
--- a/src/grammar/RustLexer.g4
+++ b/src/grammar/RustLexer.g4
@@ -12,10 +12,10 @@ tokens {
     MINUS, STAR, SLASH, PERCENT, CARET, AND, OR, SHL, SHR, BINOP,
     BINOPEQ, AT, DOT, DOTDOT, DOTDOTDOT, COMMA, SEMI, COLON,
     MOD_SEP, RARROW, FAT_ARROW, LPAREN, RPAREN, LBRACKET, RBRACKET,
-    LBRACE, RBRACE, POUND, DOLLAR, UNDERSCORE, LIT_CHAR,
+    LBRACE, RBRACE, POUND, DOLLAR, UNDERSCORE, LIT_CHAR, LIT_BYTE,
     LIT_INTEGER, LIT_FLOAT, LIT_STR, LIT_STR_RAW, LIT_BINARY,
-    LIT_BINARY_RAW, IDENT, LIFETIME, WHITESPACE, DOC_COMMENT,
-    COMMENT, SHEBANG
+    LIT_BINARY_RAW, QUESTION, IDENT, LIFETIME, WHITESPACE, DOC_COMMENT,
+    COMMENT, SHEBANG, UTF8_BOM
 }
 
 import xidstart , xidcontinue;