#[derive(Clone)]
pub struct Parser<'a> {
pub sess: &'a ParseSess,
+ /// The current non-normalized token.
+ pub token: Token,
/// The current normalized token.
/// "Normalized" means that some interpolated tokens
/// (`$i: ident` and `$l: lifetime` meta-variables) are replaced
/// with non-interpolated identifier and lifetime tokens they refer to.
- /// Use span from this token if you need an isolated span.
- pub token: Token,
- /// The current non-normalized token if it's different from `token`.
- /// Preferable use is through the `unnormalized_token()` getter.
- /// Use span from this token if you need to concatenate it with some neighbouring spans.
- unnormalized_token: Option<Token>,
+ /// Use this if you need to check for `token::Ident` or `token::Lifetime` specifically,
+ /// this also includes edition checks for edition-specific keyword identifiers.
+ pub normalized_token: Token,
+ /// The previous non-normalized token.
+ pub prev_token: Token,
/// The previous normalized token.
- /// Use span from this token if you need an isolated span.
- prev_token: Token,
- /// The previous non-normalized token if it's different from `prev_token`.
- /// Preferable use is through the `unnormalized_prev_token()` getter.
- /// Use span from this token if you need to concatenate it with some neighbouring spans.
- unnormalized_prev_token: Option<Token>,
- /// Equivalent to `unnormalized_prev_token().span`.
- /// FIXME: Remove in favor of `(unnormalized_)prev_token().span`.
+ /// Use this if you need to check for `token::Ident` or `token::Lifetime` specifically,
+ /// this also includes edition checks for edition-specific keyword identifiers.
+ pub normalized_prev_token: Token,
+ /// FIXME: Remove in favor of the equivalent `prev_token.span`.
pub prev_span: Span,
restrictions: Restrictions,
/// Used to determine the path to externally loaded source files.
]
.iter()
.cloned()
- .collect::<TokenStream>()
- .into(),
+ .collect::<TokenStream>(),
);
self.stack.push(mem::replace(
let mut parser = Parser {
sess,
token: Token::dummy(),
- unnormalized_token: None,
+ normalized_token: Token::dummy(),
prev_token: Token::dummy(),
- unnormalized_prev_token: None,
+ normalized_prev_token: Token::dummy(),
prev_span: DUMMY_SP,
restrictions: Restrictions::empty(),
recurse_into_file_modules,
root_module_name: None,
expected_tokens: Vec::new(),
token_cursor: TokenCursor {
- frame: TokenCursorFrame::new(DelimSpan::dummy(), token::NoDelim, &tokens.into()),
+ frame: TokenCursorFrame::new(DelimSpan::dummy(), token::NoDelim, &tokens),
stack: Vec::new(),
},
desugar_doc_comments,
subparser_name,
};
- parser.token = parser.next_tok();
+ // Make parser point to the first token.
+ parser.bump();
if let Some(directory) = directory {
parser.directory = directory;
}
}
- parser.process_potential_macro_variable();
parser
}
- fn unnormalized_token(&self) -> &Token {
- self.unnormalized_token.as_ref().unwrap_or(&self.token)
- }
-
- fn unnormalized_prev_token(&self) -> &Token {
- self.unnormalized_prev_token.as_ref().unwrap_or(&self.prev_token)
- }
-
- fn next_tok(&mut self) -> Token {
+ fn next_tok(&mut self, fallback_span: Span) -> Token {
let mut next = if self.desugar_doc_comments {
self.token_cursor.next_desugared()
} else {
};
if next.span.is_dummy() {
// Tweak the location for better diagnostics, but keep syntactic context intact.
- next.span = self.unnormalized_token().span.with_ctxt(next.span.ctxt());
+ next.span = fallback_span.with_ctxt(next.span.ctxt());
}
next
}
}
fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, ast::Ident> {
- match self.token.kind {
+ match self.normalized_token.kind {
token::Ident(name, _) => {
if self.token.is_reserved_ident() {
let mut err = self.expected_ident_found();
return Err(err);
}
}
- let span = self.token.span;
self.bump();
- Ok(Ident::new(name, span))
+ Ok(Ident::new(name, self.normalized_prev_token.span))
}
_ => Err(match self.prev_token.kind {
TokenKind::DocComment(..) => {
)
}
- /// Expects and consumes a `+`. if `+=` is seen, replaces it with a `=`
- /// and continues. If a `+` is not seen, returns `false`.
- ///
- /// This is used when token-splitting `+=` into `+`.
- /// See issue #47856 for an example of when this may occur.
- fn eat_plus(&mut self) -> bool {
- self.expected_tokens.push(TokenType::Token(token::BinOp(token::Plus)));
- match self.token.kind {
- token::BinOp(token::Plus) => {
- self.bump();
+ /// Eats the expected token if it's present possibly breaking
+ /// compound tokens like multi-character operators in process.
+ /// Returns `true` if the token was eaten.
+ fn break_and_eat(&mut self, expected: TokenKind) -> bool {
+ if self.token.kind == expected {
+ self.bump();
+ return true;
+ }
+ match self.token.kind.break_two_token_op() {
+ Some((first, second)) if first == expected => {
+ let first_span = self.sess.source_map().start_point(self.token.span);
+ let second_span = self.token.span.with_lo(first_span.hi());
+ self.set_token(Token::new(first, first_span));
+ self.bump_with(Token::new(second, second_span));
true
}
- token::BinOpEq(token::Plus) => {
- let start_point = self.sess.source_map().start_point(self.token.span);
- self.bump_with(token::Eq, self.token.span.with_lo(start_point.hi()));
- true
+ _ => {
+ self.expected_tokens.push(TokenType::Token(expected));
+ false
}
- _ => false,
}
}
- /// Expects and consumes an `&`. If `&&` is seen, replaces it with a single
- /// `&` and continues. If an `&` is not seen, signals an error.
+ /// Eats `+` possibly breaking tokens like `+=` in process.
+ fn eat_plus(&mut self) -> bool {
+ self.break_and_eat(token::BinOp(token::Plus))
+ }
+
+ /// Eats `&` possibly breaking tokens like `&&` in process.
+ /// Signals an error if `&` is not eaten.
fn expect_and(&mut self) -> PResult<'a, ()> {
- self.expected_tokens.push(TokenType::Token(token::BinOp(token::And)));
- match self.token.kind {
- token::BinOp(token::And) => {
- self.bump();
- Ok(())
- }
- token::AndAnd => {
- let start_point = self.sess.source_map().start_point(self.token.span);
- Ok(self
- .bump_with(token::BinOp(token::And), self.token.span.with_lo(start_point.hi())))
- }
- _ => self.unexpected(),
- }
+ if self.break_and_eat(token::BinOp(token::And)) { Ok(()) } else { self.unexpected() }
}
- /// Expects and consumes an `|`. If `||` is seen, replaces it with a single
- /// `|` and continues. If an `|` is not seen, signals an error.
+ /// Eats `|` possibly breaking tokens like `||` in process.
+ /// Signals an error if `|` was not eaten.
fn expect_or(&mut self) -> PResult<'a, ()> {
- self.expected_tokens.push(TokenType::Token(token::BinOp(token::Or)));
- match self.token.kind {
- token::BinOp(token::Or) => {
- self.bump();
- Ok(())
- }
- token::OrOr => {
- let start_point = self.sess.source_map().start_point(self.token.span);
- Ok(self
- .bump_with(token::BinOp(token::Or), self.token.span.with_lo(start_point.hi())))
- }
- _ => self.unexpected(),
- }
+ if self.break_and_eat(token::BinOp(token::Or)) { Ok(()) } else { self.unexpected() }
}
- /// Attempts to consume a `<`. If `<<` is seen, replaces it with a single
- /// `<` and continue. If `<-` is seen, replaces it with a single `<`
- /// and continue. If a `<` is not seen, returns false.
- ///
- /// This is meant to be used when parsing generics on a path to get the
- /// starting token.
+ /// Eats `<` possibly breaking tokens like `<<` in process.
fn eat_lt(&mut self) -> bool {
- self.expected_tokens.push(TokenType::Token(token::Lt));
- let ate = match self.token.kind {
- token::Lt => {
- self.bump();
- true
- }
- token::BinOp(token::Shl) => {
- let start_point = self.sess.source_map().start_point(self.token.span);
- self.bump_with(token::Lt, self.token.span.with_lo(start_point.hi()));
- true
- }
- token::LArrow => {
- let start_point = self.sess.source_map().start_point(self.token.span);
- self.bump_with(
- token::BinOp(token::Minus),
- self.token.span.with_lo(start_point.hi()),
- );
- true
- }
- _ => false,
- };
-
+ let ate = self.break_and_eat(token::Lt);
if ate {
// See doc comment for `unmatched_angle_bracket_count`.
self.unmatched_angle_bracket_count += 1;
self.max_angle_bracket_count += 1;
debug!("eat_lt: (increment) count={:?}", self.unmatched_angle_bracket_count);
}
-
ate
}
+ /// Eats `<` possibly breaking tokens like `<<` in process.
+ /// Signals an error if `<` was not eaten.
fn expect_lt(&mut self) -> PResult<'a, ()> {
- if !self.eat_lt() { self.unexpected() } else { Ok(()) }
+ if self.eat_lt() { Ok(()) } else { self.unexpected() }
}
- /// Expects and consumes a single `>` token. if a `>>` is seen, replaces it
- /// with a single `>` and continues. If a `>` is not seen, signals an error.
+ /// Eats `>` possibly breaking tokens like `>>` in process.
+ /// Signals an error if `>` was not eaten.
fn expect_gt(&mut self) -> PResult<'a, ()> {
- self.expected_tokens.push(TokenType::Token(token::Gt));
- let ate = match self.token.kind {
- token::Gt => {
- self.bump();
- Some(())
- }
- token::BinOp(token::Shr) => {
- let start_point = self.sess.source_map().start_point(self.token.span);
- Some(self.bump_with(token::Gt, self.token.span.with_lo(start_point.hi())))
- }
- token::BinOpEq(token::Shr) => {
- let start_point = self.sess.source_map().start_point(self.token.span);
- Some(self.bump_with(token::Ge, self.token.span.with_lo(start_point.hi())))
- }
- token::Ge => {
- let start_point = self.sess.source_map().start_point(self.token.span);
- Some(self.bump_with(token::Eq, self.token.span.with_lo(start_point.hi())))
- }
- _ => None,
- };
-
- match ate {
- Some(_) => {
- // See doc comment for `unmatched_angle_bracket_count`.
- if self.unmatched_angle_bracket_count > 0 {
- self.unmatched_angle_bracket_count -= 1;
- debug!("expect_gt: (decrement) count={:?}", self.unmatched_angle_bracket_count);
- }
-
- Ok(())
+ if self.break_and_eat(token::Gt) {
+ // See doc comment for `unmatched_angle_bracket_count`.
+ if self.unmatched_angle_bracket_count > 0 {
+ self.unmatched_angle_bracket_count -= 1;
+ debug!("expect_gt: (decrement) count={:?}", self.unmatched_angle_bracket_count);
}
- None => self.unexpected(),
+ Ok(())
+ } else {
+ self.unexpected()
}
}
self.parse_delim_comma_seq(token::Paren, f)
}
- /// Advance the parser by one token.
- pub fn bump(&mut self) {
+ // Interpolated identifier (`$i: ident`) and lifetime (`$l: lifetime`)
+ // tokens are replaced with usual identifier and lifetime tokens,
+ // so the former are never encountered during normal parsing.
+ crate fn set_token(&mut self, token: Token) {
+ self.token = token;
+ self.normalized_token = match &self.token.kind {
+ token::Interpolated(nt) => match **nt {
+ token::NtIdent(ident, is_raw) => {
+ Token::new(token::Ident(ident.name, is_raw), ident.span)
+ }
+ token::NtLifetime(ident) => Token::new(token::Lifetime(ident.name), ident.span),
+ _ => self.token.clone(),
+ },
+ _ => self.token.clone(),
+ }
+ }
+
+ /// Advance the parser by one token using provided token as the next one.
+ fn bump_with(&mut self, next_token: Token) {
+ // Bumping after EOF is a bad sign, usually an infinite loop.
if self.prev_token.kind == TokenKind::Eof {
- // Bumping after EOF is a bad sign, usually an infinite loop.
let msg = "attempted to bump the parser past EOF (may be stuck in a loop)";
self.span_bug(self.token.span, msg);
}
// Update the current and previous tokens.
- let next_token = self.next_tok();
- self.prev_token = mem::replace(&mut self.token, next_token);
- self.unnormalized_prev_token = self.unnormalized_token.take();
+ self.prev_token = self.token.take();
+ self.normalized_prev_token = self.normalized_token.take();
+ self.set_token(next_token);
// Update fields derived from the previous token.
- self.prev_span = self.unnormalized_prev_token().span;
+ self.prev_span = self.prev_token.span;
+ // Diagnostics.
self.expected_tokens.clear();
- // Check after each token.
- self.process_potential_macro_variable();
}
- /// Advances the parser using provided token as a next one. Use this when
- /// consuming a part of a token. For example a single `<` from `<<`.
- /// FIXME: this function sets the previous token data to some semi-nonsensical values
- /// which kind of work because they are currently used in very limited ways in practice.
- /// Correct token kinds and spans need to be calculated instead.
- fn bump_with(&mut self, next: TokenKind, span: Span) {
- // Update the current and previous tokens.
- let next_token = Token::new(next, span);
- self.prev_token = mem::replace(&mut self.token, next_token);
- self.unnormalized_prev_token = self.unnormalized_token.take();
-
- // Update fields derived from the previous token.
- self.prev_span = self.unnormalized_prev_token().span.with_hi(span.lo());
-
- self.expected_tokens.clear();
+ /// Advance the parser by one token.
+ pub fn bump(&mut self) {
+ let next_token = self.next_tok(self.token.span);
+ self.bump_with(next_token);
}
/// Look-ahead `dist` tokens of `self.token` and get access to that token there.
/// Parses asyncness: `async` or nothing.
fn parse_asyncness(&mut self) -> Async {
if self.eat_keyword(kw::Async) {
- let span = self.prev_span;
+ let span = self.normalized_prev_token.span;
Async::Yes { span, closure_id: DUMMY_NODE_ID, return_impl_trait_id: DUMMY_NODE_ID }
} else {
Async::No
}
}
- pub fn process_potential_macro_variable(&mut self) {
- let normalized_token = match self.token.kind {
- token::Dollar
- if self.token.span.from_expansion() && self.look_ahead(1, |t| t.is_ident()) =>
- {
- self.bump();
- let name = match self.token.kind {
- token::Ident(name, _) => name,
- _ => unreachable!(),
- };
- let span = self.prev_span.to(self.token.span);
- self.struct_span_err(span, &format!("unknown macro variable `{}`", name))
- .span_label(span, "unknown macro variable")
- .emit();
- self.bump();
- return;
- }
- token::Interpolated(ref nt) => {
- // Interpolated identifier and lifetime tokens are replaced with usual identifier
- // and lifetime tokens, so the former are never encountered during normal parsing.
- match **nt {
- token::NtIdent(ident, is_raw) => {
- Token::new(token::Ident(ident.name, is_raw), ident.span)
- }
- token::NtLifetime(ident) => Token::new(token::Lifetime(ident.name), ident.span),
- _ => return,
- }
- }
- _ => return,
- };
- self.unnormalized_token = Some(mem::replace(&mut self.token, normalized_token));
- }
-
/// Parses a single token tree from the input.
pub fn parse_token_tree(&mut self) -> TokenTree {
match self.token.kind {
&mut self.token_cursor.frame,
self.token_cursor.stack.pop().unwrap(),
);
- self.token.span = frame.span.entire();
+ self.set_token(Token::new(TokenKind::CloseDelim(frame.delim), frame.span.close));
self.bump();
- TokenTree::Delimited(frame.span, frame.delim, frame.tree_cursor.stream.into())
+ TokenTree::Delimited(frame.span, frame.delim, frame.tree_cursor.stream)
}
token::CloseDelim(_) | token::Eof => unreachable!(),
_ => {
- let token = self.token.clone();
self.bump();
- TokenTree::Token(token)
+ TokenTree::Token(self.prev_token.clone())
}
}
}
projects / rust.git / blobdiff