use crate::ast::NodeId;
-use crate::early_buffered_lints::BufferedEarlyLintId;
use crate::ext::tt::macro_parser;
use crate::feature_gate::Features;
use crate::parse::token::{self, Token, TokenKind};
}
impl Delimited {
- /// Returns the opening delimiter (possibly `NoDelim`).
- pub fn open_token(&self) -> TokenKind {
- token::OpenDelim(self.delim)
- }
-
- /// Returns the closing delimiter (possibly `NoDelim`).
- pub fn close_token(&self) -> TokenKind {
- token::CloseDelim(self.delim)
- }
-
/// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter.
pub fn open_tt(&self, span: Span) -> TokenTree {
let open_span = if span.is_dummy() {
} else {
span.with_lo(span.lo() + BytePos(self.delim.len() as u32))
};
- TokenTree::token(self.open_token(), open_span)
+ TokenTree::token(token::OpenDelim(self.delim), open_span)
}
/// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter.
} else {
span.with_lo(span.hi() - BytePos(self.delim.len() as u32))
};
- TokenTree::token(self.close_token(), close_span)
+ TokenTree::token(token::CloseDelim(self.delim), close_span)
}
}
/// The sequence of token trees
pub tts: Vec<TokenTree>,
/// The optional separator
- pub separator: Option<TokenKind>,
+ pub separator: Option<Token>,
/// Whether the sequence can be repeated zero (*), or one or more times (+)
pub op: KleeneOp,
/// The number of `Match`s that appear in the sequence (and subsequences)
Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => {
// Must have `(` not `{` or `[`
if delim != token::Paren {
- let tok = pprust::token_to_string(&token::OpenDelim(delim));
+ let tok = pprust::token_kind_to_string(&token::OpenDelim(delim));
let msg = format!("expected `(`, found `{}`", tok);
sess.span_diagnostic.span_err(span.entire(), &msg);
}
macro_node_id,
);
// Get the Kleene operator and optional separator
- let (separator, op) =
- parse_sep_and_kleene_op(
- trees,
- span.entire(),
- sess,
- features,
- attrs,
- edition,
- macro_node_id,
- );
+ let (separator, op) = parse_sep_and_kleene_op(trees, span.entire(), sess);
// Count the number of captured "names" (i.e., named metavars)
let name_captures = macro_parser::count_names(&sequence);
TokenTree::Sequence(
/// Takes a token and returns `Some(KleeneOp)` if the token is `+` `*` or `?`. Otherwise, return
/// `None`.
-fn kleene_op(token: &TokenKind) -> Option<KleeneOp> {
- match *token {
+fn kleene_op(token: &Token) -> Option<KleeneOp> {
+ match token.kind {
token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore),
token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore),
token::Question => Some(KleeneOp::ZeroOrOne),
/// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene
/// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an
/// error with the appropriate span is emitted to `sess` and a dummy value is returned.
-///
-/// N.B., in the 2015 edition, `*` and `+` are the only Kleene operators, and `?` is a separator.
-/// In the 2018 edition however, `?` is a Kleene operator, and not a separator.
-fn parse_sep_and_kleene_op<I>(
- input: &mut Peekable<I>,
- span: Span,
- sess: &ParseSess,
- features: &Features,
- attrs: &[ast::Attribute],
- edition: Edition,
- macro_node_id: NodeId,
-) -> (Option<TokenKind>, KleeneOp)
-where
- I: Iterator<Item = tokenstream::TokenTree>,
-{
- match edition {
- Edition::Edition2015 => parse_sep_and_kleene_op_2015(
- input,
- span,
- sess,
- features,
- attrs,
- macro_node_id,
- ),
- Edition::Edition2018 => parse_sep_and_kleene_op_2018(input, span, sess, features, attrs),
- }
-}
-
-// `?` is a separator (with a migration warning) and never a KleeneOp.
-fn parse_sep_and_kleene_op_2015<I>(
- input: &mut Peekable<I>,
+fn parse_sep_and_kleene_op(
+ input: &mut Peekable<impl Iterator<Item = tokenstream::TokenTree>>,
span: Span,
sess: &ParseSess,
- _features: &Features,
- _attrs: &[ast::Attribute],
- macro_node_id: NodeId,
-) -> (Option<TokenKind>, KleeneOp)
-where
- I: Iterator<Item = tokenstream::TokenTree>,
-{
- // We basically look at two token trees here, denoted as #1 and #2 below
- let span = match parse_kleene_op(input, span) {
- // #1 is a `+` or `*` KleeneOp
- //
- // `?` is ambiguous: it could be a separator (warning) or a Kleene::ZeroOrOne (error), so
- // we need to look ahead one more token to be sure.
- Ok(Ok((op, _))) if op != KleeneOp::ZeroOrOne => return (None, op),
-
- // #1 is `?` token, but it could be a Kleene::ZeroOrOne (error in 2015) without a separator
- // or it could be a `?` separator followed by any Kleene operator. We need to look ahead 1
- // token to find out which.
- Ok(Ok((op, op1_span))) => {
- assert_eq!(op, KleeneOp::ZeroOrOne);
-
- // Lookahead at #2. If it is a KleenOp, then #1 is a separator.
- let is_1_sep = if let Some(tokenstream::TokenTree::Token(tok2)) = input.peek() {
- kleene_op(tok2).is_some()
- } else {
- false
- };
-
- if is_1_sep {
- // #1 is a separator and #2 should be a KleepeOp.
- // (N.B. We need to advance the input iterator.)
- match parse_kleene_op(input, span) {
- // #2 is `?`, which is not allowed as a Kleene op in 2015 edition,
- // but is allowed in the 2018 edition.
- Ok(Ok((op, op2_span))) if op == KleeneOp::ZeroOrOne => {
- sess.span_diagnostic
- .struct_span_err(op2_span, "expected `*` or `+`")
- .note("`?` is not a macro repetition operator in the 2015 edition, \
- but is accepted in the 2018 edition")
- .emit();
-
- // Return a dummy
- return (None, KleeneOp::ZeroOrMore);
- }
-
- // #2 is a Kleene op, which is the only valid option
- Ok(Ok((op, _))) => {
- // Warn that `?` as a separator will be deprecated
- sess.buffer_lint(
- BufferedEarlyLintId::QuestionMarkMacroSep,
- op1_span,
- macro_node_id,
- "using `?` as a separator is deprecated and will be \
- a hard error in an upcoming edition",
- );
-
- return (Some(token::Question), op);
- }
-
- // #2 is a random token (this is an error) :(
- Ok(Err(_)) => op1_span,
-
- // #2 is not even a token at all :(
- Err(_) => op1_span,
- }
- } else {
- // `?` is not allowed as a Kleene op in 2015,
- // but is allowed in the 2018 edition
- sess.span_diagnostic
- .struct_span_err(op1_span, "expected `*` or `+`")
- .note("`?` is not a macro repetition operator in the 2015 edition, \
- but is accepted in the 2018 edition")
- .emit();
-
- // Return a dummy
- return (None, KleeneOp::ZeroOrMore);
- }
- }
-
- // #1 is a separator followed by #2, a KleeneOp
- Ok(Err(token)) => match parse_kleene_op(input, token.span) {
- // #2 is a `?`, which is not allowed as a Kleene op in 2015 edition,
- // but is allowed in the 2018 edition
- Ok(Ok((op, op2_span))) if op == KleeneOp::ZeroOrOne => {
- sess.span_diagnostic
- .struct_span_err(op2_span, "expected `*` or `+`")
- .note("`?` is not a macro repetition operator in the 2015 edition, \
- but is accepted in the 2018 edition")
- .emit();
-
- // Return a dummy
- return (None, KleeneOp::ZeroOrMore);
- }
-
- // #2 is a KleeneOp :D
- Ok(Ok((op, _))) => return (Some(token.kind), op),
-
- // #2 is a random token :(
- Ok(Err(token)) => token.span,
-
- // #2 is not a token at all :(
- Err(span) => span,
- },
-
- // #1 is not a token
- Err(span) => span,
- };
-
- sess.span_diagnostic.span_err(span, "expected `*` or `+`");
-
- // Return a dummy
- (None, KleeneOp::ZeroOrMore)
-}
-
-// `?` is a Kleene op, not a separator
-fn parse_sep_and_kleene_op_2018<I>(
- input: &mut Peekable<I>,
- span: Span,
- sess: &ParseSess,
- _features: &Features,
- _attrs: &[ast::Attribute],
-) -> (Option<TokenKind>, KleeneOp)
-where
- I: Iterator<Item = tokenstream::TokenTree>,
-{
+) -> (Option<Token>, KleeneOp) {
// We basically look at two token trees here, denoted as #1 and #2 below
let span = match parse_kleene_op(input, span) {
// #1 is a `?` (needs feature gate)
}
// #2 is a KleeneOp :D
- Ok(Ok((op, _))) => return (Some(token.kind), op),
+ Ok(Ok((op, _))) => return (Some(token), op),
// #2 is a random token :(
Ok(Err(token)) => token.span,
projects / rust.git / blobdiff