src/libsyntax/ext/tt/quoted.rs

   1 // Copyright 2017 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 use ast;
  12 use ext::tt::macro_parser;
  13 use parse::{ParseSess, token};
  14 use print::pprust;
  15 use symbol::keywords;
  16 use syntax_pos::{DUMMY_SP, Span, BytePos};
  17 use tokenstream;
  18
  19 use std::rc::Rc;
  20
  21 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)]
  22 pub struct Delimited {
  23     pub delim: token::DelimToken,
  24     pub tts: Vec<TokenTree>,
  25 }
  26
  27 impl Delimited {
  28     pub fn open_token(&self) -> token::Token {
  29         token::OpenDelim(self.delim)
  30     }
  31
  32     pub fn close_token(&self) -> token::Token {
  33         token::CloseDelim(self.delim)
  34     }
  35
  36     pub fn open_tt(&self, span: Span) -> TokenTree {
  37         let open_span = if span == DUMMY_SP {
  38             DUMMY_SP
  39         } else {
  40             span.with_lo(span.lo() + BytePos(self.delim.len() as u32))
  41         };
  42         TokenTree::Token(open_span, self.open_token())
  43     }
  44
  45     pub fn close_tt(&self, span: Span) -> TokenTree {
  46         let close_span = if span == DUMMY_SP {
  47             DUMMY_SP
  48         } else {
  49             span.with_lo(span.hi() - BytePos(self.delim.len() as u32))
  50         };
  51         TokenTree::Token(close_span, self.close_token())
  52     }
  53 }
  54
  55 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)]
  56 pub struct SequenceRepetition {
  57     /// The sequence of token trees
  58     pub tts: Vec<TokenTree>,
  59     /// The optional separator
  60     pub separator: Option<token::Token>,
  61     /// Whether the sequence can be repeated zero (*), or one or more times (+)
  62     pub op: KleeneOp,
  63     /// The number of `Match`s that appear in the sequence (and subsequences)
  64     pub num_captures: usize,
  65 }
  66
  67 /// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star)
  68 /// for token sequences.
  69 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
  70 pub enum KleeneOp {
  71     ZeroOrMore,
  72     OneOrMore,
  73 }
  74
  75 /// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)`
  76 /// are "first-class" token trees.
  77 #[derive(Debug, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash)]
  78 pub enum TokenTree {
  79     Token(Span, token::Token),
  80     Delimited(Span, Rc<Delimited>),
  81     /// A kleene-style repetition sequence
  82     Sequence(Span, Rc<SequenceRepetition>),
  83     /// E.g. `$var`
  84     MetaVar(Span, ast::Ident),
  85     /// E.g. `$var:expr`. This is only used in the left hand side of MBE macros.
  86     MetaVarDecl(Span, ast::Ident /* name to bind */, ast::Ident /* kind of nonterminal */),
  87 }
  88
  89 impl TokenTree {
  90     pub fn len(&self) -> usize {
  91         match *self {
  92             TokenTree::Delimited(_, ref delimed) => match delimed.delim {
  93                 token::NoDelim => delimed.tts.len(),
  94                 _ => delimed.tts.len() + 2,
  95             },
  96             TokenTree::Sequence(_, ref seq) => seq.tts.len(),
  97             _ => 0,
  98         }
  99     }
 100
 101     pub fn is_empty(&self) -> bool {
 102         match *self {
 103             TokenTree::Delimited(_, ref delimed) => match delimed.delim {
 104                 token::NoDelim => delimed.tts.is_empty(),
 105                 _ => false,
 106             },
 107             TokenTree::Sequence(_, ref seq) => seq.tts.is_empty(),
 108             _ => true,
 109         }
 110     }
 111
 112     pub fn get_tt(&self, index: usize) -> TokenTree {
 113         match (self, index) {
 114             (&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => {
 115                 delimed.tts[index].clone()
 116             }
 117             (&TokenTree::Delimited(span, ref delimed), _) => {
 118                 if index == 0 {
 119                     return delimed.open_tt(span);
 120                 }
 121                 if index == delimed.tts.len() + 1 {
 122                     return delimed.close_tt(span);
 123                 }
 124                 delimed.tts[index - 1].clone()
 125             }
 126             (&TokenTree::Sequence(_, ref seq), _) => seq.tts[index].clone(),
 127             _ => panic!("Cannot expand a token tree"),
 128         }
 129     }
 130
 131     /// Retrieve the `TokenTree`'s span.
 132     pub fn span(&self) -> Span {
 133         match *self {
 134             TokenTree::Token(sp, _) |
 135             TokenTree::MetaVar(sp, _) |
 136             TokenTree::MetaVarDecl(sp, _, _) |
 137             TokenTree::Delimited(sp, _) |
 138             TokenTree::Sequence(sp, _) => sp,
 139         }
 140     }
 141 }
 142
 143 pub fn parse(input: tokenstream::TokenStream, expect_matchers: bool, sess: &ParseSess)
 144              -> Vec<TokenTree> {
 145     let mut result = Vec::new();
 146     let mut trees = input.trees();
 147     while let Some(tree) = trees.next() {
 148         let tree = parse_tree(tree, &mut trees, expect_matchers, sess);
 149         match tree {
 150             TokenTree::MetaVar(start_sp, ident) if expect_matchers => {
 151                 let span = match trees.next() {
 152                     Some(tokenstream::TokenTree::Token(span, token::Colon)) => match trees.next() {
 153                         Some(tokenstream::TokenTree::Token(end_sp, ref tok)) => match tok.ident() {
 154                             Some(kind) => {
 155                                 let span = end_sp.with_lo(start_sp.lo());
 156                                 result.push(TokenTree::MetaVarDecl(span, ident, kind));
 157                                 continue
 158                             }
 159                             _ => end_sp,
 160                         },
 161                         tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
 162                     },
 163                     tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp),
 164                 };
 165                 sess.missing_fragment_specifiers.borrow_mut().insert(span);
 166                 result.push(TokenTree::MetaVarDecl(span, ident, keywords::Invalid.ident()));
 167             }
 168             _ => result.push(tree),
 169         }
 170     }
 171     result
 172 }
 173
 174 fn parse_tree<I>(tree: tokenstream::TokenTree,
 175                  trees: &mut I,
 176                  expect_matchers: bool,
 177                  sess: &ParseSess)
 178                  -> TokenTree
 179     where I: Iterator<Item = tokenstream::TokenTree>,
 180 {
 181     match tree {
 182         tokenstream::TokenTree::Token(span, token::Dollar) => match trees.next() {
 183             Some(tokenstream::TokenTree::Delimited(span, delimited)) => {
 184                 if delimited.delim != token::Paren {
 185                     let tok = pprust::token_to_string(&token::OpenDelim(delimited.delim));
 186                     let msg = format!("expected `(`, found `{}`", tok);
 187                     sess.span_diagnostic.span_err(span, &msg);
 188                 }
 189                 let sequence = parse(delimited.tts.into(), expect_matchers, sess);
 190                 let (separator, op) = parse_sep_and_kleene_op(trees, span, sess);
 191                 let name_captures = macro_parser::count_names(&sequence);
 192                 TokenTree::Sequence(span, Rc::new(SequenceRepetition {
 193                     tts: sequence,
 194                     separator,
 195                     op,
 196                     num_captures: name_captures,
 197                 }))
 198             }
 199             Some(tokenstream::TokenTree::Token(ident_span, ref token)) if token.is_ident() => {
 200                 let ident = token.ident().unwrap();
 201                 let span = ident_span.with_lo(span.lo());
 202                 if ident.name == keywords::Crate.name() {
 203                     let ident = ast::Ident { name: keywords::DollarCrate.name(), ..ident };
 204                     TokenTree::Token(span, token::Ident(ident))
 205                 } else {
 206                     TokenTree::MetaVar(span, ident)
 207                 }
 208             }
 209             Some(tokenstream::TokenTree::Token(span, tok)) => {
 210                 let msg = format!("expected identifier, found `{}`", pprust::token_to_string(&tok));
 211                 sess.span_diagnostic.span_err(span, &msg);
 212                 TokenTree::MetaVar(span, keywords::Invalid.ident())
 213             }
 214             None => TokenTree::Token(span, token::Dollar),
 215         },
 216         tokenstream::TokenTree::Token(span, tok) => TokenTree::Token(span, tok),
 217         tokenstream::TokenTree::Delimited(span, delimited) => {
 218             TokenTree::Delimited(span, Rc::new(Delimited {
 219                 delim: delimited.delim,
 220                 tts: parse(delimited.tts.into(), expect_matchers, sess),
 221             }))
 222         }
 223     }
 224 }
 225
 226 fn parse_sep_and_kleene_op<I>(input: &mut I, span: Span, sess: &ParseSess)
 227                               -> (Option<token::Token>, KleeneOp)
 228     where I: Iterator<Item = tokenstream::TokenTree>,
 229 {
 230     fn kleene_op(token: &token::Token) -> Option<KleeneOp> {
 231         match *token {
 232             token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore),
 233             token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore),
 234             _ => None,
 235         }
 236     }
 237
 238     let span = match input.next() {
 239         Some(tokenstream::TokenTree::Token(span, tok)) => match kleene_op(&tok) {
 240             Some(op) => return (None, op),
 241             None => match input.next() {
 242                 Some(tokenstream::TokenTree::Token(span, tok2)) => match kleene_op(&tok2) {
 243                     Some(op) => return (Some(tok), op),
 244                     None => span,
 245                 },
 246                 tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
 247             }
 248         },
 249         tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
 250     };
 251
 252     sess.span_diagnostic.span_err(span, "expected `*` or `+`");
 253     (None, KleeneOp::ZeroOrMore)
 254 }