1 // Copyright 2017 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
12 use ext::tt::macro_parser;
13 use parse::{ParseSess, token};
16 use syntax_pos::{DUMMY_SP, Span, BytePos};
21 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)]
22 pub struct Delimited {
23 pub delim: token::DelimToken,
24 pub tts: Vec<TokenTree>,
28 pub fn open_token(&self) -> token::Token {
29 token::OpenDelim(self.delim)
32 pub fn close_token(&self) -> token::Token {
33 token::CloseDelim(self.delim)
36 pub fn open_tt(&self, span: Span) -> TokenTree {
37 let open_span = if span == DUMMY_SP {
40 span.with_lo(span.lo() + BytePos(self.delim.len() as u32))
42 TokenTree::Token(open_span, self.open_token())
45 pub fn close_tt(&self, span: Span) -> TokenTree {
46 let close_span = if span == DUMMY_SP {
49 span.with_lo(span.hi() - BytePos(self.delim.len() as u32))
51 TokenTree::Token(close_span, self.close_token())
55 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)]
56 pub struct SequenceRepetition {
57 /// The sequence of token trees
58 pub tts: Vec<TokenTree>,
59 /// The optional separator
60 pub separator: Option<token::Token>,
61 /// Whether the sequence can be repeated zero (*), or one or more times (+)
63 /// The number of `Match`s that appear in the sequence (and subsequences)
64 pub num_captures: usize,
67 /// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star)
68 /// for token sequences.
69 #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
75 /// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)`
76 /// are "first-class" token trees.
77 #[derive(Debug, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash)]
79 Token(Span, token::Token),
80 Delimited(Span, Rc<Delimited>),
81 /// A kleene-style repetition sequence
82 Sequence(Span, Rc<SequenceRepetition>),
84 MetaVar(Span, ast::Ident),
85 /// E.g. `$var:expr`. This is only used in the left hand side of MBE macros.
86 MetaVarDecl(Span, ast::Ident /* name to bind */, ast::Ident /* kind of nonterminal */),
90 pub fn len(&self) -> usize {
92 TokenTree::Delimited(_, ref delimed) => match delimed.delim {
93 token::NoDelim => delimed.tts.len(),
94 _ => delimed.tts.len() + 2,
96 TokenTree::Sequence(_, ref seq) => seq.tts.len(),
101 pub fn is_empty(&self) -> bool {
103 TokenTree::Delimited(_, ref delimed) => match delimed.delim {
104 token::NoDelim => delimed.tts.is_empty(),
107 TokenTree::Sequence(_, ref seq) => seq.tts.is_empty(),
112 pub fn get_tt(&self, index: usize) -> TokenTree {
113 match (self, index) {
114 (&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => {
115 delimed.tts[index].clone()
117 (&TokenTree::Delimited(span, ref delimed), _) => {
119 return delimed.open_tt(span);
121 if index == delimed.tts.len() + 1 {
122 return delimed.close_tt(span);
124 delimed.tts[index - 1].clone()
126 (&TokenTree::Sequence(_, ref seq), _) => seq.tts[index].clone(),
127 _ => panic!("Cannot expand a token tree"),
131 /// Retrieve the `TokenTree`'s span.
132 pub fn span(&self) -> Span {
134 TokenTree::Token(sp, _) |
135 TokenTree::MetaVar(sp, _) |
136 TokenTree::MetaVarDecl(sp, _, _) |
137 TokenTree::Delimited(sp, _) |
138 TokenTree::Sequence(sp, _) => sp,
143 pub fn parse(input: tokenstream::TokenStream, expect_matchers: bool, sess: &ParseSess)
145 let mut result = Vec::new();
146 let mut trees = input.trees();
147 while let Some(tree) = trees.next() {
148 let tree = parse_tree(tree, &mut trees, expect_matchers, sess);
150 TokenTree::MetaVar(start_sp, ident) if expect_matchers => {
151 let span = match trees.next() {
152 Some(tokenstream::TokenTree::Token(span, token::Colon)) => match trees.next() {
153 Some(tokenstream::TokenTree::Token(end_sp, ref tok)) => match tok.ident() {
155 let span = end_sp.with_lo(start_sp.lo());
156 result.push(TokenTree::MetaVarDecl(span, ident, kind));
161 tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
163 tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp),
165 sess.missing_fragment_specifiers.borrow_mut().insert(span);
166 result.push(TokenTree::MetaVarDecl(span, ident, keywords::Invalid.ident()));
168 _ => result.push(tree),
174 fn parse_tree<I>(tree: tokenstream::TokenTree,
176 expect_matchers: bool,
179 where I: Iterator<Item = tokenstream::TokenTree>,
182 tokenstream::TokenTree::Token(span, token::Dollar) => match trees.next() {
183 Some(tokenstream::TokenTree::Delimited(span, delimited)) => {
184 if delimited.delim != token::Paren {
185 let tok = pprust::token_to_string(&token::OpenDelim(delimited.delim));
186 let msg = format!("expected `(`, found `{}`", tok);
187 sess.span_diagnostic.span_err(span, &msg);
189 let sequence = parse(delimited.tts.into(), expect_matchers, sess);
190 let (separator, op) = parse_sep_and_kleene_op(trees, span, sess);
191 let name_captures = macro_parser::count_names(&sequence);
192 TokenTree::Sequence(span, Rc::new(SequenceRepetition {
196 num_captures: name_captures,
199 Some(tokenstream::TokenTree::Token(ident_span, ref token)) if token.is_ident() => {
200 let ident = token.ident().unwrap();
201 let span = ident_span.with_lo(span.lo());
202 if ident.name == keywords::Crate.name() {
203 let ident = ast::Ident { name: keywords::DollarCrate.name(), ..ident };
204 TokenTree::Token(span, token::Ident(ident))
206 TokenTree::MetaVar(span, ident)
209 Some(tokenstream::TokenTree::Token(span, tok)) => {
210 let msg = format!("expected identifier, found `{}`", pprust::token_to_string(&tok));
211 sess.span_diagnostic.span_err(span, &msg);
212 TokenTree::MetaVar(span, keywords::Invalid.ident())
214 None => TokenTree::Token(span, token::Dollar),
216 tokenstream::TokenTree::Token(span, tok) => TokenTree::Token(span, tok),
217 tokenstream::TokenTree::Delimited(span, delimited) => {
218 TokenTree::Delimited(span, Rc::new(Delimited {
219 delim: delimited.delim,
220 tts: parse(delimited.tts.into(), expect_matchers, sess),
226 fn parse_sep_and_kleene_op<I>(input: &mut I, span: Span, sess: &ParseSess)
227 -> (Option<token::Token>, KleeneOp)
228 where I: Iterator<Item = tokenstream::TokenTree>,
230 fn kleene_op(token: &token::Token) -> Option<KleeneOp> {
232 token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore),
233 token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore),
238 let span = match input.next() {
239 Some(tokenstream::TokenTree::Token(span, tok)) => match kleene_op(&tok) {
240 Some(op) => return (None, op),
241 None => match input.next() {
242 Some(tokenstream::TokenTree::Token(span, tok2)) => match kleene_op(&tok2) {
243 Some(op) => return (Some(tok), op),
246 tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
249 tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
252 sess.span_diagnostic.span_err(span, "expected `*` or `+`");
253 (None, KleeneOp::ZeroOrMore)