2 use crate::ext::base::ExtCtxt;
3 use crate::ext::expand::Marker;
4 use crate::ext::tt::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch};
5 use crate::ext::tt::quoted;
6 use crate::mut_visit::noop_visit_tt;
7 use crate::parse::token::{self, NtTT, TokenKind};
8 use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
10 use smallvec::{smallvec, SmallVec};
11 use syntax_pos::DUMMY_SP;
13 use rustc_data_structures::fx::FxHashMap;
14 use rustc_data_structures::sync::Lrc;
18 /// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`).
20 Delimited { forest: Lrc<quoted::Delimited>, idx: usize, span: DelimSpan },
21 Sequence { forest: Lrc<quoted::SequenceRepetition>, idx: usize, sep: Option<TokenKind> },
25 /// Construct a new frame around the delimited set of tokens.
26 fn new(tts: Vec<quoted::TokenTree>) -> Frame {
27 let forest = Lrc::new(quoted::Delimited { delim: token::NoDelim, tts: tts });
28 Frame::Delimited { forest: forest, idx: 0, span: DelimSpan::dummy() }
32 impl Iterator for Frame {
33 type Item = quoted::TokenTree;
35 fn next(&mut self) -> Option<quoted::TokenTree> {
37 Frame::Delimited { ref forest, ref mut idx, .. } => {
39 forest.tts.get(*idx - 1).cloned()
41 Frame::Sequence { ref forest, ref mut idx, .. } => {
43 forest.tts.get(*idx - 1).cloned()
49 /// This can do Macro-By-Example transcription.
50 /// - `interp` is a map of meta-variables to the tokens (non-terminals) they matched in the
51 /// invocation. We are assuming we already know there is a match.
52 /// - `src` is the RHS of the MBE, that is, the "example" we are filling in.
57 /// macro_rules! foo {
58 /// ($id:ident) => { println!("{}", stringify!($id)); }
64 /// `interp` would contain `$id => bar` and `src` would contain `println!("{}", stringify!($id));`.
66 /// `transcribe` would return a `TokenStream` containing `println!("{}", stringify!(bar));`.
68 /// Along the way, we do some additional error checking.
71 interp: &FxHashMap<Ident, Rc<NamedMatch>>,
72 src: Vec<quoted::TokenTree>,
74 // Nothing for us to transcribe...
76 return TokenStream::empty();
79 // We descend into the RHS (`src`), expanding things as we go. This stack contains the things
80 // we have yet to expand/are still expanding. We start the stack off with the whole RHS.
81 let mut stack: SmallVec<[Frame; 1]> = smallvec![Frame::new(src)];
83 // As we descend in the RHS, we will need to be able to match nested sequences of matchers.
84 // `repeats` keeps track of where we are in matching at each level, with the last element being
85 // the most deeply nested sequence. This is used as a stack.
86 let mut repeats = Vec::new();
88 // `result` contains resulting token stream from the TokenTree we just finished processing. At
89 // the end, this will contain the full result of transcription, but at arbitrary points during
90 // `transcribe`, `result` will contain subsets of the final result.
92 // Specifically, as we descend into each TokenTree, we will push the existing results onto the
93 // `result_stack` and clear `results`. We will then produce the results of transcribing the
94 // TokenTree into `results`. Then, as we unwind back out of the `TokenTree`, we will pop the
95 // `result_stack` and append `results` too it to produce the new `results` up to that point.
97 // Thus, if we try to pop the `result_stack` and it is empty, we have reached the top-level
98 // again, and we are done transcribing.
99 let mut result: Vec<TreeAndJoint> = Vec::new();
100 let mut result_stack = Vec::new();
103 // Look at the last frame on the stack.
104 let tree = if let Some(tree) = stack.last_mut().unwrap().next() {
105 // If it still has a TokenTree we have not looked at yet, use that tree.
108 // The else-case never produces a value for `tree` (it `continue`s or `return`s).
110 // Otherwise, if we have just reached the end of a sequence and we can keep repeating,
111 // go back to the beginning of the sequence.
112 if let Frame::Sequence { ref mut idx, ref sep, .. } = *stack.last_mut().unwrap() {
113 let (ref mut repeat_idx, repeat_len) = *repeats.last_mut().unwrap();
115 if *repeat_idx < repeat_len {
117 if let Some(sep) = sep.clone() {
118 let prev_span = match result.last() {
119 Some((tt, _)) => tt.span(),
122 result.push(TokenTree::Token(prev_span, sep).into());
128 // We are done with the top of the stack. Pop it. Depending on what it was, we do
129 // different things. Note that the outermost item must be the delimited, wrapped RHS
130 // that was passed in originally to `transcribe`.
131 match stack.pop().unwrap() {
132 // Done with a sequence. Pop from repeats.
133 Frame::Sequence { .. } => {
137 // We are done processing a Delimited. If this is the top-level delimited, we are
138 // done. Otherwise, we unwind the result_stack to append what we have produced to
139 // any previous results.
140 Frame::Delimited { forest, span, .. } => {
141 if result_stack.is_empty() {
142 // No results left to compute! We are back at the top-level.
143 return TokenStream::new(result);
146 // Step back into the parent Delimited.
148 TokenTree::Delimited(span, forest.delim, TokenStream::new(result).into());
149 result = result_stack.pop().unwrap();
150 result.push(tree.into());
156 // At this point, we know we are in the middle of a TokenTree (the last one on `stack`).
157 // `tree` contains the next `TokenTree` to be processed.
159 // We are descending into a sequence. We first make sure that the matchers in the RHS
160 // and the matches in `interp` have the same shape. Otherwise, either the caller or the
161 // macro writer has made a mistake.
162 seq @ quoted::TokenTree::Sequence(..) => {
163 match lockstep_iter_size(&seq, interp, &repeats) {
164 LockstepIterSize::Unconstrained => {
166 seq.span(), /* blame macro writer */
167 "attempted to repeat an expression containing no syntax variables \
168 matched as repeating at this depth",
172 LockstepIterSize::Contradiction(ref msg) => {
173 // FIXME: this really ought to be caught at macro definition time... It
174 // happens when two meta-variables are used in the same repetition in a
175 // sequence, but they come from different sequence matchers and repeat
176 // different amounts.
177 cx.span_fatal(seq.span(), &msg[..]);
180 LockstepIterSize::Constraint(len, _) => {
181 // We do this to avoid an extra clone above. We know that this is a
183 let (sp, seq) = if let quoted::TokenTree::Sequence(sp, seq) = seq {
189 // Is the repetition empty?
191 if seq.op == quoted::KleeneOp::OneOrMore {
192 // FIXME: this really ought to be caught at macro definition
193 // time... It happens when the Kleene operator in the matcher and
194 // the body for the same meta-variable do not match.
195 cx.span_fatal(sp.entire(), "this must repeat at least once");
198 // 0 is the initial counter (we have done 0 repretitions so far). `len`
199 // is the total number of reptitions we should generate.
200 repeats.push((0, len));
202 // The first time we encounter the sequence we push it to the stack. It
203 // then gets reused (see the beginning of the loop) until we are done
205 stack.push(Frame::Sequence {
207 sep: seq.separator.clone(),
215 // Replace the meta-var with the matched token tree from the invocation.
216 quoted::TokenTree::MetaVar(mut sp, ident) => {
217 // Find the matched nonterminal from the macro invocation, and use it to replace
219 if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) {
220 if let MatchedNonterminal(ref nt) = *cur_matched {
221 // FIXME #2887: why do we apply a mark when matching a token tree meta-var
222 // (e.g. `$x:tt`), but not when we are matching any other type of token
224 if let NtTT(ref tt) = **nt {
225 result.push(tt.clone().into());
227 sp = sp.apply_mark(cx.current_expansion.mark);
228 let token = TokenTree::Token(sp, token::Interpolated(nt.clone()));
229 result.push(token.into());
232 // We were unable to descend far enough. This is an error.
234 sp, /* blame the macro writer */
235 &format!("variable '{}' is still repeating at this depth", ident),
239 // If we aren't able to match the meta-var, we push it back into the result but
240 // with modified syntax context. (I believe this supports nested macros).
242 Ident::new(ident.name, ident.span.apply_mark(cx.current_expansion.mark));
243 sp = sp.apply_mark(cx.current_expansion.mark);
244 result.push(TokenTree::Token(sp, token::Dollar).into());
245 result.push(TokenTree::Token(sp, token::TokenKind::from_ast_ident(ident)).into());
249 // If we are entering a new delimiter, we push its contents to the `stack` to be
250 // processed, and we push all of the currently produced results to the `result_stack`.
251 // We will produce all of the results of the inside of the `Delimited` and then we will
252 // jump back out of the Delimited, pop the result_stack and add the new results back to
253 // the previous results (from outside the Delimited).
254 quoted::TokenTree::Delimited(mut span, delimited) => {
255 span = span.apply_mark(cx.current_expansion.mark);
256 stack.push(Frame::Delimited { forest: delimited, idx: 0, span: span });
257 result_stack.push(mem::replace(&mut result, Vec::new()));
260 // Nothing much to do here. Just push the token to the result, being careful to
261 // preserve syntax context.
262 quoted::TokenTree::Token(sp, tok) => {
263 let mut marker = Marker(cx.current_expansion.mark);
264 let mut tt = TokenTree::Token(sp, tok);
265 noop_visit_tt(&mut tt, &mut marker);
266 result.push(tt.into());
269 // There should be no meta-var declarations in the invocation of a macro.
270 quoted::TokenTree::MetaVarDecl(..) => panic!("unexpected `TokenTree::MetaVarDecl"),
275 /// Lookup the meta-var named `ident` and return the matched token tree from the invocation using
276 /// the set of matches `interpolations`.
278 /// See the definition of `repeats` in the `transcribe` function. `repeats` is used to descend
279 /// into the right place in nested matchers. If we attempt to descend too far, the macro writer has
280 /// made a mistake, and we return `None`.
281 fn lookup_cur_matched(
283 interpolations: &FxHashMap<Ident, Rc<NamedMatch>>,
284 repeats: &[(usize, usize)],
285 ) -> Option<Rc<NamedMatch>> {
286 interpolations.get(&ident).map(|matched| {
287 let mut matched = matched.clone();
288 for &(idx, _) in repeats {
289 let m = matched.clone();
291 MatchedNonterminal(_) => break,
292 MatchedSeq(ref ads, _) => matched = Rc::new(ads[idx].clone()),
300 /// An accumulator over a TokenTree to be used with `fold`. During transcription, we need to make
301 /// sure that the size of each sequence and all of its nested sequences are the same as the sizes
302 /// of all the matched (nested) sequences in the macro invocation. If they don't match, somebody
303 /// has made a mistake (either the macro writer or caller).
305 enum LockstepIterSize {
306 /// No constraints on length of matcher. This is true for any TokenTree variants except a
307 /// `MetaVar` with an actual `MatchedSeq` (as opposed to a `MatchedNonterminal`).
310 /// A `MetaVar` with an actual `MatchedSeq`. The length of the match and the name of the
311 /// meta-var are returned.
312 Constraint(usize, Ident),
314 /// Two `Constraint`s on the same sequence had different lengths. This is an error.
315 Contradiction(String),
318 impl LockstepIterSize {
319 /// Find incompatibilities in matcher/invocation sizes.
320 /// - `Unconstrained` is compatible with everything.
321 /// - `Contradiction` is incompatible with everything.
322 /// - `Constraint(len)` is only compatible with other constraints of the same length.
323 fn with(self, other: LockstepIterSize) -> LockstepIterSize {
325 LockstepIterSize::Unconstrained => other,
326 LockstepIterSize::Contradiction(_) => self,
327 LockstepIterSize::Constraint(l_len, ref l_id) => match other {
328 LockstepIterSize::Unconstrained => self,
329 LockstepIterSize::Contradiction(_) => other,
330 LockstepIterSize::Constraint(r_len, _) if l_len == r_len => self,
331 LockstepIterSize::Constraint(r_len, r_id) => {
333 "meta-variable `{}` repeats {} times, but `{}` repeats {} times",
334 l_id, l_len, r_id, r_len
336 LockstepIterSize::Contradiction(msg)
343 /// Given a `tree`, make sure that all sequences have the same length as the matches for the
344 /// appropriate meta-vars in `interpolations`.
346 /// Note that if `repeats` does not match the exact correct depth of a meta-var,
347 /// `lookup_cur_matched` will return `None`, which is why this still works even in the presnece of
348 /// multiple nested matcher sequences.
349 fn lockstep_iter_size(
350 tree: "ed::TokenTree,
351 interpolations: &FxHashMap<Ident, Rc<NamedMatch>>,
352 repeats: &[(usize, usize)],
353 ) -> LockstepIterSize {
354 use quoted::TokenTree;
356 TokenTree::Delimited(_, ref delimed) => {
357 delimed.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| {
358 size.with(lockstep_iter_size(tt, interpolations, repeats))
361 TokenTree::Sequence(_, ref seq) => {
362 seq.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| {
363 size.with(lockstep_iter_size(tt, interpolations, repeats))
366 TokenTree::MetaVar(_, name) | TokenTree::MetaVarDecl(_, name, _) => {
367 match lookup_cur_matched(name, interpolations, repeats) {
368 Some(matched) => match *matched {
369 MatchedNonterminal(_) => LockstepIterSize::Unconstrained,
370 MatchedSeq(ref ads, _) => LockstepIterSize::Constraint(ads.len(), name),
372 _ => LockstepIterSize::Unconstrained,
375 TokenTree::Token(..) => LockstepIterSize::Unconstrained,