1 use crate::base::ExtCtxt;
3 use rustc_parse::lexer::nfc_normalize;
4 use rustc_parse::{nt_to_tokenstream, parse_stream_from_source_str};
5 use rustc_span::symbol::{kw, sym, Symbol};
6 use rustc_span::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span};
8 use syntax::print::pprust;
9 use syntax::sess::ParseSess;
11 use syntax::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint};
12 use syntax::util::comments;
14 use errors::Diagnostic;
15 use rustc_data_structures::sync::Lrc;
17 use pm::bridge::{server, TokenTree};
18 use pm::{Delimiter, Level, LineColumn, Spacing};
20 use std::{ascii, panic};
22 trait FromInternal<T> {
23 fn from_internal(x: T) -> Self;
27 fn to_internal(self) -> T;
30 impl FromInternal<token::DelimToken> for Delimiter {
31 fn from_internal(delim: token::DelimToken) -> Delimiter {
33 token::Paren => Delimiter::Parenthesis,
34 token::Brace => Delimiter::Brace,
35 token::Bracket => Delimiter::Bracket,
36 token::NoDelim => Delimiter::None,
41 impl ToInternal<token::DelimToken> for Delimiter {
42 fn to_internal(self) -> token::DelimToken {
44 Delimiter::Parenthesis => token::Paren,
45 Delimiter::Brace => token::Brace,
46 Delimiter::Bracket => token::Bracket,
47 Delimiter::None => token::NoDelim,
52 impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
53 for TokenTree<Group, Punct, Ident, Literal>
56 ((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec<Self>),
60 let joint = is_joint == Joint;
61 let Token { kind, span } = match tree {
62 tokenstream::TokenTree::Delimited(span, delim, tts) => {
63 let delimiter = Delimiter::from_internal(delim);
64 return TokenTree::Group(Group { delimiter, stream: tts.into(), span });
66 tokenstream::TokenTree::Token(token) => token,
70 ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => (
71 TokenTree::$ty(self::$ty {
72 $($field $(: $value)*,)+
76 ($ty:ident::$method:ident($($value:expr),*)) => (
77 TokenTree::$ty(self::$ty::$method($($value,)* span))
82 tt!(Punct::new($a, joint))
84 ($a:expr, $b:expr) => {{
85 stack.push(tt!(Punct::new($b, joint)));
86 tt!(Punct::new($a, true))
88 ($a:expr, $b:expr, $c:expr) => {{
89 stack.push(tt!(Punct::new($c, joint)));
90 stack.push(tt!(Punct::new($b, true)));
91 tt!(Punct::new($a, true))
99 EqEq => op!('=', '='),
103 AndAnd => op!('&', '&'),
104 OrOr => op!('|', '|'),
107 BinOp(Plus) => op!('+'),
108 BinOp(Minus) => op!('-'),
109 BinOp(Star) => op!('*'),
110 BinOp(Slash) => op!('/'),
111 BinOp(Percent) => op!('%'),
112 BinOp(Caret) => op!('^'),
113 BinOp(And) => op!('&'),
114 BinOp(Or) => op!('|'),
115 BinOp(Shl) => op!('<', '<'),
116 BinOp(Shr) => op!('>', '>'),
117 BinOpEq(Plus) => op!('+', '='),
118 BinOpEq(Minus) => op!('-', '='),
119 BinOpEq(Star) => op!('*', '='),
120 BinOpEq(Slash) => op!('/', '='),
121 BinOpEq(Percent) => op!('%', '='),
122 BinOpEq(Caret) => op!('^', '='),
123 BinOpEq(And) => op!('&', '='),
124 BinOpEq(Or) => op!('|', '='),
125 BinOpEq(Shl) => op!('<', '<', '='),
126 BinOpEq(Shr) => op!('>', '>', '='),
129 DotDot => op!('.', '.'),
130 DotDotDot => op!('.', '.', '.'),
131 DotDotEq => op!('.', '.', '='),
135 ModSep => op!(':', ':'),
136 RArrow => op!('-', '>'),
137 LArrow => op!('<', '-'),
138 FatArrow => op!('=', '>'),
141 Question => op!('?'),
142 SingleQuote => op!('\''),
144 Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()),
145 Ident(name, is_raw) => tt!(Ident::new(name, is_raw)),
147 let ident = ast::Ident::new(name, span).without_first_quote();
148 stack.push(tt!(Ident::new(ident.name, false)));
149 tt!(Punct::new('\'', true))
151 Literal(lit) => tt!(Literal { lit }),
153 let style = comments::doc_comment_style(&c.as_str());
154 let stripped = comments::strip_doc_comment_decoration(&c.as_str());
155 let mut escaped = String::new();
156 for ch in stripped.chars() {
157 escaped.extend(ch.escape_debug());
160 Ident(sym::doc, false),
162 TokenKind::lit(token::Str, Symbol::intern(&escaped), None),
165 .map(|kind| tokenstream::TokenTree::token(kind, span))
167 stack.push(TokenTree::Group(Group {
168 delimiter: Delimiter::Bracket,
170 span: DelimSpan::from_single(span),
172 if style == ast::AttrStyle::Inner {
173 stack.push(tt!(Punct::new('!', false)));
175 tt!(Punct::new('#', false))
178 Interpolated(nt) => {
179 let stream = nt_to_tokenstream(&nt, sess, span);
180 TokenTree::Group(Group {
181 delimiter: Delimiter::None,
183 span: DelimSpan::from_single(span),
187 OpenDelim(..) | CloseDelim(..) => unreachable!(),
188 Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => unreachable!(),
193 impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> {
194 fn to_internal(self) -> TokenStream {
195 use syntax::token::*;
197 let (ch, joint, span) = match self {
198 TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span),
199 TokenTree::Group(Group { delimiter, stream, span }) => {
200 return tokenstream::TokenTree::Delimited(
202 delimiter.to_internal(),
207 TokenTree::Ident(self::Ident { sym, is_raw, span }) => {
208 return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into();
210 TokenTree::Literal(self::Literal {
211 lit: token::Lit { kind: token::Integer, symbol, suffix },
213 }) if symbol.as_str().starts_with("-") => {
214 let minus = BinOp(BinOpToken::Minus);
215 let symbol = Symbol::intern(&symbol.as_str()[1..]);
216 let integer = TokenKind::lit(token::Integer, symbol, suffix);
217 let a = tokenstream::TokenTree::token(minus, span);
218 let b = tokenstream::TokenTree::token(integer, span);
219 return vec![a, b].into_iter().collect();
221 TokenTree::Literal(self::Literal {
222 lit: token::Lit { kind: token::Float, symbol, suffix },
224 }) if symbol.as_str().starts_with("-") => {
225 let minus = BinOp(BinOpToken::Minus);
226 let symbol = Symbol::intern(&symbol.as_str()[1..]);
227 let float = TokenKind::lit(token::Float, symbol, suffix);
228 let a = tokenstream::TokenTree::token(minus, span);
229 let b = tokenstream::TokenTree::token(float, span);
230 return vec![a, b].into_iter().collect();
232 TokenTree::Literal(self::Literal { lit, span }) => {
233 return tokenstream::TokenTree::token(Literal(lit), span).into();
237 let kind = match ch {
247 '%' => BinOp(Percent),
263 let tree = tokenstream::TokenTree::token(kind, span);
264 TokenStream::new(vec![(tree, if joint { Joint } else { NonJoint })])
268 impl ToInternal<errors::Level> for Level {
269 fn to_internal(self) -> errors::Level {
271 Level::Error => errors::Level::Error,
272 Level::Warning => errors::Level::Warning,
273 Level::Note => errors::Level::Note,
274 Level::Help => errors::Level::Help,
275 _ => unreachable!("unknown proc_macro::Level variant: {:?}", self),
281 pub struct TokenStreamIter {
282 cursor: tokenstream::Cursor,
283 stack: Vec<TokenTree<Group, Punct, Ident, Literal>>,
288 delimiter: Delimiter,
293 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
296 // NB. not using `Spacing` here because it doesn't implement `Hash`.
302 fn new(ch: char, joint: bool, span: Span) -> Punct {
303 const LEGAL_CHARS: &[char] = &[
304 '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';',
305 ':', '#', '$', '?', '\'',
307 if !LEGAL_CHARS.contains(&ch) {
308 panic!("unsupported character `{:?}`", ch)
310 Punct { ch, joint, span }
314 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
322 fn is_valid(string: &str) -> bool {
323 let mut chars = string.chars();
324 if let Some(start) = chars.next() {
325 rustc_lexer::is_id_start(start) && chars.all(rustc_lexer::is_id_continue)
330 fn new(sym: Symbol, is_raw: bool, span: Span) -> Ident {
331 let sym = nfc_normalize(&sym.as_str());
332 let string = sym.as_str();
333 if !Self::is_valid(&string) {
334 panic!("`{:?}` is not a valid identifier", string)
336 if is_raw && !sym.can_be_raw() {
337 panic!("`{}` cannot be a raw identifier", string);
339 Ident { sym, is_raw, span }
341 fn dollar_crate(span: Span) -> Ident {
342 // `$crate` is accepted as an ident only if it comes from the compiler.
343 Ident { sym: kw::DollarCrate, is_raw: false, span }
347 // FIXME(eddyb) `Literal` should not expose internal `Debug` impls.
348 #[derive(Clone, Debug)]
354 pub(crate) struct Rustc<'a> {
362 pub fn new(cx: &'a ExtCtxt<'_>) -> Self {
363 let expn_data = cx.current_expansion.id.expn_data();
366 def_site: cx.with_def_site_ctxt(expn_data.def_site),
367 call_site: cx.with_call_site_ctxt(expn_data.call_site),
368 mixed_site: cx.with_mixed_site_ctxt(expn_data.call_site),
372 fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Literal {
373 Literal { lit: token::Lit::new(kind, symbol, suffix), span: server::Span::call_site(self) }
377 impl server::Types for Rustc<'_> {
378 type TokenStream = TokenStream;
379 type TokenStreamBuilder = tokenstream::TokenStreamBuilder;
380 type TokenStreamIter = TokenStreamIter;
384 type Literal = Literal;
385 type SourceFile = Lrc<SourceFile>;
386 type MultiSpan = Vec<Span>;
387 type Diagnostic = Diagnostic;
391 impl server::TokenStream for Rustc<'_> {
392 fn new(&mut self) -> Self::TokenStream {
393 TokenStream::default()
395 fn is_empty(&mut self, stream: &Self::TokenStream) -> bool {
398 fn from_str(&mut self, src: &str) -> Self::TokenStream {
399 parse_stream_from_source_str(
400 FileName::proc_macro_source_code(src),
403 Some(self.call_site),
406 fn to_string(&mut self, stream: &Self::TokenStream) -> String {
407 pprust::tts_to_string(stream.clone())
411 tree: TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>,
412 ) -> Self::TokenStream {
415 fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter {
416 TokenStreamIter { cursor: stream.trees(), stack: vec![] }
420 impl server::TokenStreamBuilder for Rustc<'_> {
421 fn new(&mut self) -> Self::TokenStreamBuilder {
422 tokenstream::TokenStreamBuilder::new()
424 fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) {
425 builder.push(stream);
427 fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream {
432 impl server::TokenStreamIter for Rustc<'_> {
435 iter: &mut Self::TokenStreamIter,
436 ) -> Option<TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>> {
438 let tree = iter.stack.pop().or_else(|| {
439 let next = iter.cursor.next_with_joint()?;
440 Some(TokenTree::from_internal((next, self.sess, &mut iter.stack)))
442 // HACK: The condition "dummy span + group with empty delimiter" represents an AST
443 // fragment approximately converted into a token stream. This may happen, for
444 // example, with inputs to proc macro attributes, including derives. Such "groups"
445 // need to flattened during iteration over stream's token trees.
446 // Eventually this needs to be removed in favor of keeping original token trees
447 // and not doing the roundtrip through AST.
448 if let TokenTree::Group(ref group) = tree {
449 if group.delimiter == Delimiter::None && group.span.entire().is_dummy() {
450 iter.cursor.append(group.stream.clone());
459 impl server::Group for Rustc<'_> {
460 fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group {
461 Group { delimiter, stream, span: DelimSpan::from_single(server::Span::call_site(self)) }
463 fn delimiter(&mut self, group: &Self::Group) -> Delimiter {
466 fn stream(&mut self, group: &Self::Group) -> Self::TokenStream {
469 fn span(&mut self, group: &Self::Group) -> Self::Span {
472 fn span_open(&mut self, group: &Self::Group) -> Self::Span {
475 fn span_close(&mut self, group: &Self::Group) -> Self::Span {
478 fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) {
479 group.span = DelimSpan::from_single(span);
483 impl server::Punct for Rustc<'_> {
484 fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct {
485 Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self))
487 fn as_char(&mut self, punct: Self::Punct) -> char {
490 fn spacing(&mut self, punct: Self::Punct) -> Spacing {
491 if punct.joint { Spacing::Joint } else { Spacing::Alone }
493 fn span(&mut self, punct: Self::Punct) -> Self::Span {
496 fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct {
497 Punct { span, ..punct }
501 impl server::Ident for Rustc<'_> {
502 fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident {
503 Ident::new(Symbol::intern(string), is_raw, span)
505 fn span(&mut self, ident: Self::Ident) -> Self::Span {
508 fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident {
509 Ident { span, ..ident }
513 impl server::Literal for Rustc<'_> {
514 // FIXME(eddyb) `Literal` should not expose internal `Debug` impls.
515 fn debug(&mut self, literal: &Self::Literal) -> String {
516 format!("{:?}", literal)
518 fn integer(&mut self, n: &str) -> Self::Literal {
519 self.lit(token::Integer, Symbol::intern(n), None)
521 fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal {
522 self.lit(token::Integer, Symbol::intern(n), Some(Symbol::intern(kind)))
524 fn float(&mut self, n: &str) -> Self::Literal {
525 self.lit(token::Float, Symbol::intern(n), None)
527 fn f32(&mut self, n: &str) -> Self::Literal {
528 self.lit(token::Float, Symbol::intern(n), Some(sym::f32))
530 fn f64(&mut self, n: &str) -> Self::Literal {
531 self.lit(token::Float, Symbol::intern(n), Some(sym::f64))
533 fn string(&mut self, string: &str) -> Self::Literal {
534 let mut escaped = String::new();
535 for ch in string.chars() {
536 escaped.extend(ch.escape_debug());
538 self.lit(token::Str, Symbol::intern(&escaped), None)
540 fn character(&mut self, ch: char) -> Self::Literal {
541 let mut escaped = String::new();
542 escaped.extend(ch.escape_unicode());
543 self.lit(token::Char, Symbol::intern(&escaped), None)
545 fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal {
549 .flat_map(ascii::escape_default)
550 .map(Into::<char>::into)
551 .collect::<String>();
552 self.lit(token::ByteStr, Symbol::intern(&string), None)
554 fn span(&mut self, literal: &Self::Literal) -> Self::Span {
557 fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) {
562 literal: &Self::Literal,
565 ) -> Option<Self::Span> {
566 let span = literal.span;
567 let length = span.hi().to_usize() - span.lo().to_usize();
569 let start = match start {
570 Bound::Included(lo) => lo,
571 Bound::Excluded(lo) => lo + 1,
572 Bound::Unbounded => 0,
575 let end = match end {
576 Bound::Included(hi) => hi + 1,
577 Bound::Excluded(hi) => hi,
578 Bound::Unbounded => length,
581 // Bounds check the values, preventing addition overflow and OOB spans.
582 if start > u32::max_value() as usize
583 || end > u32::max_value() as usize
584 || (u32::max_value() - start as u32) < span.lo().to_u32()
585 || (u32::max_value() - end as u32) < span.lo().to_u32()
592 let new_lo = span.lo() + BytePos::from_usize(start);
593 let new_hi = span.lo() + BytePos::from_usize(end);
594 Some(span.with_lo(new_lo).with_hi(new_hi))
598 impl server::SourceFile for Rustc<'_> {
599 fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool {
600 Lrc::ptr_eq(file1, file2)
602 fn path(&mut self, file: &Self::SourceFile) -> String {
604 FileName::Real(ref path) => path
606 .expect("non-UTF8 file path in `proc_macro::SourceFile::path`")
608 _ => file.name.to_string(),
611 fn is_real(&mut self, file: &Self::SourceFile) -> bool {
616 impl server::MultiSpan for Rustc<'_> {
617 fn new(&mut self) -> Self::MultiSpan {
620 fn push(&mut self, spans: &mut Self::MultiSpan, span: Self::Span) {
625 impl server::Diagnostic for Rustc<'_> {
626 fn new(&mut self, level: Level, msg: &str, spans: Self::MultiSpan) -> Self::Diagnostic {
627 let mut diag = Diagnostic::new(level.to_internal(), msg);
628 diag.set_span(MultiSpan::from_spans(spans));
633 diag: &mut Self::Diagnostic,
636 spans: Self::MultiSpan,
638 diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None);
640 fn emit(&mut self, diag: Self::Diagnostic) {
641 self.sess.span_diagnostic.emit_diagnostic(&diag);
645 impl server::Span for Rustc<'_> {
646 fn debug(&mut self, span: Self::Span) -> String {
647 format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0)
649 fn def_site(&mut self) -> Self::Span {
652 fn call_site(&mut self) -> Self::Span {
655 fn mixed_site(&mut self) -> Self::Span {
658 fn source_file(&mut self, span: Self::Span) -> Self::SourceFile {
659 self.sess.source_map().lookup_char_pos(span.lo()).file
661 fn parent(&mut self, span: Self::Span) -> Option<Self::Span> {
664 fn source(&mut self, span: Self::Span) -> Self::Span {
665 span.source_callsite()
667 fn start(&mut self, span: Self::Span) -> LineColumn {
668 let loc = self.sess.source_map().lookup_char_pos(span.lo());
669 LineColumn { line: loc.line, column: loc.col.to_usize() }
671 fn end(&mut self, span: Self::Span) -> LineColumn {
672 let loc = self.sess.source_map().lookup_char_pos(span.hi());
673 LineColumn { line: loc.line, column: loc.col.to_usize() }
675 fn join(&mut self, first: Self::Span, second: Self::Span) -> Option<Self::Span> {
676 let self_loc = self.sess.source_map().lookup_char_pos(first.lo());
677 let other_loc = self.sess.source_map().lookup_char_pos(second.lo());
679 if self_loc.file.name != other_loc.file.name {
683 Some(first.to(second))
685 fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span {
686 span.with_ctxt(at.ctxt())
688 fn source_text(&mut self, span: Self::Span) -> Option<String> {
689 self.sess.source_map().span_to_snippet(span).ok()