1 use errors::{Diagnostic, DiagnosticBuilder};
5 use proc_macro::bridge::{server, TokenTree};
6 use proc_macro::{Delimiter, Level, LineColumn, Spacing};
8 use rustc_data_structures::sync::Lrc;
12 use syntax::ext::base::ExtCtxt;
13 use syntax::parse::lexer::comments;
14 use syntax::parse::{self, token, ParseSess};
15 use syntax::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint};
16 use syntax_pos::hygiene::{SyntaxContext, Transparency};
17 use syntax_pos::symbol::{kw, sym, Symbol};
18 use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span};
20 trait FromInternal<T> {
21 fn from_internal(x: T) -> Self;
25 fn to_internal(self) -> T;
28 impl FromInternal<token::DelimToken> for Delimiter {
29 fn from_internal(delim: token::DelimToken) -> Delimiter {
31 token::Paren => Delimiter::Parenthesis,
32 token::Brace => Delimiter::Brace,
33 token::Bracket => Delimiter::Bracket,
34 token::NoDelim => Delimiter::None,
39 impl ToInternal<token::DelimToken> for Delimiter {
40 fn to_internal(self) -> token::DelimToken {
42 Delimiter::Parenthesis => token::Paren,
43 Delimiter::Brace => token::Brace,
44 Delimiter::Bracket => token::Bracket,
45 Delimiter::None => token::NoDelim,
50 impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
51 for TokenTree<Group, Punct, Ident, Literal>
53 fn from_internal(((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec<Self>))
55 use syntax::parse::token::*;
57 let joint = is_joint == Joint;
58 let Token { kind, span } = match tree {
59 tokenstream::TokenTree::Delimited(span, delim, tts) => {
60 let delimiter = Delimiter::from_internal(delim);
61 return TokenTree::Group(Group {
67 tokenstream::TokenTree::Token(token) => token,
71 ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => (
72 TokenTree::$ty(self::$ty {
73 $($field $(: $value)*,)*
77 ($ty:ident::$method:ident($($value:expr),*)) => (
78 TokenTree::$ty(self::$ty::$method($($value,)* span))
83 tt!(Punct::new($a, joint))
85 ($a:expr, $b:expr) => {{
86 stack.push(tt!(Punct::new($b, joint)));
87 tt!(Punct::new($a, true))
89 ($a:expr, $b:expr, $c:expr) => {{
90 stack.push(tt!(Punct::new($c, joint)));
91 stack.push(tt!(Punct::new($b, true)));
92 tt!(Punct::new($a, true))
100 EqEq => op!('=', '='),
104 AndAnd => op!('&', '&'),
105 OrOr => op!('|', '|'),
108 BinOp(Plus) => op!('+'),
109 BinOp(Minus) => op!('-'),
110 BinOp(Star) => op!('*'),
111 BinOp(Slash) => op!('/'),
112 BinOp(Percent) => op!('%'),
113 BinOp(Caret) => op!('^'),
114 BinOp(And) => op!('&'),
115 BinOp(Or) => op!('|'),
116 BinOp(Shl) => op!('<', '<'),
117 BinOp(Shr) => op!('>', '>'),
118 BinOpEq(Plus) => op!('+', '='),
119 BinOpEq(Minus) => op!('-', '='),
120 BinOpEq(Star) => op!('*', '='),
121 BinOpEq(Slash) => op!('/', '='),
122 BinOpEq(Percent) => op!('%', '='),
123 BinOpEq(Caret) => op!('^', '='),
124 BinOpEq(And) => op!('&', '='),
125 BinOpEq(Or) => op!('|', '='),
126 BinOpEq(Shl) => op!('<', '<', '='),
127 BinOpEq(Shr) => op!('>', '>', '='),
130 DotDot => op!('.', '.'),
131 DotDotDot => op!('.', '.', '.'),
132 DotDotEq => op!('.', '.', '='),
136 ModSep => op!(':', ':'),
137 RArrow => op!('-', '>'),
138 LArrow => op!('<', '-'),
139 FatArrow => op!('=', '>'),
142 Question => op!('?'),
143 SingleQuote => op!('\''),
145 Ident(ident, false) if ident.name == kw::DollarCrate =>
146 tt!(Ident::dollar_crate()),
147 Ident(ident, is_raw) => tt!(Ident::new(ident.name, is_raw)),
149 let ident = ident.without_first_quote();
150 stack.push(tt!(Ident::new(ident.name, false)));
151 tt!(Punct::new('\'', true))
153 Literal(lit) => tt!(Literal { lit }),
155 let style = comments::doc_comment_style(&c.as_str());
156 let stripped = comments::strip_doc_comment_decoration(&c.as_str());
157 let mut escaped = String::new();
158 for ch in stripped.chars() {
159 escaped.extend(ch.escape_debug());
162 Ident(ast::Ident::new(sym::doc, span), false),
164 TokenKind::lit(token::Str, Symbol::intern(&escaped), None),
167 .map(|kind| tokenstream::TokenTree::token(span, kind))
169 stack.push(TokenTree::Group(Group {
170 delimiter: Delimiter::Bracket,
172 span: DelimSpan::from_single(span),
174 if style == ast::AttrStyle::Inner {
175 stack.push(tt!(Punct::new('!', false)));
177 tt!(Punct::new('#', false))
180 Interpolated(nt) => {
181 let stream = nt.to_tokenstream(sess, span);
182 TokenTree::Group(Group {
183 delimiter: Delimiter::None,
185 span: DelimSpan::from_single(span),
189 OpenDelim(..) | CloseDelim(..) => unreachable!(),
190 Whitespace | Comment | Shebang(..) | Eof => unreachable!(),
195 impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> {
196 fn to_internal(self) -> TokenStream {
197 use syntax::parse::token::*;
199 let (ch, joint, span) = match self {
200 TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span),
201 TokenTree::Group(Group {
206 return tokenstream::TokenTree::Delimited(
208 delimiter.to_internal(),
213 TokenTree::Ident(self::Ident { sym, is_raw, span }) => {
214 let token = Ident(ast::Ident::new(sym, span), is_raw);
215 return tokenstream::TokenTree::token(span, token).into();
217 TokenTree::Literal(self::Literal {
218 lit: token::Lit { kind: token::Integer, symbol, suffix },
220 }) if symbol.as_str().starts_with("-") => {
221 let minus = BinOp(BinOpToken::Minus);
222 let symbol = Symbol::intern(&symbol.as_str()[1..]);
223 let integer = TokenKind::lit(token::Integer, symbol, suffix);
224 let a = tokenstream::TokenTree::token(span, minus);
225 let b = tokenstream::TokenTree::token(span, integer);
226 return vec![a, b].into_iter().collect();
228 TokenTree::Literal(self::Literal {
229 lit: token::Lit { kind: token::Float, symbol, suffix },
231 }) if symbol.as_str().starts_with("-") => {
232 let minus = BinOp(BinOpToken::Minus);
233 let symbol = Symbol::intern(&symbol.as_str()[1..]);
234 let float = TokenKind::lit(token::Float, symbol, suffix);
235 let a = tokenstream::TokenTree::token(span, minus);
236 let b = tokenstream::TokenTree::token(span, float);
237 return vec![a, b].into_iter().collect();
239 TokenTree::Literal(self::Literal { lit, span }) => {
240 return tokenstream::TokenTree::token(span, Literal(lit)).into()
244 let kind = match ch {
254 '%' => BinOp(Percent),
270 let tree = tokenstream::TokenTree::token(span, kind);
271 TokenStream::new(vec![(tree, if joint { Joint } else { NonJoint })])
275 impl ToInternal<errors::Level> for Level {
276 fn to_internal(self) -> errors::Level {
278 Level::Error => errors::Level::Error,
279 Level::Warning => errors::Level::Warning,
280 Level::Note => errors::Level::Note,
281 Level::Help => errors::Level::Help,
282 _ => unreachable!("unknown proc_macro::Level variant: {:?}", self),
288 pub struct TokenStreamIter {
289 cursor: tokenstream::Cursor,
290 stack: Vec<TokenTree<Group, Punct, Ident, Literal>>,
295 delimiter: Delimiter,
300 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
303 // NB. not using `Spacing` here because it doesn't implement `Hash`.
309 fn new(ch: char, joint: bool, span: Span) -> Punct {
310 const LEGAL_CHARS: &[char] = &['=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^',
311 '&', '|', '@', '.', ',', ';', ':', '#', '$', '?', '\''];
312 if !LEGAL_CHARS.contains(&ch) {
313 panic!("unsupported character `{:?}`", ch)
315 Punct { ch, joint, span }
319 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
327 fn is_valid(string: &str) -> bool {
328 let mut chars = string.chars();
329 if let Some(start) = chars.next() {
330 (start == '_' || start.is_xid_start())
331 && chars.all(|cont| cont == '_' || cont.is_xid_continue())
336 fn new(sym: Symbol, is_raw: bool, span: Span) -> Ident {
337 let string = sym.as_str();
338 if !Self::is_valid(&string) {
339 panic!("`{:?}` is not a valid identifier", string)
341 if is_raw && !ast::Ident::from_interned_str(sym.as_interned_str()).can_be_raw() {
342 panic!("`{}` cannot be a raw identifier", string);
344 Ident { sym, is_raw, span }
346 fn dollar_crate(span: Span) -> Ident {
347 // `$crate` is accepted as an ident only if it comes from the compiler.
348 Ident { sym: kw::DollarCrate, is_raw: false, span }
352 // FIXME(eddyb) `Literal` should not expose internal `Debug` impls.
353 #[derive(Clone, Debug)]
359 pub(crate) struct Rustc<'a> {
366 pub fn new(cx: &'a ExtCtxt<'_>) -> Self {
367 // No way to determine def location for a proc macro right now, so use call location.
368 let location = cx.current_expansion.mark.expn_info().unwrap().call_site;
369 let to_span = |transparency| {
371 SyntaxContext::empty()
372 .apply_mark_with_transparency(cx.current_expansion.mark, transparency),
377 def_site: to_span(Transparency::Opaque),
378 call_site: to_span(Transparency::Transparent),
382 fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Literal {
384 lit: token::Lit::new(kind, symbol, suffix),
385 span: server::Span::call_site(self),
390 impl server::Types for Rustc<'_> {
391 type TokenStream = TokenStream;
392 type TokenStreamBuilder = tokenstream::TokenStreamBuilder;
393 type TokenStreamIter = TokenStreamIter;
397 type Literal = Literal;
398 type SourceFile = Lrc<SourceFile>;
399 type MultiSpan = Vec<Span>;
400 type Diagnostic = Diagnostic;
404 impl server::TokenStream for Rustc<'_> {
405 fn new(&mut self) -> Self::TokenStream {
408 fn is_empty(&mut self, stream: &Self::TokenStream) -> bool {
411 fn from_str(&mut self, src: &str) -> Self::TokenStream {
412 parse::parse_stream_from_source_str(
413 FileName::proc_macro_source_code(src.clone()),
416 Some(self.call_site),
419 fn to_string(&mut self, stream: &Self::TokenStream) -> String {
424 tree: TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>,
425 ) -> Self::TokenStream {
428 fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter {
430 cursor: stream.trees(),
436 impl server::TokenStreamBuilder for Rustc<'_> {
437 fn new(&mut self) -> Self::TokenStreamBuilder {
438 tokenstream::TokenStreamBuilder::new()
440 fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) {
441 builder.push(stream);
443 fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream {
448 impl server::TokenStreamIter for Rustc<'_> {
451 iter: &mut Self::TokenStreamIter,
452 ) -> Option<TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>> {
454 let tree = iter.stack.pop().or_else(|| {
455 let next = iter.cursor.next_with_joint()?;
456 Some(TokenTree::from_internal((next, self.sess, &mut iter.stack)))
458 // HACK: The condition "dummy span + group with empty delimiter" represents an AST
459 // fragment approximately converted into a token stream. This may happen, for
460 // example, with inputs to proc macro attributes, including derives. Such "groups"
461 // need to flattened during iteration over stream's token trees.
462 // Eventually this needs to be removed in favor of keeping original token trees
463 // and not doing the roundtrip through AST.
464 if let TokenTree::Group(ref group) = tree {
465 if group.delimiter == Delimiter::None && group.span.entire().is_dummy() {
466 iter.cursor.append(group.stream.clone());
475 impl server::Group for Rustc<'_> {
476 fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group {
480 span: DelimSpan::from_single(server::Span::call_site(self)),
483 fn delimiter(&mut self, group: &Self::Group) -> Delimiter {
486 fn stream(&mut self, group: &Self::Group) -> Self::TokenStream {
489 fn span(&mut self, group: &Self::Group) -> Self::Span {
492 fn span_open(&mut self, group: &Self::Group) -> Self::Span {
495 fn span_close(&mut self, group: &Self::Group) -> Self::Span {
498 fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) {
499 group.span = DelimSpan::from_single(span);
503 impl server::Punct for Rustc<'_> {
504 fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct {
505 Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self))
507 fn as_char(&mut self, punct: Self::Punct) -> char {
510 fn spacing(&mut self, punct: Self::Punct) -> Spacing {
517 fn span(&mut self, punct: Self::Punct) -> Self::Span {
520 fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct {
521 Punct { span, ..punct }
525 impl server::Ident for Rustc<'_> {
526 fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident {
527 Ident::new(Symbol::intern(string), is_raw, span)
529 fn span(&mut self, ident: Self::Ident) -> Self::Span {
532 fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident {
533 Ident { span, ..ident }
537 impl server::Literal for Rustc<'_> {
538 // FIXME(eddyb) `Literal` should not expose internal `Debug` impls.
539 fn debug(&mut self, literal: &Self::Literal) -> String {
540 format!("{:?}", literal)
542 fn integer(&mut self, n: &str) -> Self::Literal {
543 self.lit(token::Integer, Symbol::intern(n), None)
545 fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal {
546 self.lit(token::Integer, Symbol::intern(n), Some(Symbol::intern(kind)))
548 fn float(&mut self, n: &str) -> Self::Literal {
549 self.lit(token::Float, Symbol::intern(n), None)
551 fn f32(&mut self, n: &str) -> Self::Literal {
552 self.lit(token::Float, Symbol::intern(n), Some(Symbol::intern("f32")))
554 fn f64(&mut self, n: &str) -> Self::Literal {
555 self.lit(token::Float, Symbol::intern(n), Some(Symbol::intern("f64")))
557 fn string(&mut self, string: &str) -> Self::Literal {
558 let mut escaped = String::new();
559 for ch in string.chars() {
560 escaped.extend(ch.escape_debug());
562 self.lit(token::Str, Symbol::intern(&escaped), None)
564 fn character(&mut self, ch: char) -> Self::Literal {
565 let mut escaped = String::new();
566 escaped.extend(ch.escape_unicode());
567 self.lit(token::Char, Symbol::intern(&escaped), None)
569 fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal {
573 .flat_map(ascii::escape_default)
574 .map(Into::<char>::into)
575 .collect::<String>();
576 self.lit(token::ByteStr, Symbol::intern(&string), None)
578 fn span(&mut self, literal: &Self::Literal) -> Self::Span {
581 fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) {
586 literal: &Self::Literal,
589 ) -> Option<Self::Span> {
590 let span = literal.span;
591 let length = span.hi().to_usize() - span.lo().to_usize();
593 let start = match start {
594 Bound::Included(lo) => lo,
595 Bound::Excluded(lo) => lo + 1,
596 Bound::Unbounded => 0,
599 let end = match end {
600 Bound::Included(hi) => hi + 1,
601 Bound::Excluded(hi) => hi,
602 Bound::Unbounded => length,
605 // Bounds check the values, preventing addition overflow and OOB spans.
606 if start > u32::max_value() as usize
607 || end > u32::max_value() as usize
608 || (u32::max_value() - start as u32) < span.lo().to_u32()
609 || (u32::max_value() - end as u32) < span.lo().to_u32()
616 let new_lo = span.lo() + BytePos::from_usize(start);
617 let new_hi = span.lo() + BytePos::from_usize(end);
618 Some(span.with_lo(new_lo).with_hi(new_hi))
622 impl server::SourceFile for Rustc<'_> {
623 fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool {
624 Lrc::ptr_eq(file1, file2)
626 fn path(&mut self, file: &Self::SourceFile) -> String {
628 FileName::Real(ref path) => path
630 .expect("non-UTF8 file path in `proc_macro::SourceFile::path`")
632 _ => file.name.to_string(),
635 fn is_real(&mut self, file: &Self::SourceFile) -> bool {
640 impl server::MultiSpan for Rustc<'_> {
641 fn new(&mut self) -> Self::MultiSpan {
644 fn push(&mut self, spans: &mut Self::MultiSpan, span: Self::Span) {
649 impl server::Diagnostic for Rustc<'_> {
650 fn new(&mut self, level: Level, msg: &str, spans: Self::MultiSpan) -> Self::Diagnostic {
651 let mut diag = Diagnostic::new(level.to_internal(), msg);
652 diag.set_span(MultiSpan::from_spans(spans));
657 diag: &mut Self::Diagnostic,
660 spans: Self::MultiSpan,
662 diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None);
664 fn emit(&mut self, diag: Self::Diagnostic) {
665 DiagnosticBuilder::new_diagnostic(&self.sess.span_diagnostic, diag).emit()
669 impl server::Span for Rustc<'_> {
670 fn debug(&mut self, span: Self::Span) -> String {
671 format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0)
673 fn def_site(&mut self) -> Self::Span {
676 fn call_site(&mut self) -> Self::Span {
679 fn source_file(&mut self, span: Self::Span) -> Self::SourceFile {
680 self.sess.source_map().lookup_char_pos(span.lo()).file
682 fn parent(&mut self, span: Self::Span) -> Option<Self::Span> {
683 span.ctxt().outer_expn_info().map(|i| i.call_site)
685 fn source(&mut self, span: Self::Span) -> Self::Span {
686 span.source_callsite()
688 fn start(&mut self, span: Self::Span) -> LineColumn {
689 let loc = self.sess.source_map().lookup_char_pos(span.lo());
692 column: loc.col.to_usize(),
695 fn end(&mut self, span: Self::Span) -> LineColumn {
696 let loc = self.sess.source_map().lookup_char_pos(span.hi());
699 column: loc.col.to_usize(),
702 fn join(&mut self, first: Self::Span, second: Self::Span) -> Option<Self::Span> {
703 let self_loc = self.sess.source_map().lookup_char_pos(first.lo());
704 let other_loc = self.sess.source_map().lookup_char_pos(second.lo());
706 if self_loc.file.name != other_loc.file.name {
710 Some(first.to(second))
712 fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span {
713 span.with_ctxt(at.ctxt())
715 fn source_text(&mut self, span: Self::Span) -> Option<String> {
716 self.sess.source_map().span_to_snippet(span).ok()