1 use crate::errors::{self, Diagnostic, DiagnosticBuilder};
5 use proc_macro::bridge::{server, TokenTree};
6 use proc_macro::{Delimiter, Level, LineColumn, Spacing};
8 use rustc_data_structures::sync::Lrc;
12 use syntax::ext::base::ExtCtxt;
13 use syntax::parse::lexer::comments;
14 use syntax::parse::{self, token, ParseSess};
15 use syntax::parse::parser::emit_unclosed_delims;
16 use syntax::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint};
17 use syntax_pos::hygiene::{SyntaxContext, Transparency};
18 use syntax_pos::symbol::{keywords, Symbol};
19 use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span};
21 trait FromInternal<T> {
22 fn from_internal(x: T) -> Self;
26 fn to_internal(self) -> T;
29 impl FromInternal<token::DelimToken> for Delimiter {
30 fn from_internal(delim: token::DelimToken) -> Delimiter {
32 token::Paren => Delimiter::Parenthesis,
33 token::Brace => Delimiter::Brace,
34 token::Bracket => Delimiter::Bracket,
35 token::NoDelim => Delimiter::None,
40 impl ToInternal<token::DelimToken> for Delimiter {
41 fn to_internal(self) -> token::DelimToken {
43 Delimiter::Parenthesis => token::Paren,
44 Delimiter::Brace => token::Brace,
45 Delimiter::Bracket => token::Bracket,
46 Delimiter::None => token::NoDelim,
51 impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
52 for TokenTree<Group, Punct, Ident, Literal>
54 fn from_internal(((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec<Self>))
56 use syntax::parse::token::*;
58 let joint = is_joint == Joint;
59 let (span, token) = match tree {
60 tokenstream::TokenTree::Delimited(span, delim, tts) => {
61 let delimiter = Delimiter::from_internal(delim);
62 return TokenTree::Group(Group {
68 tokenstream::TokenTree::Token(span, token) => (span, token),
72 ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)* }) => (
73 TokenTree::$ty(self::$ty {
74 $($field $(: $value)*,)*
78 ($ty:ident::$method:ident($($value:expr),*)) => (
79 TokenTree::$ty(self::$ty::$method($($value,)* span))
84 tt!(Punct::new($a, joint))
86 ($a:expr, $b:expr) => {{
87 stack.push(tt!(Punct::new($b, joint)));
88 tt!(Punct::new($a, true))
90 ($a:expr, $b:expr, $c:expr) => {{
91 stack.push(tt!(Punct::new($c, joint)));
92 stack.push(tt!(Punct::new($b, true)));
93 tt!(Punct::new($a, true))
101 EqEq => op!('=', '='),
105 AndAnd => op!('&', '&'),
106 OrOr => op!('|', '|'),
109 BinOp(Plus) => op!('+'),
110 BinOp(Minus) => op!('-'),
111 BinOp(Star) => op!('*'),
112 BinOp(Slash) => op!('/'),
113 BinOp(Percent) => op!('%'),
114 BinOp(Caret) => op!('^'),
115 BinOp(And) => op!('&'),
116 BinOp(Or) => op!('|'),
117 BinOp(Shl) => op!('<', '<'),
118 BinOp(Shr) => op!('>', '>'),
119 BinOpEq(Plus) => op!('+', '='),
120 BinOpEq(Minus) => op!('-', '='),
121 BinOpEq(Star) => op!('*', '='),
122 BinOpEq(Slash) => op!('/', '='),
123 BinOpEq(Percent) => op!('%', '='),
124 BinOpEq(Caret) => op!('^', '='),
125 BinOpEq(And) => op!('&', '='),
126 BinOpEq(Or) => op!('|', '='),
127 BinOpEq(Shl) => op!('<', '<', '='),
128 BinOpEq(Shr) => op!('>', '>', '='),
131 DotDot => op!('.', '.'),
132 DotDotDot => op!('.', '.', '.'),
133 DotDotEq => op!('.', '.', '='),
137 ModSep => op!(':', ':'),
138 RArrow => op!('-', '>'),
139 LArrow => op!('<', '-'),
140 FatArrow => op!('=', '>'),
143 Question => op!('?'),
144 SingleQuote => op!('\''),
146 Ident(ident, false) if ident.name == keywords::DollarCrate.name() =>
147 tt!(Ident::dollar_crate()),
148 Ident(ident, is_raw) => tt!(Ident::new(ident.name, is_raw)),
150 let ident = ident.without_first_quote();
151 stack.push(tt!(Ident::new(ident.name, false)));
152 tt!(Punct::new('\'', true))
154 Literal(lit, suffix) => tt!(Literal { lit, suffix }),
156 let style = comments::doc_comment_style(&c.as_str());
157 let stripped = comments::strip_doc_comment_decoration(&c.as_str());
158 let mut escaped = String::new();
159 for ch in stripped.chars() {
160 escaped.extend(ch.escape_debug());
163 Ident(ast::Ident::new(Symbol::intern("doc"), span), false),
165 Literal(Lit::Str_(Symbol::intern(&escaped)), None),
168 .map(|token| tokenstream::TokenTree::Token(span, token))
170 stack.push(TokenTree::Group(Group {
171 delimiter: Delimiter::Bracket,
173 span: DelimSpan::from_single(span),
175 if style == ast::AttrStyle::Inner {
176 stack.push(tt!(Punct::new('!', false)));
178 tt!(Punct::new('#', false))
182 let stream = token.interpolated_to_tokenstream(sess, span);
183 TokenTree::Group(Group {
184 delimiter: Delimiter::None,
186 span: DelimSpan::from_single(span),
190 OpenDelim(..) | CloseDelim(..) => unreachable!(),
191 Whitespace | Comment | Shebang(..) | Eof => unreachable!(),
196 impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> {
197 fn to_internal(self) -> TokenStream {
198 use syntax::parse::token::*;
200 let (ch, joint, span) = match self {
201 TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span),
202 TokenTree::Group(Group {
207 return tokenstream::TokenTree::Delimited(
209 delimiter.to_internal(),
214 TokenTree::Ident(self::Ident { sym, is_raw, span }) => {
215 let token = Ident(ast::Ident::new(sym, span), is_raw);
216 return tokenstream::TokenTree::Token(span, token).into();
218 TokenTree::Literal(self::Literal {
219 lit: Lit::Integer(ref a),
222 }) if a.as_str().starts_with("-") => {
223 let minus = BinOp(BinOpToken::Minus);
224 let integer = Symbol::intern(&a.as_str()[1..]);
225 let integer = Literal(Lit::Integer(integer), suffix);
226 let a = tokenstream::TokenTree::Token(span, minus);
227 let b = tokenstream::TokenTree::Token(span, integer);
228 return vec![a, b].into_iter().collect();
230 TokenTree::Literal(self::Literal {
231 lit: Lit::Float(ref a),
234 }) if a.as_str().starts_with("-") => {
235 let minus = BinOp(BinOpToken::Minus);
236 let float = Symbol::intern(&a.as_str()[1..]);
237 let float = Literal(Lit::Float(float), suffix);
238 let a = tokenstream::TokenTree::Token(span, minus);
239 let b = tokenstream::TokenTree::Token(span, float);
240 return vec![a, b].into_iter().collect();
242 TokenTree::Literal(self::Literal { lit, suffix, span }) => {
243 return tokenstream::TokenTree::Token(span, Literal(lit, suffix)).into()
247 let token = match ch {
257 '%' => BinOp(Percent),
273 let tree = tokenstream::TokenTree::Token(span, token);
274 TokenStream::new(vec![(tree, if joint { Joint } else { NonJoint })])
278 impl ToInternal<errors::Level> for Level {
279 fn to_internal(self) -> errors::Level {
281 Level::Error => errors::Level::Error,
282 Level::Warning => errors::Level::Warning,
283 Level::Note => errors::Level::Note,
284 Level::Help => errors::Level::Help,
285 _ => unreachable!("unknown proc_macro::Level variant: {:?}", self),
291 pub struct TokenStreamIter {
292 cursor: tokenstream::Cursor,
293 stack: Vec<TokenTree<Group, Punct, Ident, Literal>>,
298 delimiter: Delimiter,
303 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
306 // NB. not using `Spacing` here because it doesn't implement `Hash`.
312 fn new(ch: char, joint: bool, span: Span) -> Punct {
313 const LEGAL_CHARS: &[char] = &['=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^',
314 '&', '|', '@', '.', ',', ';', ':', '#', '$', '?', '\''];
315 if !LEGAL_CHARS.contains(&ch) {
316 panic!("unsupported character `{:?}`", ch)
318 Punct { ch, joint, span }
322 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
330 fn is_valid(string: &str) -> bool {
331 let mut chars = string.chars();
332 if let Some(start) = chars.next() {
333 (start == '_' || start.is_xid_start())
334 && chars.all(|cont| cont == '_' || cont.is_xid_continue())
339 fn new(sym: Symbol, is_raw: bool, span: Span) -> Ident {
340 let string = sym.as_str().get();
341 if !Self::is_valid(string) {
342 panic!("`{:?}` is not a valid identifier", string)
345 let normalized_sym = Symbol::intern(string);
346 if normalized_sym == keywords::Underscore.name() ||
347 ast::Ident::with_empty_ctxt(normalized_sym).is_path_segment_keyword() {
348 panic!("`{:?}` is not a valid raw identifier", string)
351 Ident { sym, is_raw, span }
353 fn dollar_crate(span: Span) -> Ident {
354 // `$crate` is accepted as an ident only if it comes from the compiler.
355 Ident { sym: keywords::DollarCrate.name(), is_raw: false, span }
359 // FIXME(eddyb) `Literal` should not expose internal `Debug` impls.
360 #[derive(Clone, Debug)]
363 suffix: Option<Symbol>,
367 pub(crate) struct Rustc<'a> {
374 pub fn new(cx: &'a ExtCtxt<'_>) -> Self {
375 // No way to determine def location for a proc macro right now, so use call location.
376 let location = cx.current_expansion.mark.expn_info().unwrap().call_site;
377 let to_span = |transparency| {
379 SyntaxContext::empty()
380 .apply_mark_with_transparency(cx.current_expansion.mark, transparency),
385 def_site: to_span(Transparency::Opaque),
386 call_site: to_span(Transparency::Transparent),
391 impl server::Types for Rustc<'_> {
392 type TokenStream = TokenStream;
393 type TokenStreamBuilder = tokenstream::TokenStreamBuilder;
394 type TokenStreamIter = TokenStreamIter;
398 type Literal = Literal;
399 type SourceFile = Lrc<SourceFile>;
400 type MultiSpan = Vec<Span>;
401 type Diagnostic = Diagnostic;
405 impl server::TokenStream for Rustc<'_> {
406 fn new(&mut self) -> Self::TokenStream {
409 fn is_empty(&mut self, stream: &Self::TokenStream) -> bool {
412 fn from_str(&mut self, src: &str) -> Self::TokenStream {
413 let (tokens, errors) = parse::parse_stream_from_source_str(
414 FileName::proc_macro_source_code(src.clone()),
417 Some(self.call_site),
419 emit_unclosed_delims(&errors, &self.sess.span_diagnostic);
422 fn to_string(&mut self, stream: &Self::TokenStream) -> String {
427 tree: TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>,
428 ) -> Self::TokenStream {
431 fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter {
433 cursor: stream.trees(),
439 impl server::TokenStreamBuilder for Rustc<'_> {
440 fn new(&mut self) -> Self::TokenStreamBuilder {
441 tokenstream::TokenStreamBuilder::new()
443 fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) {
444 builder.push(stream);
446 fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream {
451 impl server::TokenStreamIter for Rustc<'_> {
454 iter: &mut Self::TokenStreamIter,
455 ) -> Option<TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>> {
457 let tree = iter.stack.pop().or_else(|| {
458 let next = iter.cursor.next_with_joint()?;
459 Some(TokenTree::from_internal((next, self.sess, &mut iter.stack)))
461 // HACK: The condition "dummy span + group with empty delimiter" represents an AST
462 // fragment approximately converted into a token stream. This may happen, for
463 // example, with inputs to proc macro attributes, including derives. Such "groups"
464 // need to flattened during iteration over stream's token trees.
465 // Eventually this needs to be removed in favor of keeping original token trees
466 // and not doing the roundtrip through AST.
467 if let TokenTree::Group(ref group) = tree {
468 if group.delimiter == Delimiter::None && group.span.entire().is_dummy() {
469 iter.cursor.append(group.stream.clone());
478 impl server::Group for Rustc<'_> {
479 fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group {
483 span: DelimSpan::from_single(server::Span::call_site(self)),
486 fn delimiter(&mut self, group: &Self::Group) -> Delimiter {
489 fn stream(&mut self, group: &Self::Group) -> Self::TokenStream {
492 fn span(&mut self, group: &Self::Group) -> Self::Span {
495 fn span_open(&mut self, group: &Self::Group) -> Self::Span {
498 fn span_close(&mut self, group: &Self::Group) -> Self::Span {
501 fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) {
502 group.span = DelimSpan::from_single(span);
506 impl server::Punct for Rustc<'_> {
507 fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct {
508 Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self))
510 fn as_char(&mut self, punct: Self::Punct) -> char {
513 fn spacing(&mut self, punct: Self::Punct) -> Spacing {
520 fn span(&mut self, punct: Self::Punct) -> Self::Span {
523 fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct {
524 Punct { span, ..punct }
528 impl server::Ident for Rustc<'_> {
529 fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident {
530 Ident::new(Symbol::intern(string), is_raw, span)
532 fn span(&mut self, ident: Self::Ident) -> Self::Span {
535 fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident {
536 Ident { span, ..ident }
540 impl server::Literal for Rustc<'_> {
541 // FIXME(eddyb) `Literal` should not expose internal `Debug` impls.
542 fn debug(&mut self, literal: &Self::Literal) -> String {
543 format!("{:?}", literal)
545 fn integer(&mut self, n: &str) -> Self::Literal {
547 lit: token::Lit::Integer(Symbol::intern(n)),
549 span: server::Span::call_site(self),
552 fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal {
554 lit: token::Lit::Integer(Symbol::intern(n)),
555 suffix: Some(Symbol::intern(kind)),
556 span: server::Span::call_site(self),
559 fn float(&mut self, n: &str) -> Self::Literal {
561 lit: token::Lit::Float(Symbol::intern(n)),
563 span: server::Span::call_site(self),
566 fn f32(&mut self, n: &str) -> Self::Literal {
568 lit: token::Lit::Float(Symbol::intern(n)),
569 suffix: Some(Symbol::intern("f32")),
570 span: server::Span::call_site(self),
573 fn f64(&mut self, n: &str) -> Self::Literal {
575 lit: token::Lit::Float(Symbol::intern(n)),
576 suffix: Some(Symbol::intern("f64")),
577 span: server::Span::call_site(self),
580 fn string(&mut self, string: &str) -> Self::Literal {
581 let mut escaped = String::new();
582 for ch in string.chars() {
583 escaped.extend(ch.escape_debug());
586 lit: token::Lit::Str_(Symbol::intern(&escaped)),
588 span: server::Span::call_site(self),
591 fn character(&mut self, ch: char) -> Self::Literal {
592 let mut escaped = String::new();
593 escaped.extend(ch.escape_unicode());
595 lit: token::Lit::Char(Symbol::intern(&escaped)),
597 span: server::Span::call_site(self),
600 fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal {
604 .flat_map(ascii::escape_default)
605 .map(Into::<char>::into)
606 .collect::<String>();
608 lit: token::Lit::ByteStr(Symbol::intern(&string)),
610 span: server::Span::call_site(self),
613 fn span(&mut self, literal: &Self::Literal) -> Self::Span {
616 fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) {
621 literal: &Self::Literal,
624 ) -> Option<Self::Span> {
625 let span = literal.span;
626 let length = span.hi().to_usize() - span.lo().to_usize();
628 let start = match start {
629 Bound::Included(lo) => lo,
630 Bound::Excluded(lo) => lo + 1,
631 Bound::Unbounded => 0,
634 let end = match end {
635 Bound::Included(hi) => hi + 1,
636 Bound::Excluded(hi) => hi,
637 Bound::Unbounded => length,
640 // Bounds check the values, preventing addition overflow and OOB spans.
641 if start > u32::max_value() as usize
642 || end > u32::max_value() as usize
643 || (u32::max_value() - start as u32) < span.lo().to_u32()
644 || (u32::max_value() - end as u32) < span.lo().to_u32()
651 let new_lo = span.lo() + BytePos::from_usize(start);
652 let new_hi = span.lo() + BytePos::from_usize(end);
653 Some(span.with_lo(new_lo).with_hi(new_hi))
657 impl server::SourceFile for Rustc<'_> {
658 fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool {
659 Lrc::ptr_eq(file1, file2)
661 fn path(&mut self, file: &Self::SourceFile) -> String {
663 FileName::Real(ref path) => path
665 .expect("non-UTF8 file path in `proc_macro::SourceFile::path`")
667 _ => file.name.to_string(),
670 fn is_real(&mut self, file: &Self::SourceFile) -> bool {
675 impl server::MultiSpan for Rustc<'_> {
676 fn new(&mut self) -> Self::MultiSpan {
679 fn push(&mut self, spans: &mut Self::MultiSpan, span: Self::Span) {
684 impl server::Diagnostic for Rustc<'_> {
685 fn new(&mut self, level: Level, msg: &str, spans: Self::MultiSpan) -> Self::Diagnostic {
686 let mut diag = Diagnostic::new(level.to_internal(), msg);
687 diag.set_span(MultiSpan::from_spans(spans));
692 diag: &mut Self::Diagnostic,
695 spans: Self::MultiSpan,
697 diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None);
699 fn emit(&mut self, diag: Self::Diagnostic) {
700 DiagnosticBuilder::new_diagnostic(&self.sess.span_diagnostic, diag).emit()
704 impl server::Span for Rustc<'_> {
705 fn debug(&mut self, span: Self::Span) -> String {
706 format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0)
708 fn def_site(&mut self) -> Self::Span {
711 fn call_site(&mut self) -> Self::Span {
714 fn source_file(&mut self, span: Self::Span) -> Self::SourceFile {
715 self.sess.source_map().lookup_char_pos(span.lo()).file
717 fn parent(&mut self, span: Self::Span) -> Option<Self::Span> {
718 span.ctxt().outer().expn_info().map(|i| i.call_site)
720 fn source(&mut self, span: Self::Span) -> Self::Span {
721 span.source_callsite()
723 fn start(&mut self, span: Self::Span) -> LineColumn {
724 let loc = self.sess.source_map().lookup_char_pos(span.lo());
727 column: loc.col.to_usize(),
730 fn end(&mut self, span: Self::Span) -> LineColumn {
731 let loc = self.sess.source_map().lookup_char_pos(span.hi());
734 column: loc.col.to_usize(),
737 fn join(&mut self, first: Self::Span, second: Self::Span) -> Option<Self::Span> {
738 let self_loc = self.sess.source_map().lookup_char_pos(first.lo());
739 let other_loc = self.sess.source_map().lookup_char_pos(second.lo());
741 if self_loc.file.name != other_loc.file.name {
745 Some(first.to(second))
747 fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span {
748 span.with_ctxt(at.ctxt())