1 use crate::base::ExtCtxt;
5 use rustc_ast::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint};
6 use rustc_ast_pretty::pprust;
7 use rustc_data_structures::sync::Lrc;
8 use rustc_errors::Diagnostic;
9 use rustc_parse::lexer::nfc_normalize;
10 use rustc_parse::{nt_to_tokenstream, parse_stream_from_source_str};
11 use rustc_session::parse::ParseSess;
12 use rustc_span::symbol::{self, kw, sym, Symbol};
13 use rustc_span::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span};
15 use pm::bridge::{server, TokenTree};
16 use pm::{Delimiter, Level, LineColumn, Spacing};
18 use std::{ascii, panic};
20 trait FromInternal<T> {
21 fn from_internal(x: T) -> Self;
25 fn to_internal(self) -> T;
28 impl FromInternal<token::DelimToken> for Delimiter {
29 fn from_internal(delim: token::DelimToken) -> Delimiter {
31 token::Paren => Delimiter::Parenthesis,
32 token::Brace => Delimiter::Brace,
33 token::Bracket => Delimiter::Bracket,
34 token::NoDelim => Delimiter::None,
39 impl ToInternal<token::DelimToken> for Delimiter {
40 fn to_internal(self) -> token::DelimToken {
42 Delimiter::Parenthesis => token::Paren,
43 Delimiter::Brace => token::Brace,
44 Delimiter::Bracket => token::Bracket,
45 Delimiter::None => token::NoDelim,
50 impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
51 for TokenTree<Group, Punct, Ident, Literal>
54 ((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec<Self>),
56 use rustc_ast::token::*;
58 let joint = is_joint == Joint;
59 let Token { kind, span } = match tree {
60 tokenstream::TokenTree::Delimited(span, delim, tts) => {
61 let delimiter = Delimiter::from_internal(delim);
62 return TokenTree::Group(Group { delimiter, stream: tts, span, flatten: false });
64 tokenstream::TokenTree::Token(token) => token,
68 ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => (
69 TokenTree::$ty(self::$ty {
70 $($field $(: $value)*,)+
74 ($ty:ident::$method:ident($($value:expr),*)) => (
75 TokenTree::$ty(self::$ty::$method($($value,)* span))
80 tt!(Punct::new($a, joint))
82 ($a:expr, $b:expr) => {{
83 stack.push(tt!(Punct::new($b, joint)));
84 tt!(Punct::new($a, true))
86 ($a:expr, $b:expr, $c:expr) => {{
87 stack.push(tt!(Punct::new($c, joint)));
88 stack.push(tt!(Punct::new($b, true)));
89 tt!(Punct::new($a, true))
97 EqEq => op!('=', '='),
101 AndAnd => op!('&', '&'),
102 OrOr => op!('|', '|'),
105 BinOp(Plus) => op!('+'),
106 BinOp(Minus) => op!('-'),
107 BinOp(Star) => op!('*'),
108 BinOp(Slash) => op!('/'),
109 BinOp(Percent) => op!('%'),
110 BinOp(Caret) => op!('^'),
111 BinOp(And) => op!('&'),
112 BinOp(Or) => op!('|'),
113 BinOp(Shl) => op!('<', '<'),
114 BinOp(Shr) => op!('>', '>'),
115 BinOpEq(Plus) => op!('+', '='),
116 BinOpEq(Minus) => op!('-', '='),
117 BinOpEq(Star) => op!('*', '='),
118 BinOpEq(Slash) => op!('/', '='),
119 BinOpEq(Percent) => op!('%', '='),
120 BinOpEq(Caret) => op!('^', '='),
121 BinOpEq(And) => op!('&', '='),
122 BinOpEq(Or) => op!('|', '='),
123 BinOpEq(Shl) => op!('<', '<', '='),
124 BinOpEq(Shr) => op!('>', '>', '='),
127 DotDot => op!('.', '.'),
128 DotDotDot => op!('.', '.', '.'),
129 DotDotEq => op!('.', '.', '='),
133 ModSep => op!(':', ':'),
134 RArrow => op!('-', '>'),
135 LArrow => op!('<', '-'),
136 FatArrow => op!('=', '>'),
139 Question => op!('?'),
140 SingleQuote => op!('\''),
142 Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()),
143 Ident(name, is_raw) => tt!(Ident::new(sess, name, is_raw)),
145 let ident = symbol::Ident::new(name, span).without_first_quote();
146 stack.push(tt!(Ident::new(sess, ident.name, false)));
147 tt!(Punct::new('\'', true))
149 Literal(lit) => tt!(Literal { lit }),
150 DocComment(_, attr_style, data) => {
151 let mut escaped = String::new();
152 for ch in data.as_str().chars() {
153 escaped.extend(ch.escape_debug());
156 Ident(sym::doc, false),
158 TokenKind::lit(token::Str, Symbol::intern(&escaped), None),
161 .map(|kind| tokenstream::TokenTree::token(kind, span))
163 stack.push(TokenTree::Group(Group {
164 delimiter: Delimiter::Bracket,
166 span: DelimSpan::from_single(span),
169 if attr_style == ast::AttrStyle::Inner {
170 stack.push(tt!(Punct::new('!', false)));
172 tt!(Punct::new('#', false))
175 Interpolated(nt) => {
176 let stream = nt_to_tokenstream(&nt, sess, span);
177 TokenTree::Group(Group {
178 delimiter: Delimiter::None,
180 span: DelimSpan::from_single(span),
181 flatten: nt.pretty_printing_compatibility_hack(),
185 OpenDelim(..) | CloseDelim(..) => unreachable!(),
186 Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => unreachable!(),
191 impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> {
192 fn to_internal(self) -> TokenStream {
193 use rustc_ast::token::*;
195 let (ch, joint, span) = match self {
196 TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span),
197 TokenTree::Group(Group { delimiter, stream, span, .. }) => {
198 return tokenstream::TokenTree::Delimited(span, delimiter.to_internal(), stream)
201 TokenTree::Ident(self::Ident { sym, is_raw, span }) => {
202 return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into();
204 TokenTree::Literal(self::Literal {
205 lit: token::Lit { kind: token::Integer, symbol, suffix },
207 }) if symbol.as_str().starts_with('-') => {
208 let minus = BinOp(BinOpToken::Minus);
209 let symbol = Symbol::intern(&symbol.as_str()[1..]);
210 let integer = TokenKind::lit(token::Integer, symbol, suffix);
211 let a = tokenstream::TokenTree::token(minus, span);
212 let b = tokenstream::TokenTree::token(integer, span);
213 return vec![a, b].into_iter().collect();
215 TokenTree::Literal(self::Literal {
216 lit: token::Lit { kind: token::Float, symbol, suffix },
218 }) if symbol.as_str().starts_with('-') => {
219 let minus = BinOp(BinOpToken::Minus);
220 let symbol = Symbol::intern(&symbol.as_str()[1..]);
221 let float = TokenKind::lit(token::Float, symbol, suffix);
222 let a = tokenstream::TokenTree::token(minus, span);
223 let b = tokenstream::TokenTree::token(float, span);
224 return vec![a, b].into_iter().collect();
226 TokenTree::Literal(self::Literal { lit, span }) => {
227 return tokenstream::TokenTree::token(Literal(lit), span).into();
231 let kind = match ch {
241 '%' => BinOp(Percent),
257 let tree = tokenstream::TokenTree::token(kind, span);
258 TokenStream::new(vec![(tree, if joint { Joint } else { NonJoint })])
262 impl ToInternal<rustc_errors::Level> for Level {
263 fn to_internal(self) -> rustc_errors::Level {
265 Level::Error => rustc_errors::Level::Error,
266 Level::Warning => rustc_errors::Level::Warning,
267 Level::Note => rustc_errors::Level::Note,
268 Level::Help => rustc_errors::Level::Help,
269 _ => unreachable!("unknown proc_macro::Level variant: {:?}", self),
274 pub struct FreeFunctions;
277 pub struct TokenStreamIter {
278 cursor: tokenstream::Cursor,
279 stack: Vec<TokenTree<Group, Punct, Ident, Literal>>,
284 delimiter: Delimiter,
287 /// A hack used to pass AST fragments to attribute and derive macros
288 /// as a single nonterminal token instead of a token stream.
289 /// FIXME: It needs to be removed, but there are some compatibility issues (see #73345).
293 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
296 // NB. not using `Spacing` here because it doesn't implement `Hash`.
302 fn new(ch: char, joint: bool, span: Span) -> Punct {
303 const LEGAL_CHARS: &[char] = &[
304 '=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', '&', '|', '@', '.', ',', ';',
305 ':', '#', '$', '?', '\'',
307 if !LEGAL_CHARS.contains(&ch) {
308 panic!("unsupported character `{:?}`", ch)
310 Punct { ch, joint, span }
314 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
322 fn new(sess: &ParseSess, sym: Symbol, is_raw: bool, span: Span) -> Ident {
323 let sym = nfc_normalize(&sym.as_str());
324 let string = sym.as_str();
325 if !rustc_lexer::is_ident(&string) {
326 panic!("`{:?}` is not a valid identifier", string)
328 if is_raw && !sym.can_be_raw() {
329 panic!("`{}` cannot be a raw identifier", string);
331 sess.symbol_gallery.insert(sym, span);
332 Ident { sym, is_raw, span }
334 fn dollar_crate(span: Span) -> Ident {
335 // `$crate` is accepted as an ident only if it comes from the compiler.
336 Ident { sym: kw::DollarCrate, is_raw: false, span }
340 // FIXME(eddyb) `Literal` should not expose internal `Debug` impls.
341 #[derive(Clone, Debug)]
347 pub(crate) struct Rustc<'a> {
356 pub fn new(cx: &'a ExtCtxt<'_>) -> Self {
357 let expn_data = cx.current_expansion.id.expn_data();
359 sess: &cx.sess.parse_sess,
360 def_site: cx.with_def_site_ctxt(expn_data.def_site),
361 call_site: cx.with_call_site_ctxt(expn_data.call_site),
362 mixed_site: cx.with_mixed_site_ctxt(expn_data.call_site),
363 span_debug: cx.ecfg.span_debug,
367 fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Literal {
368 Literal { lit: token::Lit::new(kind, symbol, suffix), span: server::Span::call_site(self) }
372 impl server::Types for Rustc<'_> {
373 type FreeFunctions = FreeFunctions;
374 type TokenStream = TokenStream;
375 type TokenStreamBuilder = tokenstream::TokenStreamBuilder;
376 type TokenStreamIter = TokenStreamIter;
380 type Literal = Literal;
381 type SourceFile = Lrc<SourceFile>;
382 type MultiSpan = Vec<Span>;
383 type Diagnostic = Diagnostic;
387 impl server::FreeFunctions for Rustc<'_> {
388 fn track_env_var(&mut self, var: &str, value: Option<&str>) {
389 self.sess.env_depinfo.borrow_mut().insert((Symbol::intern(var), value.map(Symbol::intern)));
393 impl server::TokenStream for Rustc<'_> {
394 fn new(&mut self) -> Self::TokenStream {
395 TokenStream::default()
397 fn is_empty(&mut self, stream: &Self::TokenStream) -> bool {
400 fn from_str(&mut self, src: &str) -> Self::TokenStream {
401 parse_stream_from_source_str(
402 FileName::proc_macro_source_code(src),
405 Some(self.call_site),
408 fn to_string(&mut self, stream: &Self::TokenStream) -> String {
409 pprust::tts_to_string(stream)
413 tree: TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>,
414 ) -> Self::TokenStream {
417 fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter {
418 TokenStreamIter { cursor: stream.trees(), stack: vec![] }
422 impl server::TokenStreamBuilder for Rustc<'_> {
423 fn new(&mut self) -> Self::TokenStreamBuilder {
424 tokenstream::TokenStreamBuilder::new()
426 fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) {
427 builder.push(stream);
429 fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream {
434 impl server::TokenStreamIter for Rustc<'_> {
437 iter: &mut Self::TokenStreamIter,
438 ) -> Option<TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>> {
440 let tree = iter.stack.pop().or_else(|| {
441 let next = iter.cursor.next_with_joint()?;
442 Some(TokenTree::from_internal((next, self.sess, &mut iter.stack)))
444 // A hack used to pass AST fragments to attribute and derive macros
445 // as a single nonterminal token instead of a token stream.
446 // Such token needs to be "unwrapped" and not represented as a delimited group.
447 // FIXME: It needs to be removed, but there are some compatibility issues (see #73345).
448 if let TokenTree::Group(ref group) = tree {
450 iter.cursor.append(group.stream.clone());
459 impl server::Group for Rustc<'_> {
460 fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group {
464 span: DelimSpan::from_single(server::Span::call_site(self)),
468 fn delimiter(&mut self, group: &Self::Group) -> Delimiter {
471 fn stream(&mut self, group: &Self::Group) -> Self::TokenStream {
474 fn span(&mut self, group: &Self::Group) -> Self::Span {
477 fn span_open(&mut self, group: &Self::Group) -> Self::Span {
480 fn span_close(&mut self, group: &Self::Group) -> Self::Span {
483 fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) {
484 group.span = DelimSpan::from_single(span);
488 impl server::Punct for Rustc<'_> {
489 fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct {
490 Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self))
492 fn as_char(&mut self, punct: Self::Punct) -> char {
495 fn spacing(&mut self, punct: Self::Punct) -> Spacing {
496 if punct.joint { Spacing::Joint } else { Spacing::Alone }
498 fn span(&mut self, punct: Self::Punct) -> Self::Span {
501 fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct {
502 Punct { span, ..punct }
506 impl server::Ident for Rustc<'_> {
507 fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident {
508 Ident::new(self.sess, Symbol::intern(string), is_raw, span)
510 fn span(&mut self, ident: Self::Ident) -> Self::Span {
513 fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident {
514 Ident { span, ..ident }
518 impl server::Literal for Rustc<'_> {
519 fn debug_kind(&mut self, literal: &Self::Literal) -> String {
520 format!("{:?}", literal.lit.kind)
522 fn symbol(&mut self, literal: &Self::Literal) -> String {
523 literal.lit.symbol.to_string()
525 fn suffix(&mut self, literal: &Self::Literal) -> Option<String> {
526 literal.lit.suffix.as_ref().map(Symbol::to_string)
528 fn integer(&mut self, n: &str) -> Self::Literal {
529 self.lit(token::Integer, Symbol::intern(n), None)
531 fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal {
532 self.lit(token::Integer, Symbol::intern(n), Some(Symbol::intern(kind)))
534 fn float(&mut self, n: &str) -> Self::Literal {
535 self.lit(token::Float, Symbol::intern(n), None)
537 fn f32(&mut self, n: &str) -> Self::Literal {
538 self.lit(token::Float, Symbol::intern(n), Some(sym::f32))
540 fn f64(&mut self, n: &str) -> Self::Literal {
541 self.lit(token::Float, Symbol::intern(n), Some(sym::f64))
543 fn string(&mut self, string: &str) -> Self::Literal {
544 let mut escaped = String::new();
545 for ch in string.chars() {
546 escaped.extend(ch.escape_debug());
548 self.lit(token::Str, Symbol::intern(&escaped), None)
550 fn character(&mut self, ch: char) -> Self::Literal {
551 let mut escaped = String::new();
552 escaped.extend(ch.escape_unicode());
553 self.lit(token::Char, Symbol::intern(&escaped), None)
555 fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal {
559 .flat_map(ascii::escape_default)
560 .map(Into::<char>::into)
561 .collect::<String>();
562 self.lit(token::ByteStr, Symbol::intern(&string), None)
564 fn span(&mut self, literal: &Self::Literal) -> Self::Span {
567 fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) {
572 literal: &Self::Literal,
575 ) -> Option<Self::Span> {
576 let span = literal.span;
577 let length = span.hi().to_usize() - span.lo().to_usize();
579 let start = match start {
580 Bound::Included(lo) => lo,
581 Bound::Excluded(lo) => lo + 1,
582 Bound::Unbounded => 0,
585 let end = match end {
586 Bound::Included(hi) => hi + 1,
587 Bound::Excluded(hi) => hi,
588 Bound::Unbounded => length,
591 // Bounds check the values, preventing addition overflow and OOB spans.
592 if start > u32::MAX as usize
593 || end > u32::MAX as usize
594 || (u32::MAX - start as u32) < span.lo().to_u32()
595 || (u32::MAX - end as u32) < span.lo().to_u32()
602 let new_lo = span.lo() + BytePos::from_usize(start);
603 let new_hi = span.lo() + BytePos::from_usize(end);
604 Some(span.with_lo(new_lo).with_hi(new_hi))
608 impl server::SourceFile for Rustc<'_> {
609 fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool {
610 Lrc::ptr_eq(file1, file2)
612 fn path(&mut self, file: &Self::SourceFile) -> String {
614 FileName::Real(ref name) => name
617 .expect("non-UTF8 file path in `proc_macro::SourceFile::path`")
619 _ => file.name.to_string(),
622 fn is_real(&mut self, file: &Self::SourceFile) -> bool {
627 impl server::MultiSpan for Rustc<'_> {
628 fn new(&mut self) -> Self::MultiSpan {
631 fn push(&mut self, spans: &mut Self::MultiSpan, span: Self::Span) {
636 impl server::Diagnostic for Rustc<'_> {
637 fn new(&mut self, level: Level, msg: &str, spans: Self::MultiSpan) -> Self::Diagnostic {
638 let mut diag = Diagnostic::new(level.to_internal(), msg);
639 diag.set_span(MultiSpan::from_spans(spans));
644 diag: &mut Self::Diagnostic,
647 spans: Self::MultiSpan,
649 diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None);
651 fn emit(&mut self, diag: Self::Diagnostic) {
652 self.sess.span_diagnostic.emit_diagnostic(&diag);
656 impl server::Span for Rustc<'_> {
657 fn debug(&mut self, span: Self::Span) -> String {
659 format!("{:?}", span)
661 format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0)
664 fn def_site(&mut self) -> Self::Span {
667 fn call_site(&mut self) -> Self::Span {
670 fn mixed_site(&mut self) -> Self::Span {
673 fn source_file(&mut self, span: Self::Span) -> Self::SourceFile {
674 self.sess.source_map().lookup_char_pos(span.lo()).file
676 fn parent(&mut self, span: Self::Span) -> Option<Self::Span> {
679 fn source(&mut self, span: Self::Span) -> Self::Span {
680 span.source_callsite()
682 fn start(&mut self, span: Self::Span) -> LineColumn {
683 let loc = self.sess.source_map().lookup_char_pos(span.lo());
684 LineColumn { line: loc.line, column: loc.col.to_usize() }
686 fn end(&mut self, span: Self::Span) -> LineColumn {
687 let loc = self.sess.source_map().lookup_char_pos(span.hi());
688 LineColumn { line: loc.line, column: loc.col.to_usize() }
690 fn join(&mut self, first: Self::Span, second: Self::Span) -> Option<Self::Span> {
691 let self_loc = self.sess.source_map().lookup_char_pos(first.lo());
692 let other_loc = self.sess.source_map().lookup_char_pos(second.lo());
694 if self_loc.file.name != other_loc.file.name {
698 Some(first.to(second))
700 fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span {
701 span.with_ctxt(at.ctxt())
703 fn source_text(&mut self, span: Self::Span) -> Option<String> {
704 self.sess.source_map().span_to_snippet(span).ok()