1 use crate::base::ExtCtxt;
5 use rustc_ast::tokenstream::{self, Spacing::*, TokenStream};
6 use rustc_ast_pretty::pprust;
7 use rustc_data_structures::fx::FxHashMap;
8 use rustc_data_structures::sync::Lrc;
9 use rustc_errors::{Diagnostic, MultiSpan, PResult};
10 use rustc_parse::lexer::nfc_normalize;
11 use rustc_parse::parse_stream_from_source_str;
12 use rustc_session::parse::ParseSess;
13 use rustc_span::def_id::CrateNum;
14 use rustc_span::symbol::{self, sym, Symbol};
15 use rustc_span::{BytePos, FileName, Pos, SourceFile, Span};
18 server, DelimSpan, ExpnGlobals, Group, Ident, LitKind, Literal, Punct, TokenTree,
20 use pm::{Delimiter, Level, LineColumn};
23 trait FromInternal<T> {
24 fn from_internal(x: T) -> Self;
28 fn to_internal(self) -> T;
31 impl FromInternal<token::Delimiter> for Delimiter {
32 fn from_internal(delim: token::Delimiter) -> Delimiter {
34 token::Delimiter::Parenthesis => Delimiter::Parenthesis,
35 token::Delimiter::Brace => Delimiter::Brace,
36 token::Delimiter::Bracket => Delimiter::Bracket,
37 token::Delimiter::Invisible => Delimiter::None,
42 impl ToInternal<token::Delimiter> for Delimiter {
43 fn to_internal(self) -> token::Delimiter {
45 Delimiter::Parenthesis => token::Delimiter::Parenthesis,
46 Delimiter::Brace => token::Delimiter::Brace,
47 Delimiter::Bracket => token::Delimiter::Bracket,
48 Delimiter::None => token::Delimiter::Invisible,
53 impl FromInternal<token::LitKind> for LitKind {
54 fn from_internal(kind: token::LitKind) -> Self {
56 token::Byte => LitKind::Byte,
57 token::Char => LitKind::Char,
58 token::Integer => LitKind::Integer,
59 token::Float => LitKind::Float,
60 token::Str => LitKind::Str,
61 token::StrRaw(n) => LitKind::StrRaw(n),
62 token::ByteStr => LitKind::ByteStr,
63 token::ByteStrRaw(n) => LitKind::ByteStrRaw(n),
64 token::Err => LitKind::Err,
65 token::Bool => unreachable!(),
70 impl ToInternal<token::LitKind> for LitKind {
71 fn to_internal(self) -> token::LitKind {
73 LitKind::Byte => token::Byte,
74 LitKind::Char => token::Char,
75 LitKind::Integer => token::Integer,
76 LitKind::Float => token::Float,
77 LitKind::Str => token::Str,
78 LitKind::StrRaw(n) => token::StrRaw(n),
79 LitKind::ByteStr => token::ByteStr,
80 LitKind::ByteStrRaw(n) => token::ByteStrRaw(n),
81 LitKind::Err => token::Err,
86 impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStream, Span, Symbol>> {
87 fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self {
88 use rustc_ast::token::*;
90 // Estimate the capacity as `stream.len()` rounded up to the next power
91 // of two to limit the number of required reallocations.
92 let mut trees = Vec::with_capacity(stream.len().next_power_of_two());
93 let mut cursor = stream.into_trees();
95 while let Some((tree, spacing)) = cursor.next_with_spacing() {
96 let joint = spacing == Joint;
97 let Token { kind, span } = match tree {
98 tokenstream::TokenTree::Delimited(span, delim, tts) => {
99 let delimiter = pm::Delimiter::from_internal(delim);
100 trees.push(TokenTree::Group(Group {
106 entire: span.entire(),
111 tokenstream::TokenTree::Token(token) => token,
114 let mut op = |s: &str| {
115 assert!(s.is_ascii());
116 trees.extend(s.as_bytes().iter().enumerate().map(|(idx, &ch)| {
117 TokenTree::Punct(Punct { ch, joint: joint || idx != s.len() - 1, span })
133 BinOp(Plus) => op("+"),
134 BinOp(Minus) => op("-"),
135 BinOp(Star) => op("*"),
136 BinOp(Slash) => op("/"),
137 BinOp(Percent) => op("%"),
138 BinOp(Caret) => op("^"),
139 BinOp(And) => op("&"),
140 BinOp(Or) => op("|"),
141 BinOp(Shl) => op("<<"),
142 BinOp(Shr) => op(">>"),
143 BinOpEq(Plus) => op("+="),
144 BinOpEq(Minus) => op("-="),
145 BinOpEq(Star) => op("*="),
146 BinOpEq(Slash) => op("/="),
147 BinOpEq(Percent) => op("%="),
148 BinOpEq(Caret) => op("^="),
149 BinOpEq(And) => op("&="),
150 BinOpEq(Or) => op("|="),
151 BinOpEq(Shl) => op("<<="),
152 BinOpEq(Shr) => op(">>="),
156 DotDotDot => op("..."),
157 DotDotEq => op("..="),
164 FatArrow => op("=>"),
168 SingleQuote => op("'"),
170 Ident(sym, is_raw) => trees.push(TokenTree::Ident(Ident { sym, is_raw, span })),
172 let ident = symbol::Ident::new(name, span).without_first_quote();
174 TokenTree::Punct(Punct { ch: b'\'', joint: true, span }),
175 TokenTree::Ident(Ident { sym: ident.name, is_raw: false, span }),
178 Literal(token::Lit { kind, symbol, suffix }) => {
179 trees.push(TokenTree::Literal(self::Literal {
180 kind: FromInternal::from_internal(kind),
186 DocComment(_, attr_style, data) => {
187 let mut escaped = String::new();
188 for ch in data.as_str().chars() {
189 escaped.extend(ch.escape_debug());
192 Ident(sym::doc, false),
194 TokenKind::lit(token::Str, Symbol::intern(&escaped), None),
197 .map(|kind| tokenstream::TokenTree::token(kind, span))
199 trees.push(TokenTree::Punct(Punct { ch: b'#', joint: false, span }));
200 if attr_style == ast::AttrStyle::Inner {
201 trees.push(TokenTree::Punct(Punct { ch: b'!', joint: false, span }));
203 trees.push(TokenTree::Group(Group {
204 delimiter: pm::Delimiter::Bracket,
205 stream: Some(stream),
206 span: DelimSpan::from_single(span),
210 Interpolated(nt) if let NtIdent(ident, is_raw) = *nt => {
211 trees.push(TokenTree::Ident(Ident { sym: ident.name, is_raw, span: ident.span }))
214 Interpolated(nt) => {
215 let stream = TokenStream::from_nonterminal_ast(&nt);
216 // A hack used to pass AST fragments to attribute and derive
217 // macros as a single nonterminal token instead of a token
218 // stream. Such token needs to be "unwrapped" and not
219 // represented as a delimited group.
220 // FIXME: It needs to be removed, but there are some
221 // compatibility issues (see #73345).
222 if crate::base::nt_pretty_printing_compatibility_hack(&nt, rustc.sess()) {
223 trees.extend(Self::from_internal((stream, rustc)));
225 trees.push(TokenTree::Group(Group {
226 delimiter: pm::Delimiter::None,
227 stream: Some(stream),
228 span: DelimSpan::from_single(span),
233 OpenDelim(..) | CloseDelim(..) => unreachable!(),
234 Eof => unreachable!(),
241 impl ToInternal<TokenStream> for (TokenTree<TokenStream, Span, Symbol>, &mut Rustc<'_, '_>) {
242 fn to_internal(self) -> TokenStream {
243 use rustc_ast::token::*;
245 let (tree, rustc) = self;
246 let (ch, joint, span) = match tree {
247 TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span),
248 TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => {
249 return tokenstream::TokenTree::Delimited(
250 tokenstream::DelimSpan { open, close },
251 delimiter.to_internal(),
252 stream.unwrap_or_default(),
256 TokenTree::Ident(self::Ident { sym, is_raw, span }) => {
257 rustc.sess().symbol_gallery.insert(sym, span);
258 return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into();
260 TokenTree::Literal(self::Literal {
261 kind: self::LitKind::Integer,
265 }) if symbol.as_str().starts_with('-') => {
266 let minus = BinOp(BinOpToken::Minus);
267 let symbol = Symbol::intern(&symbol.as_str()[1..]);
268 let integer = TokenKind::lit(token::Integer, symbol, suffix);
269 let a = tokenstream::TokenTree::token(minus, span);
270 let b = tokenstream::TokenTree::token(integer, span);
271 return [a, b].into_iter().collect();
273 TokenTree::Literal(self::Literal {
274 kind: self::LitKind::Float,
278 }) if symbol.as_str().starts_with('-') => {
279 let minus = BinOp(BinOpToken::Minus);
280 let symbol = Symbol::intern(&symbol.as_str()[1..]);
281 let float = TokenKind::lit(token::Float, symbol, suffix);
282 let a = tokenstream::TokenTree::token(minus, span);
283 let b = tokenstream::TokenTree::token(float, span);
284 return [a, b].into_iter().collect();
286 TokenTree::Literal(self::Literal { kind, symbol, suffix, span }) => {
287 return tokenstream::TokenTree::token(
288 TokenKind::lit(kind.to_internal(), symbol, suffix),
295 let kind = match ch {
302 b'-' => BinOp(Minus),
304 b'/' => BinOp(Slash),
305 b'%' => BinOp(Percent),
306 b'^' => BinOp(Caret),
317 b'\'' => SingleQuote,
321 let tree = tokenstream::TokenTree::token(kind, span);
322 TokenStream::new(vec![(tree, if joint { Joint } else { Alone })])
326 impl ToInternal<rustc_errors::Level> for Level {
327 fn to_internal(self) -> rustc_errors::Level {
329 Level::Error => rustc_errors::Level::Error { lint: false },
330 Level::Warning => rustc_errors::Level::Warning(None),
331 Level::Note => rustc_errors::Level::Note,
332 Level::Help => rustc_errors::Level::Help,
333 _ => unreachable!("unknown proc_macro::Level variant: {:?}", self),
338 pub struct FreeFunctions;
340 pub(crate) struct Rustc<'a, 'b> {
341 ecx: &'a mut ExtCtxt<'b>,
346 rebased_spans: FxHashMap<usize, Span>,
349 impl<'a, 'b> Rustc<'a, 'b> {
350 pub fn new(ecx: &'a mut ExtCtxt<'b>) -> Self {
351 let expn_data = ecx.current_expansion.id.expn_data();
353 def_site: ecx.with_def_site_ctxt(expn_data.def_site),
354 call_site: ecx.with_call_site_ctxt(expn_data.call_site),
355 mixed_site: ecx.with_mixed_site_ctxt(expn_data.call_site),
356 krate: expn_data.macro_def_id.unwrap().krate,
357 rebased_spans: FxHashMap::default(),
362 fn sess(&self) -> &ParseSess {
363 self.ecx.parse_sess()
367 impl server::Types for Rustc<'_, '_> {
368 type FreeFunctions = FreeFunctions;
369 type TokenStream = TokenStream;
370 type SourceFile = Lrc<SourceFile>;
371 type MultiSpan = Vec<Span>;
372 type Diagnostic = Diagnostic;
374 type Symbol = Symbol;
377 impl server::FreeFunctions for Rustc<'_, '_> {
378 fn track_env_var(&mut self, var: &str, value: Option<&str>) {
382 .insert((Symbol::intern(var), value.map(Symbol::intern)));
385 fn track_path(&mut self, path: &str) {
386 self.sess().file_depinfo.borrow_mut().insert(Symbol::intern(path));
389 fn literal_from_str(&mut self, s: &str) -> Result<Literal<Self::Span, Self::Symbol>, ()> {
390 let name = FileName::proc_macro_source_code(s);
391 let mut parser = rustc_parse::new_parser_from_source_str(self.sess(), name, s.to_owned());
393 let first_span = parser.token.span.data();
394 let minus_present = parser.eat(&token::BinOp(token::Minus));
396 let lit_span = parser.token.span.data();
397 let token::Literal(mut lit) = parser.token.kind else {
401 // Check no comment or whitespace surrounding the (possibly negative)
402 // literal, or more tokens after it.
403 if (lit_span.hi.0 - first_span.lo.0) as usize != s.len() {
408 // If minus is present, check no comment or whitespace in between it
409 // and the literal token.
410 if first_span.hi.0 != lit_span.lo.0 {
414 // Check literal is a kind we allow to be negated in a proc macro token.
417 | token::LitKind::Byte
418 | token::LitKind::Char
419 | token::LitKind::Str
420 | token::LitKind::StrRaw(_)
421 | token::LitKind::ByteStr
422 | token::LitKind::ByteStrRaw(_)
423 | token::LitKind::Err => return Err(()),
424 token::LitKind::Integer | token::LitKind::Float => {}
427 // Synthesize a new symbol that includes the minus sign.
428 let symbol = Symbol::intern(&s[..1 + lit.symbol.as_str().len()]);
429 lit = token::Lit::new(lit.kind, symbol, lit.suffix);
431 let token::Lit { kind, symbol, suffix } = lit;
433 kind: FromInternal::from_internal(kind),
436 span: self.call_site,
441 impl server::TokenStream for Rustc<'_, '_> {
442 fn is_empty(&mut self, stream: &Self::TokenStream) -> bool {
446 fn from_str(&mut self, src: &str) -> Self::TokenStream {
447 parse_stream_from_source_str(
448 FileName::proc_macro_source_code(src),
451 Some(self.call_site),
455 fn to_string(&mut self, stream: &Self::TokenStream) -> String {
456 pprust::tts_to_string(stream)
459 fn expand_expr(&mut self, stream: &Self::TokenStream) -> Result<Self::TokenStream, ()> {
460 // Parse the expression from our tokenstream.
461 let expr: PResult<'_, _> = try {
462 let mut p = rustc_parse::stream_to_parser(
465 Some("proc_macro expand expr"),
467 let expr = p.parse_expr()?;
468 if p.token != token::Eof {
473 let expr = expr.map_err(|mut err| {
477 // Perform eager expansion on the expression.
481 .fully_expand_fragment(crate::expand::AstFragment::Expr(expr))
484 // NOTE: For now, limit `expand_expr` to exclusively expand to literals.
485 // This may be relaxed in the future.
486 // We don't use `TokenStream::from_ast` as the tokenstream currently cannot
487 // be recovered in the general case.
489 ast::ExprKind::Lit(l) if l.token.kind == token::Bool => {
490 Ok(tokenstream::TokenTree::token(token::Ident(l.token.symbol, false), l.span)
493 ast::ExprKind::Lit(l) => {
494 Ok(tokenstream::TokenTree::token(token::Literal(l.token), l.span).into())
496 ast::ExprKind::Unary(ast::UnOp::Neg, e) => match &e.kind {
497 ast::ExprKind::Lit(l) => match l.token {
498 token::Lit { kind: token::Integer | token::Float, .. } => {
499 Ok(Self::TokenStream::from_iter([
500 // FIXME: The span of the `-` token is lost when
501 // parsing, so we cannot faithfully recover it here.
502 tokenstream::TokenTree::token(token::BinOp(token::Minus), e.span),
503 tokenstream::TokenTree::token(token::Literal(l.token), l.span),
516 tree: TokenTree<Self::TokenStream, Self::Span, Self::Symbol>,
517 ) -> Self::TokenStream {
518 (tree, &mut *self).to_internal()
523 base: Option<Self::TokenStream>,
524 trees: Vec<TokenTree<Self::TokenStream, Self::Span, Self::Symbol>>,
525 ) -> Self::TokenStream {
526 let mut builder = tokenstream::TokenStreamBuilder::new();
527 if let Some(base) = base {
531 builder.push((tree, &mut *self).to_internal());
538 base: Option<Self::TokenStream>,
539 streams: Vec<Self::TokenStream>,
540 ) -> Self::TokenStream {
541 let mut builder = tokenstream::TokenStreamBuilder::new();
542 if let Some(base) = base {
545 for stream in streams {
546 builder.push(stream);
553 stream: Self::TokenStream,
554 ) -> Vec<TokenTree<Self::TokenStream, Self::Span, Self::Symbol>> {
555 FromInternal::from_internal((stream, self))
559 impl server::SourceFile for Rustc<'_, '_> {
560 fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool {
561 Lrc::ptr_eq(file1, file2)
564 fn path(&mut self, file: &Self::SourceFile) -> String {
566 FileName::Real(ref name) => name
568 .expect("attempting to get a file path in an imported file in `proc_macro::SourceFile::path`")
570 .expect("non-UTF8 file path in `proc_macro::SourceFile::path`")
572 _ => file.name.prefer_local().to_string(),
576 fn is_real(&mut self, file: &Self::SourceFile) -> bool {
581 impl server::MultiSpan for Rustc<'_, '_> {
582 fn new(&mut self) -> Self::MultiSpan {
586 fn push(&mut self, spans: &mut Self::MultiSpan, span: Self::Span) {
591 impl server::Diagnostic for Rustc<'_, '_> {
592 fn new(&mut self, level: Level, msg: &str, spans: Self::MultiSpan) -> Self::Diagnostic {
593 let mut diag = Diagnostic::new(level.to_internal(), msg);
594 diag.set_span(MultiSpan::from_spans(spans));
600 diag: &mut Self::Diagnostic,
603 spans: Self::MultiSpan,
605 diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None);
608 fn emit(&mut self, mut diag: Self::Diagnostic) {
609 self.sess().span_diagnostic.emit_diagnostic(&mut diag);
613 impl server::Span for Rustc<'_, '_> {
614 fn debug(&mut self, span: Self::Span) -> String {
615 if self.ecx.ecfg.span_debug {
616 format!("{:?}", span)
618 format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0)
622 fn source_file(&mut self, span: Self::Span) -> Self::SourceFile {
623 self.sess().source_map().lookup_char_pos(span.lo()).file
626 fn parent(&mut self, span: Self::Span) -> Option<Self::Span> {
627 span.parent_callsite()
630 fn source(&mut self, span: Self::Span) -> Self::Span {
631 span.source_callsite()
634 fn start(&mut self, span: Self::Span) -> LineColumn {
635 let loc = self.sess().source_map().lookup_char_pos(span.lo());
636 LineColumn { line: loc.line, column: loc.col.to_usize() }
639 fn end(&mut self, span: Self::Span) -> LineColumn {
640 let loc = self.sess().source_map().lookup_char_pos(span.hi());
641 LineColumn { line: loc.line, column: loc.col.to_usize() }
644 fn before(&mut self, span: Self::Span) -> Self::Span {
648 fn after(&mut self, span: Self::Span) -> Self::Span {
652 fn join(&mut self, first: Self::Span, second: Self::Span) -> Option<Self::Span> {
653 let self_loc = self.sess().source_map().lookup_char_pos(first.lo());
654 let other_loc = self.sess().source_map().lookup_char_pos(second.lo());
656 if self_loc.file.name != other_loc.file.name {
660 Some(first.to(second))
668 ) -> Option<Self::Span> {
669 let length = span.hi().to_usize() - span.lo().to_usize();
671 let start = match start {
672 Bound::Included(lo) => lo,
673 Bound::Excluded(lo) => lo.checked_add(1)?,
674 Bound::Unbounded => 0,
677 let end = match end {
678 Bound::Included(hi) => hi.checked_add(1)?,
679 Bound::Excluded(hi) => hi,
680 Bound::Unbounded => length,
683 // Bounds check the values, preventing addition overflow and OOB spans.
684 if start > u32::MAX as usize
685 || end > u32::MAX as usize
686 || (u32::MAX - start as u32) < span.lo().to_u32()
687 || (u32::MAX - end as u32) < span.lo().to_u32()
694 let new_lo = span.lo() + BytePos::from_usize(start);
695 let new_hi = span.lo() + BytePos::from_usize(end);
696 Some(span.with_lo(new_lo).with_hi(new_hi))
699 fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span {
700 span.with_ctxt(at.ctxt())
703 fn source_text(&mut self, span: Self::Span) -> Option<String> {
704 self.sess().source_map().span_to_snippet(span).ok()
706 /// Saves the provided span into the metadata of
707 /// *the crate we are currently compiling*, which must
708 /// be a proc-macro crate. This id can be passed to
709 /// `recover_proc_macro_span` when our current crate
710 /// is *run* as a proc-macro.
712 /// Let's suppose that we have two crates - `my_client`
713 /// and `my_proc_macro`. The `my_proc_macro` crate
714 /// contains a procedural macro `my_macro`, which
715 /// is implemented as: `quote! { "hello" }`
717 /// When we *compile* `my_proc_macro`, we will execute
718 /// the `quote` proc-macro. This will save the span of
719 /// "hello" into the metadata of `my_proc_macro`. As a result,
720 /// the body of `my_proc_macro` (after expansion) will end
721 /// up containing a call that looks like this:
722 /// `proc_macro::Ident::new("hello", proc_macro::Span::recover_proc_macro_span(0))`
724 /// where `0` is the id returned by this function.
725 /// When `my_proc_macro` *executes* (during the compilation of `my_client`),
726 /// the call to `recover_proc_macro_span` will load the corresponding
727 /// span from the metadata of `my_proc_macro` (which we have access to,
728 /// since we've loaded `my_proc_macro` from disk in order to execute it).
729 /// In this way, we have obtained a span pointing into `my_proc_macro`
730 fn save_span(&mut self, span: Self::Span) -> usize {
731 self.sess().save_proc_macro_span(span)
734 fn recover_proc_macro_span(&mut self, id: usize) -> Self::Span {
735 let (resolver, krate, def_site) = (&*self.ecx.resolver, self.krate, self.def_site);
736 *self.rebased_spans.entry(id).or_insert_with(|| {
737 // FIXME: `SyntaxContext` for spans from proc macro crates is lost during encoding,
738 // replace it with a def-site context until we are encoding it properly.
739 resolver.get_proc_macro_quoted_span(krate, id).with_ctxt(def_site.ctxt())
744 impl server::Symbol for Rustc<'_, '_> {
745 fn normalize_and_validate_ident(&mut self, string: &str) -> Result<Self::Symbol, ()> {
746 let sym = nfc_normalize(string);
747 if rustc_lexer::is_ident(sym.as_str()) { Ok(sym) } else { Err(()) }
751 impl server::Server for Rustc<'_, '_> {
752 fn globals(&mut self) -> ExpnGlobals<Self::Span> {
754 def_site: self.def_site,
755 call_site: self.call_site,
756 mixed_site: self.mixed_site,
760 fn intern_symbol(string: &str) -> Self::Symbol {
761 Symbol::intern(string)
764 fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) {