1 use crate::base::ExtCtxt;
3 server, DelimSpan, Diagnostic, ExpnGlobals, Group, Ident, LitKind, Literal, Punct, TokenTree,
5 use pm::{Delimiter, Level, LineColumn};
8 use rustc_ast::tokenstream::{self, Spacing::*, TokenStream};
9 use rustc_ast_pretty::pprust;
10 use rustc_data_structures::fx::FxHashMap;
11 use rustc_data_structures::sync::Lrc;
12 use rustc_errors::{MultiSpan, PResult};
13 use rustc_parse::lexer::nfc_normalize;
14 use rustc_parse::parse_stream_from_source_str;
15 use rustc_session::parse::ParseSess;
16 use rustc_span::def_id::CrateNum;
17 use rustc_span::symbol::{self, sym, Symbol};
18 use rustc_span::{BytePos, FileName, Pos, SourceFile, Span};
19 use smallvec::{smallvec, SmallVec};
22 trait FromInternal<T> {
23 fn from_internal(x: T) -> Self;
27 fn to_internal(self) -> T;
30 impl FromInternal<token::Delimiter> for Delimiter {
31 fn from_internal(delim: token::Delimiter) -> Delimiter {
33 token::Delimiter::Parenthesis => Delimiter::Parenthesis,
34 token::Delimiter::Brace => Delimiter::Brace,
35 token::Delimiter::Bracket => Delimiter::Bracket,
36 token::Delimiter::Invisible => Delimiter::None,
41 impl ToInternal<token::Delimiter> for Delimiter {
42 fn to_internal(self) -> token::Delimiter {
44 Delimiter::Parenthesis => token::Delimiter::Parenthesis,
45 Delimiter::Brace => token::Delimiter::Brace,
46 Delimiter::Bracket => token::Delimiter::Bracket,
47 Delimiter::None => token::Delimiter::Invisible,
52 impl FromInternal<token::LitKind> for LitKind {
53 fn from_internal(kind: token::LitKind) -> Self {
55 token::Byte => LitKind::Byte,
56 token::Char => LitKind::Char,
57 token::Integer => LitKind::Integer,
58 token::Float => LitKind::Float,
59 token::Str => LitKind::Str,
60 token::StrRaw(n) => LitKind::StrRaw(n),
61 token::ByteStr => LitKind::ByteStr,
62 token::ByteStrRaw(n) => LitKind::ByteStrRaw(n),
63 token::Err => LitKind::Err,
64 token::Bool => unreachable!(),
69 impl ToInternal<token::LitKind> for LitKind {
70 fn to_internal(self) -> token::LitKind {
72 LitKind::Byte => token::Byte,
73 LitKind::Char => token::Char,
74 LitKind::Integer => token::Integer,
75 LitKind::Float => token::Float,
76 LitKind::Str => token::Str,
77 LitKind::StrRaw(n) => token::StrRaw(n),
78 LitKind::ByteStr => token::ByteStr,
79 LitKind::ByteStrRaw(n) => token::ByteStrRaw(n),
80 LitKind::Err => token::Err,
85 impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStream, Span, Symbol>> {
86 fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self {
87 use rustc_ast::token::*;
89 // Estimate the capacity as `stream.len()` rounded up to the next power
90 // of two to limit the number of required reallocations.
91 let mut trees = Vec::with_capacity(stream.len().next_power_of_two());
92 let mut cursor = stream.into_trees();
94 while let Some(tree) = cursor.next() {
95 let (Token { kind, span }, joint) = match tree {
96 tokenstream::TokenTree::Delimited(span, delim, tts) => {
97 let delimiter = pm::Delimiter::from_internal(delim);
98 trees.push(TokenTree::Group(Group {
104 entire: span.entire(),
109 tokenstream::TokenTree::Token(token, spacing) => (token, spacing == Joint),
112 // Split the operator into one or more `Punct`s, one per character.
113 // The final one inherits the jointness of the original token. Any
114 // before that get `joint = true`.
115 let mut op = |s: &str| {
116 assert!(s.is_ascii());
117 trees.extend(s.bytes().enumerate().map(|(i, ch)| {
118 let is_final = i == s.len() - 1;
119 // Split the token span into single chars. Unless the span
120 // is an unusual one, e.g. due to proc macro expansion. We
121 // determine this by assuming any span with a length that
122 // matches the operator length is a normal one, and any
123 // span with a different length is an unusual one.
124 let span = if (span.hi() - span.lo()).to_usize() == s.len() {
125 let lo = span.lo() + BytePos::from_usize(i);
126 let hi = lo + BytePos::from_usize(1);
127 span.with_lo(lo).with_hi(hi)
131 TokenTree::Punct(Punct { ch, joint: if is_final { joint } else { true }, span })
147 BinOp(Plus) => op("+"),
148 BinOp(Minus) => op("-"),
149 BinOp(Star) => op("*"),
150 BinOp(Slash) => op("/"),
151 BinOp(Percent) => op("%"),
152 BinOp(Caret) => op("^"),
153 BinOp(And) => op("&"),
154 BinOp(Or) => op("|"),
155 BinOp(Shl) => op("<<"),
156 BinOp(Shr) => op(">>"),
157 BinOpEq(Plus) => op("+="),
158 BinOpEq(Minus) => op("-="),
159 BinOpEq(Star) => op("*="),
160 BinOpEq(Slash) => op("/="),
161 BinOpEq(Percent) => op("%="),
162 BinOpEq(Caret) => op("^="),
163 BinOpEq(And) => op("&="),
164 BinOpEq(Or) => op("|="),
165 BinOpEq(Shl) => op("<<="),
166 BinOpEq(Shr) => op(">>="),
170 DotDotDot => op("..."),
171 DotDotEq => op("..="),
178 FatArrow => op("=>"),
182 SingleQuote => op("'"),
184 Ident(sym, is_raw) => trees.push(TokenTree::Ident(Ident { sym, is_raw, span })),
186 let ident = symbol::Ident::new(name, span).without_first_quote();
188 TokenTree::Punct(Punct { ch: b'\'', joint: true, span }),
189 TokenTree::Ident(Ident { sym: ident.name, is_raw: false, span }),
192 Literal(token::Lit { kind, symbol, suffix }) => {
193 trees.push(TokenTree::Literal(self::Literal {
194 kind: FromInternal::from_internal(kind),
200 DocComment(_, attr_style, data) => {
201 let mut escaped = String::new();
202 for ch in data.as_str().chars() {
203 escaped.extend(ch.escape_debug());
206 Ident(sym::doc, false),
208 TokenKind::lit(token::Str, Symbol::intern(&escaped), None),
211 .map(|kind| tokenstream::TokenTree::token_alone(kind, span))
213 trees.push(TokenTree::Punct(Punct { ch: b'#', joint: false, span }));
214 if attr_style == ast::AttrStyle::Inner {
215 trees.push(TokenTree::Punct(Punct { ch: b'!', joint: false, span }));
217 trees.push(TokenTree::Group(Group {
218 delimiter: pm::Delimiter::Bracket,
219 stream: Some(stream),
220 span: DelimSpan::from_single(span),
224 Interpolated(nt) if let NtIdent(ident, is_raw) = *nt => {
225 trees.push(TokenTree::Ident(Ident { sym: ident.name, is_raw, span: ident.span }))
228 Interpolated(nt) => {
229 let stream = TokenStream::from_nonterminal_ast(&nt);
230 // A hack used to pass AST fragments to attribute and derive
231 // macros as a single nonterminal token instead of a token
232 // stream. Such token needs to be "unwrapped" and not
233 // represented as a delimited group.
234 // FIXME: It needs to be removed, but there are some
235 // compatibility issues (see #73345).
236 if crate::base::nt_pretty_printing_compatibility_hack(&nt, rustc.sess()) {
237 trees.extend(Self::from_internal((stream, rustc)));
239 trees.push(TokenTree::Group(Group {
240 delimiter: pm::Delimiter::None,
241 stream: Some(stream),
242 span: DelimSpan::from_single(span),
247 OpenDelim(..) | CloseDelim(..) => unreachable!(),
248 Eof => unreachable!(),
255 // We use a `SmallVec` because the output size is always one or two `TokenTree`s.
256 impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
257 for (TokenTree<TokenStream, Span, Symbol>, &mut Rustc<'_, '_>)
259 fn to_internal(self) -> SmallVec<[tokenstream::TokenTree; 2]> {
260 use rustc_ast::token::*;
262 let (tree, rustc) = self;
264 TokenTree::Punct(Punct { ch, joint, span }) => {
265 let kind = match ch {
272 b'-' => BinOp(Minus),
274 b'/' => BinOp(Slash),
275 b'%' => BinOp(Percent),
276 b'^' => BinOp(Caret),
287 b'\'' => SingleQuote,
291 tokenstream::TokenTree::token_joint(kind, span)
293 tokenstream::TokenTree::token_alone(kind, span)
296 TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => {
297 smallvec![tokenstream::TokenTree::Delimited(
298 tokenstream::DelimSpan { open, close },
299 delimiter.to_internal(),
300 stream.unwrap_or_default(),
303 TokenTree::Ident(self::Ident { sym, is_raw, span }) => {
304 rustc.sess().symbol_gallery.insert(sym, span);
305 smallvec![tokenstream::TokenTree::token_alone(Ident(sym, is_raw), span)]
307 TokenTree::Literal(self::Literal {
308 kind: self::LitKind::Integer,
312 }) if symbol.as_str().starts_with('-') => {
313 let minus = BinOp(BinOpToken::Minus);
314 let symbol = Symbol::intern(&symbol.as_str()[1..]);
315 let integer = TokenKind::lit(token::Integer, symbol, suffix);
316 let a = tokenstream::TokenTree::token_alone(minus, span);
317 let b = tokenstream::TokenTree::token_alone(integer, span);
320 TokenTree::Literal(self::Literal {
321 kind: self::LitKind::Float,
325 }) if symbol.as_str().starts_with('-') => {
326 let minus = BinOp(BinOpToken::Minus);
327 let symbol = Symbol::intern(&symbol.as_str()[1..]);
328 let float = TokenKind::lit(token::Float, symbol, suffix);
329 let a = tokenstream::TokenTree::token_alone(minus, span);
330 let b = tokenstream::TokenTree::token_alone(float, span);
333 TokenTree::Literal(self::Literal { kind, symbol, suffix, span }) => {
334 smallvec![tokenstream::TokenTree::token_alone(
335 TokenKind::lit(kind.to_internal(), symbol, suffix),
343 impl ToInternal<rustc_errors::Level> for Level {
344 fn to_internal(self) -> rustc_errors::Level {
346 Level::Error => rustc_errors::Level::Error { lint: false },
347 Level::Warning => rustc_errors::Level::Warning(None),
348 Level::Note => rustc_errors::Level::Note,
349 Level::Help => rustc_errors::Level::Help,
350 _ => unreachable!("unknown proc_macro::Level variant: {:?}", self),
355 pub struct FreeFunctions;
357 pub(crate) struct Rustc<'a, 'b> {
358 ecx: &'a mut ExtCtxt<'b>,
363 rebased_spans: FxHashMap<usize, Span>,
366 impl<'a, 'b> Rustc<'a, 'b> {
367 pub fn new(ecx: &'a mut ExtCtxt<'b>) -> Self {
368 let expn_data = ecx.current_expansion.id.expn_data();
370 def_site: ecx.with_def_site_ctxt(expn_data.def_site),
371 call_site: ecx.with_call_site_ctxt(expn_data.call_site),
372 mixed_site: ecx.with_mixed_site_ctxt(expn_data.call_site),
373 krate: expn_data.macro_def_id.unwrap().krate,
374 rebased_spans: FxHashMap::default(),
379 fn sess(&self) -> &ParseSess {
380 self.ecx.parse_sess()
384 impl server::Types for Rustc<'_, '_> {
385 type FreeFunctions = FreeFunctions;
386 type TokenStream = TokenStream;
387 type SourceFile = Lrc<SourceFile>;
389 type Symbol = Symbol;
392 impl server::FreeFunctions for Rustc<'_, '_> {
393 fn track_env_var(&mut self, var: &str, value: Option<&str>) {
397 .insert((Symbol::intern(var), value.map(Symbol::intern)));
400 fn track_path(&mut self, path: &str) {
401 self.sess().file_depinfo.borrow_mut().insert(Symbol::intern(path));
404 fn literal_from_str(&mut self, s: &str) -> Result<Literal<Self::Span, Self::Symbol>, ()> {
405 let name = FileName::proc_macro_source_code(s);
406 let mut parser = rustc_parse::new_parser_from_source_str(self.sess(), name, s.to_owned());
408 let first_span = parser.token.span.data();
409 let minus_present = parser.eat(&token::BinOp(token::Minus));
411 let lit_span = parser.token.span.data();
412 let token::Literal(mut lit) = parser.token.kind else {
416 // Check no comment or whitespace surrounding the (possibly negative)
417 // literal, or more tokens after it.
418 if (lit_span.hi.0 - first_span.lo.0) as usize != s.len() {
423 // If minus is present, check no comment or whitespace in between it
424 // and the literal token.
425 if first_span.hi.0 != lit_span.lo.0 {
429 // Check literal is a kind we allow to be negated in a proc macro token.
432 | token::LitKind::Byte
433 | token::LitKind::Char
434 | token::LitKind::Str
435 | token::LitKind::StrRaw(_)
436 | token::LitKind::ByteStr
437 | token::LitKind::ByteStrRaw(_)
438 | token::LitKind::Err => return Err(()),
439 token::LitKind::Integer | token::LitKind::Float => {}
442 // Synthesize a new symbol that includes the minus sign.
443 let symbol = Symbol::intern(&s[..1 + lit.symbol.as_str().len()]);
444 lit = token::Lit::new(lit.kind, symbol, lit.suffix);
446 let token::Lit { kind, symbol, suffix } = lit;
448 kind: FromInternal::from_internal(kind),
451 span: self.call_site,
455 fn emit_diagnostic(&mut self, diagnostic: Diagnostic<Self::Span>) {
457 rustc_errors::Diagnostic::new(diagnostic.level.to_internal(), diagnostic.message);
458 diag.set_span(MultiSpan::from_spans(diagnostic.spans));
459 for child in diagnostic.children {
461 child.level.to_internal(),
463 MultiSpan::from_spans(child.spans),
467 self.sess().span_diagnostic.emit_diagnostic(&mut diag);
471 impl server::TokenStream for Rustc<'_, '_> {
472 fn is_empty(&mut self, stream: &Self::TokenStream) -> bool {
476 fn from_str(&mut self, src: &str) -> Self::TokenStream {
477 parse_stream_from_source_str(
478 FileName::proc_macro_source_code(src),
481 Some(self.call_site),
485 fn to_string(&mut self, stream: &Self::TokenStream) -> String {
486 pprust::tts_to_string(stream)
489 fn expand_expr(&mut self, stream: &Self::TokenStream) -> Result<Self::TokenStream, ()> {
490 // Parse the expression from our tokenstream.
491 let expr: PResult<'_, _> = try {
492 let mut p = rustc_parse::stream_to_parser(
495 Some("proc_macro expand expr"),
497 let expr = p.parse_expr()?;
498 if p.token != token::Eof {
503 let expr = expr.map_err(|mut err| {
507 // Perform eager expansion on the expression.
511 .fully_expand_fragment(crate::expand::AstFragment::Expr(expr))
514 // NOTE: For now, limit `expand_expr` to exclusively expand to literals.
515 // This may be relaxed in the future.
516 // We don't use `TokenStream::from_ast` as the tokenstream currently cannot
517 // be recovered in the general case.
519 ast::ExprKind::Lit(token_lit) if token_lit.kind == token::Bool => {
520 Ok(tokenstream::TokenStream::token_alone(
521 token::Ident(token_lit.symbol, false),
525 ast::ExprKind::Lit(token_lit) => {
526 Ok(tokenstream::TokenStream::token_alone(token::Literal(*token_lit), expr.span))
528 ast::ExprKind::IncludedBytes(bytes) => {
529 let lit = ast::LitKind::ByteStr(bytes.clone()).to_token_lit();
530 Ok(tokenstream::TokenStream::token_alone(token::TokenKind::Literal(lit), expr.span))
532 ast::ExprKind::Unary(ast::UnOp::Neg, e) => match &e.kind {
533 ast::ExprKind::Lit(token_lit) => match token_lit {
534 token::Lit { kind: token::Integer | token::Float, .. } => {
535 Ok(Self::TokenStream::from_iter([
536 // FIXME: The span of the `-` token is lost when
537 // parsing, so we cannot faithfully recover it here.
538 tokenstream::TokenTree::token_alone(token::BinOp(token::Minus), e.span),
539 tokenstream::TokenTree::token_alone(token::Literal(*token_lit), e.span),
552 tree: TokenTree<Self::TokenStream, Self::Span, Self::Symbol>,
553 ) -> Self::TokenStream {
554 Self::TokenStream::new((tree, &mut *self).to_internal().into_iter().collect::<Vec<_>>())
559 base: Option<Self::TokenStream>,
560 trees: Vec<TokenTree<Self::TokenStream, Self::Span, Self::Symbol>>,
561 ) -> Self::TokenStream {
563 if let Some(base) = base { base } else { tokenstream::TokenStream::default() };
565 for tt in (tree, &mut *self).to_internal() {
566 stream.push_tree(tt);
574 base: Option<Self::TokenStream>,
575 streams: Vec<Self::TokenStream>,
576 ) -> Self::TokenStream {
578 if let Some(base) = base { base } else { tokenstream::TokenStream::default() };
580 stream.push_stream(s);
587 stream: Self::TokenStream,
588 ) -> Vec<TokenTree<Self::TokenStream, Self::Span, Self::Symbol>> {
589 FromInternal::from_internal((stream, self))
593 impl server::SourceFile for Rustc<'_, '_> {
594 fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool {
595 Lrc::ptr_eq(file1, file2)
598 fn path(&mut self, file: &Self::SourceFile) -> String {
600 FileName::Real(ref name) => name
602 .expect("attempting to get a file path in an imported file in `proc_macro::SourceFile::path`")
604 .expect("non-UTF8 file path in `proc_macro::SourceFile::path`")
606 _ => file.name.prefer_local().to_string(),
610 fn is_real(&mut self, file: &Self::SourceFile) -> bool {
615 impl server::Span for Rustc<'_, '_> {
616 fn debug(&mut self, span: Self::Span) -> String {
617 if self.ecx.ecfg.span_debug {
618 format!("{:?}", span)
620 format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0)
624 fn source_file(&mut self, span: Self::Span) -> Self::SourceFile {
625 self.sess().source_map().lookup_char_pos(span.lo()).file
628 fn parent(&mut self, span: Self::Span) -> Option<Self::Span> {
629 span.parent_callsite()
632 fn source(&mut self, span: Self::Span) -> Self::Span {
633 span.source_callsite()
636 fn start(&mut self, span: Self::Span) -> LineColumn {
637 let loc = self.sess().source_map().lookup_char_pos(span.lo());
638 LineColumn { line: loc.line, column: loc.col.to_usize() }
641 fn end(&mut self, span: Self::Span) -> LineColumn {
642 let loc = self.sess().source_map().lookup_char_pos(span.hi());
643 LineColumn { line: loc.line, column: loc.col.to_usize() }
646 fn before(&mut self, span: Self::Span) -> Self::Span {
650 fn after(&mut self, span: Self::Span) -> Self::Span {
654 fn join(&mut self, first: Self::Span, second: Self::Span) -> Option<Self::Span> {
655 let self_loc = self.sess().source_map().lookup_char_pos(first.lo());
656 let other_loc = self.sess().source_map().lookup_char_pos(second.lo());
658 if self_loc.file.name != other_loc.file.name {
662 Some(first.to(second))
670 ) -> Option<Self::Span> {
671 let length = span.hi().to_usize() - span.lo().to_usize();
673 let start = match start {
674 Bound::Included(lo) => lo,
675 Bound::Excluded(lo) => lo.checked_add(1)?,
676 Bound::Unbounded => 0,
679 let end = match end {
680 Bound::Included(hi) => hi.checked_add(1)?,
681 Bound::Excluded(hi) => hi,
682 Bound::Unbounded => length,
685 // Bounds check the values, preventing addition overflow and OOB spans.
686 if start > u32::MAX as usize
687 || end > u32::MAX as usize
688 || (u32::MAX - start as u32) < span.lo().to_u32()
689 || (u32::MAX - end as u32) < span.lo().to_u32()
696 let new_lo = span.lo() + BytePos::from_usize(start);
697 let new_hi = span.lo() + BytePos::from_usize(end);
698 Some(span.with_lo(new_lo).with_hi(new_hi))
701 fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span {
702 span.with_ctxt(at.ctxt())
705 fn source_text(&mut self, span: Self::Span) -> Option<String> {
706 self.sess().source_map().span_to_snippet(span).ok()
709 /// Saves the provided span into the metadata of
710 /// *the crate we are currently compiling*, which must
711 /// be a proc-macro crate. This id can be passed to
712 /// `recover_proc_macro_span` when our current crate
713 /// is *run* as a proc-macro.
715 /// Let's suppose that we have two crates - `my_client`
716 /// and `my_proc_macro`. The `my_proc_macro` crate
717 /// contains a procedural macro `my_macro`, which
718 /// is implemented as: `quote! { "hello" }`
720 /// When we *compile* `my_proc_macro`, we will execute
721 /// the `quote` proc-macro. This will save the span of
722 /// "hello" into the metadata of `my_proc_macro`. As a result,
723 /// the body of `my_proc_macro` (after expansion) will end
724 /// up containing a call that looks like this:
725 /// `proc_macro::Ident::new("hello", proc_macro::Span::recover_proc_macro_span(0))`
727 /// where `0` is the id returned by this function.
728 /// When `my_proc_macro` *executes* (during the compilation of `my_client`),
729 /// the call to `recover_proc_macro_span` will load the corresponding
730 /// span from the metadata of `my_proc_macro` (which we have access to,
731 /// since we've loaded `my_proc_macro` from disk in order to execute it).
732 /// In this way, we have obtained a span pointing into `my_proc_macro`
733 fn save_span(&mut self, span: Self::Span) -> usize {
734 self.sess().save_proc_macro_span(span)
737 fn recover_proc_macro_span(&mut self, id: usize) -> Self::Span {
738 let (resolver, krate, def_site) = (&*self.ecx.resolver, self.krate, self.def_site);
739 *self.rebased_spans.entry(id).or_insert_with(|| {
740 // FIXME: `SyntaxContext` for spans from proc macro crates is lost during encoding,
741 // replace it with a def-site context until we are encoding it properly.
742 resolver.get_proc_macro_quoted_span(krate, id).with_ctxt(def_site.ctxt())
747 impl server::Symbol for Rustc<'_, '_> {
748 fn normalize_and_validate_ident(&mut self, string: &str) -> Result<Self::Symbol, ()> {
749 let sym = nfc_normalize(string);
750 if rustc_lexer::is_ident(sym.as_str()) { Ok(sym) } else { Err(()) }
754 impl server::Server for Rustc<'_, '_> {
755 fn globals(&mut self) -> ExpnGlobals<Self::Span> {
757 def_site: self.def_site,
758 call_site: self.call_site,
759 mixed_site: self.mixed_site,
763 fn intern_symbol(string: &str) -> Self::Symbol {
764 Symbol::intern(string)
767 fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) {