1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! A support library for macro authors when defining new macros.
13 //! This library, provided by the standard distribution, provides the types
14 //! consumed in the interfaces of procedurally defined macro definitions such as
15 //! function-like macros `#[proc_macro]`, macro attributes `#[proc_macro_attribute]` and
16 //! custom derive attributes`#[proc_macro_derive]`.
18 //! See [the book](../book/first-edition/procedural-macros.html) for more.
20 #![stable(feature = "proc_macro_lib", since = "1.15.0")]
21 #![deny(missing_docs)]
22 #![doc(html_logo_url = "https://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png",
23 html_favicon_url = "https://doc.rust-lang.org/favicon.ico",
24 html_root_url = "https://doc.rust-lang.org/nightly/",
25 html_playground_url = "https://play.rust-lang.org/",
26 issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/",
27 test(no_crate_inject, attr(deny(warnings))),
28 test(attr(allow(dead_code, deprecated, unused_variables, unused_mut))))]
31 #![feature(rustc_private)]
32 #![feature(staged_api)]
33 #![feature(lang_items)]
34 #![feature(optin_builtin_traits)]
35 #![feature(non_exhaustive)]
37 #![recursion_limit="256"]
40 extern crate syntax_pos;
41 extern crate rustc_errors;
42 extern crate rustc_data_structures;
44 #[unstable(feature = "proc_macro_internals", issue = "27812")]
50 #[unstable(feature = "proc_macro_diagnostic", issue = "54140")]
51 pub use diagnostic::{Diagnostic, Level, MultiSpan};
53 use std::ops::{Bound, RangeBounds};
54 use std::{ascii, fmt, iter};
55 use std::path::PathBuf;
56 use rustc_data_structures::sync::Lrc;
57 use std::str::FromStr;
59 use syntax::errors::DiagnosticBuilder;
60 use syntax::parse::{self, token};
61 use syntax::symbol::Symbol;
62 use syntax::tokenstream::{self, DelimSpan};
63 use syntax_pos::{Pos, FileName, BytePos};
65 /// The main type provided by this crate, representing an abstract stream of
66 /// tokens, or, more specifically, a sequence of token trees.
67 /// The type provide interfaces for iterating over those token trees and, conversely,
68 /// collecting a number of token trees into one stream.
70 /// This is both the input and output of `#[proc_macro]`, `#[proc_macro_attribute]`
71 /// and `#[proc_macro_derive]` definitions.
72 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
74 pub struct TokenStream(tokenstream::TokenStream);
76 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
77 impl !Send for TokenStream {}
78 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
79 impl !Sync for TokenStream {}
81 /// Error returned from `TokenStream::from_str`.
82 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
88 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
89 impl !Send for LexError {}
90 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
91 impl !Sync for LexError {}
94 /// Returns an empty `TokenStream` containing no token trees.
95 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
96 pub fn new() -> TokenStream {
97 TokenStream(tokenstream::TokenStream::empty())
100 /// Checks if this `TokenStream` is empty.
101 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
102 pub fn is_empty(&self) -> bool {
107 /// Attempts to break the string into tokens and parse those tokens into a token stream.
108 /// May fail for a number of reasons, for example, if the string contains unbalanced delimiters
109 /// or characters not existing in the language.
110 /// All tokens in the parsed stream get `Span::call_site()` spans.
112 /// NOTE: Some errors may cause panics instead of returning `LexError`. We reserve the right to
113 /// change these errors into `LexError`s later.
114 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
115 impl FromStr for TokenStream {
118 fn from_str(src: &str) -> Result<TokenStream, LexError> {
119 __internal::with_sess(|sess, data| {
120 Ok(__internal::token_stream_wrap(parse::parse_stream_from_source_str(
121 FileName::ProcMacroSourceCode, src.to_string(), sess, Some(data.call_site.0)
127 /// Prints the token stream as a string that is supposed to be losslessly convertible back
128 /// into the same token stream (modulo spans), except for possibly `TokenTree::Group`s
129 /// with `Delimiter::None` delimiters and negative numeric literals.
130 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
131 impl fmt::Display for TokenStream {
132 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
137 /// Prints token in a form convenient for debugging.
138 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
139 impl fmt::Debug for TokenStream {
140 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
141 f.write_str("TokenStream ")?;
142 f.debug_list().entries(self.clone()).finish()
146 #[unstable(feature = "proc_macro_quote", issue = "54722")]
147 pub use quote::{quote, quote_span};
149 /// Creates a token stream containing a single token tree.
150 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
151 impl From<TokenTree> for TokenStream {
152 fn from(tree: TokenTree) -> TokenStream {
153 TokenStream(tree.to_internal())
157 /// Collects a number of token trees into a single stream.
158 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
159 impl iter::FromIterator<TokenTree> for TokenStream {
160 fn from_iter<I: IntoIterator<Item = TokenTree>>(trees: I) -> Self {
161 trees.into_iter().map(TokenStream::from).collect()
165 /// A "flattening" operation on token streams, collects token trees
166 /// from multiple token streams into a single stream.
167 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
168 impl iter::FromIterator<TokenStream> for TokenStream {
169 fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self {
170 let mut builder = tokenstream::TokenStreamBuilder::new();
171 for stream in streams {
172 builder.push(stream.0);
174 TokenStream(builder.build())
178 #[stable(feature = "token_stream_extend", since = "1.30.0")]
179 impl Extend<TokenTree> for TokenStream {
180 fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, trees: I) {
181 self.extend(trees.into_iter().map(TokenStream::from));
185 #[stable(feature = "token_stream_extend", since = "1.30.0")]
186 impl Extend<TokenStream> for TokenStream {
187 fn extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I) {
188 self.0.extend(streams.into_iter().map(|stream| stream.0));
192 /// Public implementation details for the `TokenStream` type, such as iterators.
193 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
194 pub mod token_stream {
195 use syntax::tokenstream;
196 use {TokenTree, TokenStream, Delimiter};
198 /// An iterator over `TokenStream`'s `TokenTree`s.
199 /// The iteration is "shallow", e.g. the iterator doesn't recurse into delimited groups,
200 /// and returns whole groups as token trees.
202 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
203 pub struct IntoIter {
204 cursor: tokenstream::Cursor,
205 stack: Vec<TokenTree>,
208 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
209 impl Iterator for IntoIter {
210 type Item = TokenTree;
212 fn next(&mut self) -> Option<TokenTree> {
214 let tree = self.stack.pop().or_else(|| {
215 let next = self.cursor.next_as_stream()?;
216 Some(TokenTree::from_internal(next, &mut self.stack))
218 // HACK: The condition "dummy span + group with empty delimiter" represents an AST
219 // fragment approximately converted into a token stream. This may happen, for
220 // example, with inputs to proc macro attributes, including derives. Such "groups"
221 // need to flattened during iteration over stream's token trees.
222 // Eventually this needs to be removed in favor of keeping original token trees
223 // and not doing the roundtrip through AST.
224 if tree.span().0.is_dummy() {
225 if let TokenTree::Group(ref group) = tree {
226 if group.delimiter() == Delimiter::None {
227 self.cursor.insert(group.stream.clone().0);
237 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
238 impl IntoIterator for TokenStream {
239 type Item = TokenTree;
240 type IntoIter = IntoIter;
242 fn into_iter(self) -> IntoIter {
243 IntoIter { cursor: self.0.trees(), stack: Vec::new() }
248 /// `quote!(..)` accepts arbitrary tokens and expands into a `TokenStream` describing the input.
249 /// For example, `quote!(a + b)` will produce a expression, that, when evaluated, constructs
250 /// the `TokenStream` `[Ident("a"), Punct('+', Alone), Ident("b")]`.
252 /// Unquoting is done with `$`, and works by taking the single next ident as the unquoted term.
253 /// To quote `$` itself, use `$$`.
255 /// This is a dummy macro, the actual implementation is in `quote::quote`.`
256 #[unstable(feature = "proc_macro_quote", issue = "54722")]
258 macro_rules! quote { () => {} }
260 #[unstable(feature = "proc_macro_internals", issue = "27812")]
264 /// A region of source code, along with macro expansion information.
265 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
266 #[derive(Copy, Clone)]
267 pub struct Span(syntax_pos::Span);
269 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
270 impl !Send for Span {}
271 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
272 impl !Sync for Span {}
274 macro_rules! diagnostic_method {
275 ($name:ident, $level:expr) => (
276 /// Create a new `Diagnostic` with the given `message` at the span
278 #[unstable(feature = "proc_macro_diagnostic", issue = "54140")]
279 pub fn $name<T: Into<String>>(self, message: T) -> Diagnostic {
280 Diagnostic::spanned(self, $level, message)
286 /// A span that resolves at the macro definition site.
287 #[unstable(feature = "proc_macro_def_site", issue = "54724")]
288 pub fn def_site() -> Span {
289 ::__internal::with_sess(|_, data| data.def_site)
292 /// The span of the invocation of the current procedural macro.
293 /// Identifiers created with this span will be resolved as if they were written
294 /// directly at the macro call location (call-site hygiene) and other code
295 /// at the macro call site will be able to refer to them as well.
296 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
297 pub fn call_site() -> Span {
298 ::__internal::with_sess(|_, data| data.call_site)
301 /// The original source file into which this span points.
302 #[unstable(feature = "proc_macro_span", issue = "54725")]
303 pub fn source_file(&self) -> SourceFile {
305 source_file: __internal::lookup_char_pos(self.0.lo()).file,
309 /// The `Span` for the tokens in the previous macro expansion from which
310 /// `self` was generated from, if any.
311 #[unstable(feature = "proc_macro_span", issue = "54725")]
312 pub fn parent(&self) -> Option<Span> {
313 self.0.parent().map(Span)
316 /// The span for the origin source code that `self` was generated from. If
317 /// this `Span` wasn't generated from other macro expansions then the return
318 /// value is the same as `*self`.
319 #[unstable(feature = "proc_macro_span", issue = "54725")]
320 pub fn source(&self) -> Span {
321 Span(self.0.source_callsite())
324 /// Get the starting line/column in the source file for this span.
325 #[unstable(feature = "proc_macro_span", issue = "54725")]
326 pub fn start(&self) -> LineColumn {
327 let loc = __internal::lookup_char_pos(self.0.lo());
330 column: loc.col.to_usize()
334 /// Get the ending line/column in the source file for this span.
335 #[unstable(feature = "proc_macro_span", issue = "54725")]
336 pub fn end(&self) -> LineColumn {
337 let loc = __internal::lookup_char_pos(self.0.hi());
340 column: loc.col.to_usize()
344 /// Create a new span encompassing `self` and `other`.
346 /// Returns `None` if `self` and `other` are from different files.
347 #[unstable(feature = "proc_macro_span", issue = "54725")]
348 pub fn join(&self, other: Span) -> Option<Span> {
349 let self_loc = __internal::lookup_char_pos(self.0.lo());
350 let other_loc = __internal::lookup_char_pos(other.0.lo());
352 if self_loc.file.name != other_loc.file.name { return None }
354 Some(Span(self.0.to(other.0)))
357 /// Creates a new span with the same line/column information as `self` but
358 /// that resolves symbols as though it were at `other`.
359 #[unstable(feature = "proc_macro_span", issue = "54725")]
360 pub fn resolved_at(&self, other: Span) -> Span {
361 Span(self.0.with_ctxt(other.0.ctxt()))
364 /// Creates a new span with the same name resolution behavior as `self` but
365 /// with the line/column information of `other`.
366 #[unstable(feature = "proc_macro_span", issue = "54725")]
367 pub fn located_at(&self, other: Span) -> Span {
368 other.resolved_at(*self)
371 /// Compares to spans to see if they're equal.
372 #[unstable(feature = "proc_macro_span", issue = "54725")]
373 pub fn eq(&self, other: &Span) -> bool {
377 diagnostic_method!(error, Level::Error);
378 diagnostic_method!(warning, Level::Warning);
379 diagnostic_method!(note, Level::Note);
380 diagnostic_method!(help, Level::Help);
383 /// Prints a span in a form convenient for debugging.
384 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
385 impl fmt::Debug for Span {
386 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
387 write!(f, "{:?} bytes({}..{})",
394 /// A line-column pair representing the start or end of a `Span`.
395 #[unstable(feature = "proc_macro_span", issue = "54725")]
396 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
397 pub struct LineColumn {
398 /// The 1-indexed line in the source file on which the span starts or ends (inclusive).
399 #[unstable(feature = "proc_macro_span", issue = "54725")]
401 /// The 0-indexed column (in UTF-8 characters) in the source file on which
402 /// the span starts or ends (inclusive).
403 #[unstable(feature = "proc_macro_span", issue = "54725")]
407 #[unstable(feature = "proc_macro_span", issue = "54725")]
408 impl !Send for LineColumn {}
409 #[unstable(feature = "proc_macro_span", issue = "54725")]
410 impl !Sync for LineColumn {}
412 /// The source file of a given `Span`.
413 #[unstable(feature = "proc_macro_span", issue = "54725")]
415 pub struct SourceFile {
416 source_file: Lrc<syntax_pos::SourceFile>,
419 #[unstable(feature = "proc_macro_span", issue = "54725")]
420 impl !Send for SourceFile {}
421 #[unstable(feature = "proc_macro_span", issue = "54725")]
422 impl !Sync for SourceFile {}
425 /// Get the path to this source file.
428 /// If the code span associated with this `SourceFile` was generated by an external macro, this
429 /// macro, this may not be an actual path on the filesystem. Use [`is_real`] to check.
431 /// Also note that even if `is_real` returns `true`, if `--remap-path-prefix` was passed on
432 /// the command line, the path as given may not actually be valid.
434 /// [`is_real`]: #method.is_real
435 #[unstable(feature = "proc_macro_span", issue = "54725")]
436 pub fn path(&self) -> PathBuf {
437 match self.source_file.name {
438 FileName::Real(ref path) => path.clone(),
439 _ => PathBuf::from(self.source_file.name.to_string())
443 /// Returns `true` if this source file is a real source file, and not generated by an external
444 /// macro's expansion.
445 #[unstable(feature = "proc_macro_span", issue = "54725")]
446 pub fn is_real(&self) -> bool {
447 // This is a hack until intercrate spans are implemented and we can have real source files
448 // for spans generated in external macros.
449 // https://github.com/rust-lang/rust/pull/43604#issuecomment-333334368
450 self.source_file.is_real_file()
455 #[unstable(feature = "proc_macro_span", issue = "54725")]
456 impl fmt::Debug for SourceFile {
457 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
458 f.debug_struct("SourceFile")
459 .field("path", &self.path())
460 .field("is_real", &self.is_real())
465 #[unstable(feature = "proc_macro_span", issue = "54725")]
466 impl PartialEq for SourceFile {
467 fn eq(&self, other: &Self) -> bool {
468 Lrc::ptr_eq(&self.source_file, &other.source_file)
472 #[unstable(feature = "proc_macro_span", issue = "54725")]
473 impl Eq for SourceFile {}
475 /// A single token or a delimited sequence of token trees (e.g. `[1, (), ..]`).
476 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
479 /// A token stream surrounded by bracket delimiters.
480 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
482 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
486 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
488 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
491 /// A single punctuation character (`+`, `,`, `$`, etc.).
492 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
494 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
497 /// A literal character (`'a'`), string (`"hello"`), number (`2.3`), etc.
498 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
500 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
505 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
506 impl !Send for TokenTree {}
507 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
508 impl !Sync for TokenTree {}
511 /// Returns the span of this tree, delegating to the `span` method of
512 /// the contained token or a delimited stream.
513 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
514 pub fn span(&self) -> Span {
516 TokenTree::Group(ref t) => t.span(),
517 TokenTree::Ident(ref t) => t.span(),
518 TokenTree::Punct(ref t) => t.span(),
519 TokenTree::Literal(ref t) => t.span(),
523 /// Configures the span for *only this token*.
525 /// Note that if this token is a `Group` then this method will not configure
526 /// the span of each of the internal tokens, this will simply delegate to
527 /// the `set_span` method of each variant.
528 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
529 pub fn set_span(&mut self, span: Span) {
531 TokenTree::Group(ref mut t) => t.set_span(span),
532 TokenTree::Ident(ref mut t) => t.set_span(span),
533 TokenTree::Punct(ref mut t) => t.set_span(span),
534 TokenTree::Literal(ref mut t) => t.set_span(span),
539 /// Prints token tree in a form convenient for debugging.
540 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
541 impl fmt::Debug for TokenTree {
542 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
543 // Each of these has the name in the struct type in the derived debug,
544 // so don't bother with an extra layer of indirection
546 TokenTree::Group(ref tt) => tt.fmt(f),
547 TokenTree::Ident(ref tt) => tt.fmt(f),
548 TokenTree::Punct(ref tt) => tt.fmt(f),
549 TokenTree::Literal(ref tt) => tt.fmt(f),
554 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
555 impl From<Group> for TokenTree {
556 fn from(g: Group) -> TokenTree {
561 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
562 impl From<Ident> for TokenTree {
563 fn from(g: Ident) -> TokenTree {
568 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
569 impl From<Punct> for TokenTree {
570 fn from(g: Punct) -> TokenTree {
575 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
576 impl From<Literal> for TokenTree {
577 fn from(g: Literal) -> TokenTree {
578 TokenTree::Literal(g)
582 /// Prints the token tree as a string that is supposed to be losslessly convertible back
583 /// into the same token tree (modulo spans), except for possibly `TokenTree::Group`s
584 /// with `Delimiter::None` delimiters and negative numeric literals.
585 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
586 impl fmt::Display for TokenTree {
587 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
589 TokenTree::Group(ref t) => t.fmt(f),
590 TokenTree::Ident(ref t) => t.fmt(f),
591 TokenTree::Punct(ref t) => t.fmt(f),
592 TokenTree::Literal(ref t) => t.fmt(f),
597 /// A delimited token stream.
599 /// A `Group` internally contains a `TokenStream` which is surrounded by `Delimiter`s.
601 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
603 delimiter: Delimiter,
608 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
609 impl !Send for Group {}
610 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
611 impl !Sync for Group {}
613 /// Describes how a sequence of token trees is delimited.
614 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
615 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
618 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
621 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
624 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
627 /// An implicit delimiter, that may, for example, appear around tokens coming from a
628 /// "macro variable" `$var`. It is important to preserve operator priorities in cases like
629 /// `$var * 3` where `$var` is `1 + 2`.
630 /// Implicit delimiters may not survive roundtrip of a token stream through a string.
631 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
636 /// Creates a new `Group` with the given delimiter and token stream.
638 /// This constructor will set the span for this group to
639 /// `Span::call_site()`. To change the span you can use the `set_span`
641 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
642 pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group {
644 delimiter: delimiter,
646 span: DelimSpan::from_single(Span::call_site().0),
650 /// Returns the delimiter of this `Group`
651 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
652 pub fn delimiter(&self) -> Delimiter {
656 /// Returns the `TokenStream` of tokens that are delimited in this `Group`.
658 /// Note that the returned token stream does not include the delimiter
660 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
661 pub fn stream(&self) -> TokenStream {
665 /// Returns the span for the delimiters of this token stream, spanning the
669 /// pub fn span(&self) -> Span {
672 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
673 pub fn span(&self) -> Span {
674 Span(self.span.entire())
677 /// Returns the span pointing to the opening delimiter of this group.
680 /// pub fn span_open(&self) -> Span {
683 #[unstable(feature = "proc_macro_span", issue = "54725")]
684 pub fn span_open(&self) -> Span {
688 /// Returns the span pointing to the closing delimiter of this group.
691 /// pub fn span_close(&self) -> Span {
694 #[unstable(feature = "proc_macro_span", issue = "54725")]
695 pub fn span_close(&self) -> Span {
696 Span(self.span.close)
699 /// Configures the span for this `Group`'s delimiters, but not its internal
702 /// This method will **not** set the span of all the internal tokens spanned
703 /// by this group, but rather it will only set the span of the delimiter
704 /// tokens at the level of the `Group`.
705 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
706 pub fn set_span(&mut self, span: Span) {
707 self.span = DelimSpan::from_single(span.0);
711 /// Prints the group as a string that should be losslessly convertible back
712 /// into the same group (modulo spans), except for possibly `TokenTree::Group`s
713 /// with `Delimiter::None` delimiters.
714 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
715 impl fmt::Display for Group {
716 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
717 TokenStream::from(TokenTree::from(self.clone())).fmt(f)
721 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
722 impl fmt::Debug for Group {
723 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
724 f.debug_struct("Group")
725 .field("delimiter", &self.delimiter())
726 .field("stream", &self.stream())
727 .field("span", &self.span())
732 /// An `Punct` is an single punctuation character like `+`, `-` or `#`.
734 /// Multi-character operators like `+=` are represented as two instances of `Punct` with different
735 /// forms of `Spacing` returned.
736 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
744 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
745 impl !Send for Punct {}
746 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
747 impl !Sync for Punct {}
749 /// Whether an `Punct` is followed immediately by another `Punct` or
750 /// followed by another token or whitespace.
751 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
752 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
754 /// E.g. `+` is `Alone` in `+ =`, `+ident` or `+()`.
755 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
757 /// E.g. `+` is `Joint` in `+=` or `'#`.
758 /// Additionally, single quote `'` can join with identifiers to form lifetimes `'ident`.
759 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
764 /// Creates a new `Punct` from the given character and spacing.
765 /// The `ch` argument must be a valid punctuation character permitted by the language,
766 /// otherwise the function will panic.
768 /// The returned `Punct` will have the default span of `Span::call_site()`
769 /// which can be further configured with the `set_span` method below.
770 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
771 pub fn new(ch: char, spacing: Spacing) -> Punct {
772 const LEGAL_CHARS: &[char] = &['=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^',
773 '&', '|', '@', '.', ',', ';', ':', '#', '$', '?', '\''];
774 if !LEGAL_CHARS.contains(&ch) {
775 panic!("unsupported character `{:?}`", ch)
780 span: Span::call_site(),
784 /// Returns the value of this punctuation character as `char`.
785 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
786 pub fn as_char(&self) -> char {
790 /// Returns the spacing of this punctuation character, indicating whether it's immediately
791 /// followed by another `Punct` in the token stream, so they can potentially be combined into
792 /// a multi-character operator (`Joint`), or it's followed by some other token or whitespace
793 /// (`Alone`) so the operator has certainly ended.
794 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
795 pub fn spacing(&self) -> Spacing {
799 /// Returns the span for this punctuation character.
800 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
801 pub fn span(&self) -> Span {
805 /// Configure the span for this punctuation character.
806 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
807 pub fn set_span(&mut self, span: Span) {
812 /// Prints the punctuation character as a string that should be losslessly convertible
813 /// back into the same character.
814 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
815 impl fmt::Display for Punct {
816 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
817 TokenStream::from(TokenTree::from(self.clone())).fmt(f)
821 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
822 impl fmt::Debug for Punct {
823 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
824 f.debug_struct("Punct")
825 .field("ch", &self.as_char())
826 .field("spacing", &self.spacing())
827 .field("span", &self.span())
832 /// An identifier (`ident`).
834 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
841 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
842 impl !Send for Ident {}
843 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
844 impl !Sync for Ident {}
847 fn is_valid(string: &str) -> bool {
848 let mut chars = string.chars();
849 if let Some(start) = chars.next() {
850 (start == '_' || start.is_xid_start())
851 && chars.all(|cont| cont == '_' || cont.is_xid_continue())
857 /// Creates a new `Ident` with the given `string` as well as the specified
859 /// The `string` argument must be a valid identifier permitted by the
860 /// language, otherwise the function will panic.
862 /// Note that `span`, currently in rustc, configures the hygiene information
863 /// for this identifier.
865 /// As of this time `Span::call_site()` explicitly opts-in to "call-site" hygiene
866 /// meaning that identifiers created with this span will be resolved as if they were written
867 /// directly at the location of the macro call, and other code at the macro call site will be
868 /// able to refer to them as well.
870 /// Later spans like `Span::def_site()` will allow to opt-in to "definition-site" hygiene
871 /// meaning that identifiers created with this span will be resolved at the location of the
872 /// macro definition and other code at the macro call site will not be able to refer to them.
874 /// Due to the current importance of hygiene this constructor, unlike other
875 /// tokens, requires a `Span` to be specified at construction.
876 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
877 pub fn new(string: &str, span: Span) -> Ident {
878 if !Ident::is_valid(string) {
879 panic!("`{:?}` is not a valid identifier", string)
881 Ident::new_maybe_raw(string, span, false)
884 /// Same as `Ident::new`, but creates a raw identifier (`r#ident`).
885 #[unstable(feature = "proc_macro_raw_ident", issue = "54723")]
886 pub fn new_raw(string: &str, span: Span) -> Ident {
887 if !Ident::is_valid(string) {
888 panic!("`{:?}` is not a valid identifier", string)
890 Ident::new_maybe_raw(string, span, true)
893 /// Returns the span of this `Ident`, encompassing the entire string returned
895 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
896 pub fn span(&self) -> Span {
900 /// Configures the span of this `Ident`, possibly changing its hygiene context.
901 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
902 pub fn set_span(&mut self, span: Span) {
907 /// Prints the identifier as a string that should be losslessly convertible
908 /// back into the same identifier.
909 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
910 impl fmt::Display for Ident {
911 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
912 TokenStream::from(TokenTree::from(self.clone())).fmt(f)
916 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
917 impl fmt::Debug for Ident {
918 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
919 f.debug_struct("Ident")
920 .field("ident", &self.to_string())
921 .field("span", &self.span())
926 /// A literal string (`"hello"`), byte string (`b"hello"`),
927 /// character (`'a'`), byte character (`b'a'`), an integer or floating point number
928 /// with or without a suffix (`1`, `1u8`, `2.3`, `2.3f32`).
929 /// Boolean literals like `true` and `false` do not belong here, they are `Ident`s.
930 // FIXME(eddyb) `Literal` should not expose internal `Debug` impls.
931 #[derive(Clone, Debug)]
932 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
935 suffix: Option<Symbol>,
939 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
940 impl !Send for Literal {}
941 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
942 impl !Sync for Literal {}
944 macro_rules! suffixed_int_literals {
945 ($($name:ident => $kind:ident,)*) => ($(
946 /// Creates a new suffixed integer literal with the specified value.
948 /// This function will create an integer like `1u32` where the integer
949 /// value specified is the first part of the token and the integral is
950 /// also suffixed at the end.
951 /// Literals created from negative numbers may not survive round-trips through
952 /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal).
954 /// Literals created through this method have the `Span::call_site()`
955 /// span by default, which can be configured with the `set_span` method
957 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
958 pub fn $name(n: $kind) -> Literal {
960 lit: token::Lit::Integer(Symbol::intern(&n.to_string())),
961 suffix: Some(Symbol::intern(stringify!($kind))),
962 span: Span::call_site(),
968 macro_rules! unsuffixed_int_literals {
969 ($($name:ident => $kind:ident,)*) => ($(
970 /// Creates a new unsuffixed integer literal with the specified value.
972 /// This function will create an integer like `1` where the integer
973 /// value specified is the first part of the token. No suffix is
974 /// specified on this token, meaning that invocations like
975 /// `Literal::i8_unsuffixed(1)` are equivalent to
976 /// `Literal::u32_unsuffixed(1)`.
977 /// Literals created from negative numbers may not survive rountrips through
978 /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal).
980 /// Literals created through this method have the `Span::call_site()`
981 /// span by default, which can be configured with the `set_span` method
983 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
984 pub fn $name(n: $kind) -> Literal {
986 lit: token::Lit::Integer(Symbol::intern(&n.to_string())),
988 span: Span::call_site(),
995 suffixed_int_literals! {
1000 u128_suffixed => u128,
1001 usize_suffixed => usize,
1003 i16_suffixed => i16,
1004 i32_suffixed => i32,
1005 i64_suffixed => i64,
1006 i128_suffixed => i128,
1007 isize_suffixed => isize,
1010 unsuffixed_int_literals! {
1011 u8_unsuffixed => u8,
1012 u16_unsuffixed => u16,
1013 u32_unsuffixed => u32,
1014 u64_unsuffixed => u64,
1015 u128_unsuffixed => u128,
1016 usize_unsuffixed => usize,
1017 i8_unsuffixed => i8,
1018 i16_unsuffixed => i16,
1019 i32_unsuffixed => i32,
1020 i64_unsuffixed => i64,
1021 i128_unsuffixed => i128,
1022 isize_unsuffixed => isize,
1025 /// Creates a new unsuffixed floating-point literal.
1027 /// This constructor is similar to those like `Literal::i8_unsuffixed` where
1028 /// the float's value is emitted directly into the token but no suffix is
1029 /// used, so it may be inferred to be a `f64` later in the compiler.
1030 /// Literals created from negative numbers may not survive rountrips through
1031 /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal).
1035 /// This function requires that the specified float is finite, for
1036 /// example if it is infinity or NaN this function will panic.
1037 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
1038 pub fn f32_unsuffixed(n: f32) -> Literal {
1040 panic!("Invalid float literal {}", n);
1043 lit: token::Lit::Float(Symbol::intern(&n.to_string())),
1045 span: Span::call_site(),
1049 /// Creates a new suffixed floating-point literal.
1051 /// This constructor will create a literal like `1.0f32` where the value
1052 /// specified is the preceding part of the token and `f32` is the suffix of
1053 /// the token. This token will always be inferred to be an `f32` in the
1055 /// Literals created from negative numbers may not survive rountrips through
1056 /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal).
1060 /// This function requires that the specified float is finite, for
1061 /// example if it is infinity or NaN this function will panic.
1062 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
1063 pub fn f32_suffixed(n: f32) -> Literal {
1065 panic!("Invalid float literal {}", n);
1068 lit: token::Lit::Float(Symbol::intern(&n.to_string())),
1069 suffix: Some(Symbol::intern("f32")),
1070 span: Span::call_site(),
1074 /// Creates a new unsuffixed floating-point literal.
1076 /// This constructor is similar to those like `Literal::i8_unsuffixed` where
1077 /// the float's value is emitted directly into the token but no suffix is
1078 /// used, so it may be inferred to be a `f64` later in the compiler.
1079 /// Literals created from negative numbers may not survive rountrips through
1080 /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal).
1084 /// This function requires that the specified float is finite, for
1085 /// example if it is infinity or NaN this function will panic.
1086 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
1087 pub fn f64_unsuffixed(n: f64) -> Literal {
1089 panic!("Invalid float literal {}", n);
1092 lit: token::Lit::Float(Symbol::intern(&n.to_string())),
1094 span: Span::call_site(),
1098 /// Creates a new suffixed floating-point literal.
1100 /// This constructor will create a literal like `1.0f64` where the value
1101 /// specified is the preceding part of the token and `f64` is the suffix of
1102 /// the token. This token will always be inferred to be an `f64` in the
1104 /// Literals created from negative numbers may not survive rountrips through
1105 /// `TokenStream` or strings and may be broken into two tokens (`-` and positive literal).
1109 /// This function requires that the specified float is finite, for
1110 /// example if it is infinity or NaN this function will panic.
1111 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
1112 pub fn f64_suffixed(n: f64) -> Literal {
1114 panic!("Invalid float literal {}", n);
1117 lit: token::Lit::Float(Symbol::intern(&n.to_string())),
1118 suffix: Some(Symbol::intern("f64")),
1119 span: Span::call_site(),
1124 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
1125 pub fn string(string: &str) -> Literal {
1126 let mut escaped = String::new();
1127 for ch in string.chars() {
1128 escaped.extend(ch.escape_debug());
1131 lit: token::Lit::Str_(Symbol::intern(&escaped)),
1133 span: Span::call_site(),
1137 /// Character literal.
1138 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
1139 pub fn character(ch: char) -> Literal {
1140 let mut escaped = String::new();
1141 escaped.extend(ch.escape_unicode());
1143 lit: token::Lit::Char(Symbol::intern(&escaped)),
1145 span: Span::call_site(),
1149 /// Byte string literal.
1150 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
1151 pub fn byte_string(bytes: &[u8]) -> Literal {
1152 let string = bytes.iter().cloned().flat_map(ascii::escape_default)
1153 .map(Into::<char>::into).collect::<String>();
1155 lit: token::Lit::ByteStr(Symbol::intern(&string)),
1157 span: Span::call_site(),
1161 /// Returns the span encompassing this literal.
1162 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
1163 pub fn span(&self) -> Span {
1167 /// Configures the span associated for this literal.
1168 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
1169 pub fn set_span(&mut self, span: Span) {
1173 /// Returns a `Span` that is a subset of `self.span()` containing only the
1174 /// source bytes in range `range`. Returns `None` if the would-be trimmed
1175 /// span is outside the bounds of `self`.
1176 // FIXME(SergioBenitez): check that the byte range starts and ends at a
1177 // UTF-8 boundary of the source. otherwise, it's likely that a panic will
1178 // occur elsewhere when the source text is printed.
1179 // FIXME(SergioBenitez): there is no way for the user to know what
1180 // `self.span()` actually maps to, so this method can currently only be
1181 // called blindly. For example, `to_string()` for the character 'c' returns
1182 // "'\u{63}'"; there is no way for the user to know whether the source text
1183 // was 'c' or whether it was '\u{63}'.
1184 #[unstable(feature = "proc_macro_span", issue = "54725")]
1185 pub fn subspan<R: RangeBounds<usize>>(&self, range: R) -> Option<Span> {
1186 let inner = self.span().0;
1187 let length = inner.hi().to_usize() - inner.lo().to_usize();
1189 let start = match range.start_bound() {
1190 Bound::Included(&lo) => lo,
1191 Bound::Excluded(&lo) => lo + 1,
1192 Bound::Unbounded => 0,
1195 let end = match range.end_bound() {
1196 Bound::Included(&hi) => hi + 1,
1197 Bound::Excluded(&hi) => hi,
1198 Bound::Unbounded => length,
1201 // Bounds check the values, preventing addition overflow and OOB spans.
1202 if start > u32::max_value() as usize
1203 || end > u32::max_value() as usize
1204 || (u32::max_value() - start as u32) < inner.lo().to_u32()
1205 || (u32::max_value() - end as u32) < inner.lo().to_u32()
1212 let new_lo = inner.lo() + BytePos::from_usize(start);
1213 let new_hi = inner.lo() + BytePos::from_usize(end);
1214 Some(Span(inner.with_lo(new_lo).with_hi(new_hi)))
1218 /// Prints the literal as a string that should be losslessly convertible
1219 /// back into the same literal (except for possible rounding for floating point literals).
1220 #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
1221 impl fmt::Display for Literal {
1222 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1223 TokenStream::from(TokenTree::from(self.clone())).fmt(f)
1227 /// Permanently unstable internal implementation details of this crate. This
1228 /// should not be used.
1230 /// These methods are used by the rest of the compiler to generate instances of
1231 /// `TokenStream` to hand to macro definitions, as well as consume the output.
1233 /// Note that this module is also intentionally separate from the rest of the
1234 /// crate. This allows the `#[unstable]` directive below to naturally apply to
1235 /// all of the contents.
1236 #[unstable(feature = "proc_macro_internals", issue = "27812")]
1238 pub mod __internal {
1239 use std::cell::Cell;
1243 use syntax::ext::base::ExtCtxt;
1245 use syntax::parse::{self, ParseSess};
1246 use syntax::parse::token::{self, Token};
1247 use syntax::tokenstream;
1248 use syntax_pos::{BytePos, Loc, DUMMY_SP};
1249 use syntax_pos::hygiene::{SyntaxContext, Transparency};
1251 use super::{TokenStream, LexError, Span};
1253 pub fn lookup_char_pos(pos: BytePos) -> Loc {
1254 with_sess(|sess, _| sess.source_map().lookup_char_pos(pos))
1257 pub fn new_token_stream(item: P<ast::Item>) -> TokenStream {
1258 let token = Token::interpolated(token::NtItem(item));
1259 TokenStream(tokenstream::TokenTree::Token(DUMMY_SP, token).into())
1262 pub fn token_stream_wrap(inner: tokenstream::TokenStream) -> TokenStream {
1266 pub fn token_stream_parse_items(stream: TokenStream) -> Result<Vec<P<ast::Item>>, LexError> {
1267 with_sess(move |sess, _| {
1268 let mut parser = parse::stream_to_parser(sess, stream.0);
1269 let mut items = Vec::new();
1271 while let Some(item) = try!(parser.parse_item().map_err(super::parse_to_lex_err)) {
1279 pub fn token_stream_inner(stream: TokenStream) -> tokenstream::TokenStream {
1283 pub trait Registry {
1284 fn register_custom_derive(&mut self,
1286 expand: fn(TokenStream) -> TokenStream,
1287 attributes: &[&'static str]);
1289 fn register_attr_proc_macro(&mut self,
1291 expand: fn(TokenStream, TokenStream) -> TokenStream);
1293 fn register_bang_proc_macro(&mut self,
1295 expand: fn(TokenStream) -> TokenStream);
1298 #[derive(Clone, Copy)]
1299 pub struct ProcMacroData {
1301 pub call_site: Span,
1304 #[derive(Clone, Copy)]
1305 struct ProcMacroSess {
1306 parse_sess: *const ParseSess,
1307 data: ProcMacroData,
1310 // Emulate scoped_thread_local!() here essentially
1312 static CURRENT_SESS: Cell<ProcMacroSess> = Cell::new(ProcMacroSess {
1313 parse_sess: ptr::null(),
1314 data: ProcMacroData { def_site: Span(DUMMY_SP), call_site: Span(DUMMY_SP) },
1318 pub fn set_sess<F, R>(cx: &ExtCtxt, f: F) -> R
1319 where F: FnOnce() -> R
1321 struct Reset { prev: ProcMacroSess }
1323 impl Drop for Reset {
1324 fn drop(&mut self) {
1325 CURRENT_SESS.with(|p| p.set(self.prev));
1329 CURRENT_SESS.with(|p| {
1330 let _reset = Reset { prev: p.get() };
1332 // No way to determine def location for a proc macro right now, so use call location.
1333 let location = cx.current_expansion.mark.expn_info().unwrap().call_site;
1334 let to_span = |transparency| Span(location.with_ctxt(
1335 SyntaxContext::empty().apply_mark_with_transparency(cx.current_expansion.mark,
1338 p.set(ProcMacroSess {
1339 parse_sess: cx.parse_sess,
1340 data: ProcMacroData {
1341 def_site: to_span(Transparency::Opaque),
1342 call_site: to_span(Transparency::Transparent),
1349 pub fn in_sess() -> bool
1351 !CURRENT_SESS.with(|sess| sess.get()).parse_sess.is_null()
1354 pub fn with_sess<F, R>(f: F) -> R
1355 where F: FnOnce(&ParseSess, &ProcMacroData) -> R
1357 let sess = CURRENT_SESS.with(|sess| sess.get());
1358 if sess.parse_sess.is_null() {
1359 panic!("procedural macro API is used outside of a procedural macro");
1361 f(unsafe { &*sess.parse_sess }, &sess.data)
1365 fn parse_to_lex_err(mut err: DiagnosticBuilder) -> LexError {
1367 LexError { _inner: () }