1 // Copyright 2016 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! A support library for macro authors when defining new macros.
13 //! This library, provided by the standard distribution, provides the types
14 //! consumed in the interfaces of procedurally defined macro definitions.
15 //! Currently the primary use of this crate is to provide the ability to define
16 //! new custom derive modes through `#[proc_macro_derive]`.
18 //! Note that this crate is intentionally very bare-bones currently. The main
19 //! type, `TokenStream`, only supports `fmt::Display` and `FromStr`
20 //! implementations, indicating that it can only go to and come from a string.
21 //! This functionality is intended to be expanded over time as more surface
22 //! area for macro authors is stabilized.
24 //! See [the book](../book/first-edition/procedural-macros.html) for more.
26 #![stable(feature = "proc_macro_lib", since = "1.15.0")]
27 #![deny(missing_docs)]
28 #![doc(html_logo_url = "https://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png",
29 html_favicon_url = "https://doc.rust-lang.org/favicon.ico",
30 html_root_url = "https://doc.rust-lang.org/nightly/",
31 html_playground_url = "https://play.rust-lang.org/",
32 issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/",
33 test(no_crate_inject, attr(deny(warnings))),
34 test(attr(allow(dead_code, deprecated, unused_variables, unused_mut))))]
36 #![feature(rustc_private)]
37 #![feature(staged_api)]
38 #![feature(lang_items)]
39 #![feature(optin_builtin_traits)]
42 extern crate syntax_pos;
43 extern crate rustc_errors;
44 extern crate rustc_data_structures;
48 #[unstable(feature = "proc_macro", issue = "38356")]
49 pub use diagnostic::{Diagnostic, Level};
51 use std::{ascii, fmt, iter};
52 use rustc_data_structures::sync::Lrc;
53 use std::str::FromStr;
56 use syntax::errors::DiagnosticBuilder;
57 use syntax::parse::{self, token};
58 use syntax::symbol::Symbol;
59 use syntax::tokenstream;
60 use syntax::parse::lexer::comments;
61 use syntax_pos::{FileMap, Pos, SyntaxContext, FileName};
62 use syntax_pos::hygiene::Mark;
64 /// The main type provided by this crate, representing an abstract stream of
67 /// This is both the input and output of `#[proc_macro_derive]` definitions.
68 /// Currently it's required to be a list of valid Rust items, but this
69 /// restriction may be lifted in the future.
71 /// The API of this type is intentionally bare-bones, but it'll be expanded over
73 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
75 pub struct TokenStream(tokenstream::TokenStream);
77 /// Error returned from `TokenStream::from_str`.
78 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
85 /// Returns an empty `TokenStream`.
86 #[unstable(feature = "proc_macro", issue = "38356")]
87 pub fn empty() -> TokenStream {
88 TokenStream(tokenstream::TokenStream::empty())
91 /// Checks if this `TokenStream` is empty.
92 #[unstable(feature = "proc_macro", issue = "38356")]
93 pub fn is_empty(&self) -> bool {
98 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
99 impl FromStr for TokenStream {
102 fn from_str(src: &str) -> Result<TokenStream, LexError> {
103 __internal::with_sess(|(sess, mark)| {
104 let src = src.to_string();
105 let name = FileName::ProcMacroSourceCode;
106 let expn_info = mark.expn_info().unwrap();
107 let call_site = expn_info.call_site;
108 // notify the expansion info that it is unhygienic
109 let mark = Mark::fresh(mark);
110 mark.set_expn_info(expn_info);
111 let span = call_site.with_ctxt(SyntaxContext::empty().apply_mark(mark));
112 let stream = parse::parse_stream_from_source_str(name, src, sess, Some(span));
113 Ok(__internal::token_stream_wrap(stream))
118 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
119 impl fmt::Display for TokenStream {
120 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
125 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
126 impl fmt::Debug for TokenStream {
127 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
128 f.write_str("TokenStream ")?;
129 f.debug_list().entries(self.clone()).finish()
133 #[unstable(feature = "proc_macro", issue = "38356")]
134 impl From<TokenTree> for TokenStream {
135 fn from(tree: TokenTree) -> TokenStream {
136 TokenStream(tree.to_internal())
140 #[unstable(feature = "proc_macro", issue = "38356")]
141 impl iter::FromIterator<TokenTree> for TokenStream {
142 fn from_iter<I: IntoIterator<Item = TokenTree>>(trees: I) -> Self {
143 trees.into_iter().map(TokenStream::from).collect()
147 #[unstable(feature = "proc_macro", issue = "38356")]
148 impl iter::FromIterator<TokenStream> for TokenStream {
149 fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self {
150 let mut builder = tokenstream::TokenStreamBuilder::new();
151 for stream in streams {
152 builder.push(stream.0);
154 TokenStream(builder.build())
158 /// Implementation details for the `TokenTree` type, such as iterators.
159 #[unstable(feature = "proc_macro", issue = "38356")]
160 pub mod token_stream {
161 use syntax::tokenstream;
162 use syntax_pos::DUMMY_SP;
164 use {TokenTree, TokenStream, Delimiter};
166 /// An iterator over `TokenTree`s.
168 #[unstable(feature = "proc_macro", issue = "38356")]
169 pub struct IntoIter {
170 cursor: tokenstream::Cursor,
171 stack: Vec<TokenTree>,
174 #[unstable(feature = "proc_macro", issue = "38356")]
175 impl Iterator for IntoIter {
176 type Item = TokenTree;
178 fn next(&mut self) -> Option<TokenTree> {
180 let tree = self.stack.pop().or_else(|| {
181 let next = self.cursor.next_as_stream()?;
182 Some(TokenTree::from_internal(next, &mut self.stack))
184 if tree.span().0 == DUMMY_SP {
185 if let TokenTree::Group(ref group) = tree {
186 if group.delimiter() == Delimiter::None {
187 self.cursor.insert(group.stream.clone().0);
197 #[unstable(feature = "proc_macro", issue = "38356")]
198 impl IntoIterator for TokenStream {
199 type Item = TokenTree;
200 type IntoIter = IntoIter;
202 fn into_iter(self) -> IntoIter {
203 IntoIter { cursor: self.0.trees(), stack: Vec::new() }
208 /// `quote!(..)` accepts arbitrary tokens and expands into a `TokenStream` describing the input.
209 /// For example, `quote!(a + b)` will produce a expression, that, when evaluated, constructs
210 /// the `TokenStream` `[Word("a"), Op('+', Alone), Word("b")]`.
212 /// Unquoting is done with `$`, and works by taking the single next ident as the unquoted term.
213 /// To quote `$` itself, use `$$`.
214 #[unstable(feature = "proc_macro", issue = "38356")]
216 macro_rules! quote { () => {} }
218 #[unstable(feature = "proc_macro_internals", issue = "27812")]
222 /// Quote a `Span` into a `TokenStream`.
223 /// This is needed to implement a custom quoter.
224 #[unstable(feature = "proc_macro", issue = "38356")]
225 pub fn quote_span(span: Span) -> TokenStream {
226 quote::Quote::quote(span)
229 /// A region of source code, along with macro expansion information.
230 #[unstable(feature = "proc_macro", issue = "38356")]
231 #[derive(Copy, Clone)]
232 pub struct Span(syntax_pos::Span);
234 macro_rules! diagnostic_method {
235 ($name:ident, $level:expr) => (
236 /// Create a new `Diagnostic` with the given `message` at the span
238 #[unstable(feature = "proc_macro", issue = "38356")]
239 pub fn $name<T: Into<String>>(self, message: T) -> Diagnostic {
240 Diagnostic::spanned(self, $level, message)
246 /// A span that resolves at the macro definition site.
247 #[unstable(feature = "proc_macro", issue = "38356")]
248 pub fn def_site() -> Span {
249 ::__internal::with_sess(|(_, mark)| {
250 let call_site = mark.expn_info().unwrap().call_site;
251 Span(call_site.with_ctxt(SyntaxContext::empty().apply_mark(mark)))
255 /// The span of the invocation of the current procedural macro.
256 #[unstable(feature = "proc_macro", issue = "38356")]
257 pub fn call_site() -> Span {
258 ::__internal::with_sess(|(_, mark)| Span(mark.expn_info().unwrap().call_site))
261 /// The original source file into which this span points.
262 #[unstable(feature = "proc_macro", issue = "38356")]
263 pub fn source_file(&self) -> SourceFile {
265 filemap: __internal::lookup_char_pos(self.0.lo()).file,
269 /// The `Span` for the tokens in the previous macro expansion from which
270 /// `self` was generated from, if any.
271 #[unstable(feature = "proc_macro", issue = "38356")]
272 pub fn parent(&self) -> Option<Span> {
273 self.0.parent().map(Span)
276 /// The span for the origin source code that `self` was generated from. If
277 /// this `Span` wasn't generated from other macro expansions then the return
278 /// value is the same as `*self`.
279 #[unstable(feature = "proc_macro", issue = "38356")]
280 pub fn source(&self) -> Span {
281 Span(self.0.source_callsite())
284 /// Get the starting line/column in the source file for this span.
285 #[unstable(feature = "proc_macro", issue = "38356")]
286 pub fn start(&self) -> LineColumn {
287 let loc = __internal::lookup_char_pos(self.0.lo());
290 column: loc.col.to_usize()
294 /// Get the ending line/column in the source file for this span.
295 #[unstable(feature = "proc_macro", issue = "38356")]
296 pub fn end(&self) -> LineColumn {
297 let loc = __internal::lookup_char_pos(self.0.hi());
300 column: loc.col.to_usize()
304 /// Create a new span encompassing `self` and `other`.
306 /// Returns `None` if `self` and `other` are from different files.
307 #[unstable(feature = "proc_macro", issue = "38356")]
308 pub fn join(&self, other: Span) -> Option<Span> {
309 let self_loc = __internal::lookup_char_pos(self.0.lo());
310 let other_loc = __internal::lookup_char_pos(other.0.lo());
312 if self_loc.file.name != other_loc.file.name { return None }
314 Some(Span(self.0.to(other.0)))
317 /// Creates a new span with the same line/column information as `self` but
318 /// that resolves symbols as though it were at `other`.
319 #[unstable(feature = "proc_macro", issue = "38356")]
320 pub fn resolved_at(&self, other: Span) -> Span {
321 Span(self.0.with_ctxt(other.0.ctxt()))
324 /// Creates a new span with the same name resolution behavior as `self` but
325 /// with the line/column information of `other`.
326 #[unstable(feature = "proc_macro", issue = "38356")]
327 pub fn located_at(&self, other: Span) -> Span {
328 other.resolved_at(*self)
331 /// Compares to spans to see if they're equal.
332 #[unstable(feature = "proc_macro", issue = "38356")]
333 pub fn eq(&self, other: &Span) -> bool {
337 diagnostic_method!(error, Level::Error);
338 diagnostic_method!(warning, Level::Warning);
339 diagnostic_method!(note, Level::Note);
340 diagnostic_method!(help, Level::Help);
343 #[unstable(feature = "proc_macro", issue = "38356")]
344 impl fmt::Debug for Span {
345 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
346 write!(f, "{:?} bytes({}..{})",
353 /// A line-column pair representing the start or end of a `Span`.
354 #[unstable(feature = "proc_macro", issue = "38356")]
355 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
356 pub struct LineColumn {
357 /// The 1-indexed line in the source file on which the span starts or ends (inclusive).
358 #[unstable(feature = "proc_macro", issue = "38356")]
360 /// The 0-indexed column (in UTF-8 characters) in the source file on which
361 /// the span starts or ends (inclusive).
362 #[unstable(feature = "proc_macro", issue = "38356")]
366 /// The source file of a given `Span`.
367 #[unstable(feature = "proc_macro", issue = "38356")]
369 pub struct SourceFile {
370 filemap: Lrc<FileMap>,
373 #[unstable(feature = "proc_macro", issue = "38356")]
374 impl !Send for SourceFile {}
375 #[unstable(feature = "proc_macro", issue = "38356")]
376 impl !Sync for SourceFile {}
379 /// Get the path to this source file.
382 /// If the code span associated with this `SourceFile` was generated by an external macro, this
383 /// may not be an actual path on the filesystem. Use [`is_real`] to check.
385 /// Also note that even if `is_real` returns `true`, if `--remap-path-prefix` was passed on
386 /// the command line, the path as given may not actually be valid.
388 /// [`is_real`]: #method.is_real
389 # [unstable(feature = "proc_macro", issue = "38356")]
390 pub fn path(&self) -> &FileName {
394 /// Returns `true` if this source file is a real source file, and not generated by an external
395 /// macro's expansion.
396 # [unstable(feature = "proc_macro", issue = "38356")]
397 pub fn is_real(&self) -> bool {
398 // This is a hack until intercrate spans are implemented and we can have real source files
399 // for spans generated in external macros.
400 // https://github.com/rust-lang/rust/pull/43604#issuecomment-333334368
401 self.filemap.is_real_file()
405 #[unstable(feature = "proc_macro", issue = "38356")]
406 impl AsRef<FileName> for SourceFile {
407 fn as_ref(&self) -> &FileName {
412 #[unstable(feature = "proc_macro", issue = "38356")]
413 impl fmt::Debug for SourceFile {
414 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
415 f.debug_struct("SourceFile")
416 .field("path", self.path())
417 .field("is_real", &self.is_real())
422 #[unstable(feature = "proc_macro", issue = "38356")]
423 impl PartialEq for SourceFile {
424 fn eq(&self, other: &Self) -> bool {
425 Lrc::ptr_eq(&self.filemap, &other.filemap)
429 #[unstable(feature = "proc_macro", issue = "38356")]
430 impl Eq for SourceFile {}
432 #[unstable(feature = "proc_macro", issue = "38356")]
433 impl PartialEq<FileName> for SourceFile {
434 fn eq(&self, other: &FileName) -> bool {
435 self.as_ref() == other
439 /// A single token or a delimited sequence of token trees (e.g. `[1, (), ..]`).
440 #[unstable(feature = "proc_macro", issue = "38356")]
443 /// A delimited tokenstream
445 /// A unicode identifier
447 /// A punctuation character (`+`, `,`, `$`, etc.).
449 /// A literal character (`'a'`), string (`"hello"`), number (`2.3`), etc.
454 /// Returns the span of this token, accessing the `span` method of each of
455 /// the internal tokens.
456 #[unstable(feature = "proc_macro", issue = "38356")]
457 pub fn span(&self) -> Span {
459 TokenTree::Group(ref t) => t.span(),
460 TokenTree::Term(ref t) => t.span(),
461 TokenTree::Op(ref t) => t.span(),
462 TokenTree::Literal(ref t) => t.span(),
466 /// Configures the span for *only this token*.
468 /// Note that if this token is a `Group` then this method will not configure
469 /// the span of each of the internal tokens, this will simply delegate to
470 /// the `set_span` method of each variant.
471 #[unstable(feature = "proc_macro", issue = "38356")]
472 pub fn set_span(&mut self, span: Span) {
474 TokenTree::Group(ref mut t) => t.set_span(span),
475 TokenTree::Term(ref mut t) => t.set_span(span),
476 TokenTree::Op(ref mut t) => t.set_span(span),
477 TokenTree::Literal(ref mut t) => t.set_span(span),
482 #[unstable(feature = "proc_macro", issue = "38356")]
483 impl fmt::Debug for TokenTree {
484 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
485 // Each of these has the name in the struct type in the derived debug,
486 // so don't bother with an extra layer of indirection
488 TokenTree::Group(ref tt) => tt.fmt(f),
489 TokenTree::Term(ref tt) => tt.fmt(f),
490 TokenTree::Op(ref tt) => tt.fmt(f),
491 TokenTree::Literal(ref tt) => tt.fmt(f),
496 #[unstable(feature = "proc_macro", issue = "38356")]
497 impl From<Group> for TokenTree {
498 fn from(g: Group) -> TokenTree {
503 #[unstable(feature = "proc_macro", issue = "38356")]
504 impl From<Term> for TokenTree {
505 fn from(g: Term) -> TokenTree {
510 #[unstable(feature = "proc_macro", issue = "38356")]
511 impl From<Op> for TokenTree {
512 fn from(g: Op) -> TokenTree {
517 #[unstable(feature = "proc_macro", issue = "38356")]
518 impl From<Literal> for TokenTree {
519 fn from(g: Literal) -> TokenTree {
520 TokenTree::Literal(g)
524 #[unstable(feature = "proc_macro", issue = "38356")]
525 impl fmt::Display for TokenTree {
526 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
528 TokenTree::Group(ref t) => t.fmt(f),
529 TokenTree::Term(ref t) => t.fmt(f),
530 TokenTree::Op(ref t) => t.fmt(f),
531 TokenTree::Literal(ref t) => t.fmt(f),
536 /// A delimited token stream
538 /// A `Group` internally contains a `TokenStream` which is delimited by a
539 /// `Delimiter`. Groups represent multiple tokens internally and have a `Span`
540 /// for the entire stream.
541 #[derive(Clone, Debug)]
542 #[unstable(feature = "proc_macro", issue = "38356")]
544 delimiter: Delimiter,
549 /// Describes how a sequence of token trees is delimited.
550 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
551 #[unstable(feature = "proc_macro", issue = "38356")]
559 /// An implicit delimiter, e.g. `$var`, where $var is `...`.
564 /// Creates a new `group` with the given delimiter and token stream.
566 /// This constructor will set the span for this group to
567 /// `Span::call_site()`. To change the span you can use the `set_span`
569 #[unstable(feature = "proc_macro", issue = "38356")]
570 pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group {
572 delimiter: delimiter,
574 span: Span::call_site(),
578 /// Returns the delimiter of this `Group`
579 #[unstable(feature = "proc_macro", issue = "38356")]
580 pub fn delimiter(&self) -> Delimiter {
584 /// Returns the `TokenStream` of tokens that are delimited in this `Group`.
586 /// Note that the returned token stream does not include the delimiter
588 #[unstable(feature = "proc_macro", issue = "38356")]
589 pub fn stream(&self) -> TokenStream {
593 /// Returns the span for the delimiters of this token stream, spanning the
595 #[unstable(feature = "proc_macro", issue = "38356")]
596 pub fn span(&self) -> Span {
600 /// Configures the span for this `Group`'s delimiters, but not its internal
603 /// This method will **not** set the span of all the internal tokens spanned
604 /// by this group, but rather it will only set the span of the delimiter
605 /// tokens at the level of the `Group`.
606 #[unstable(feature = "proc_macro", issue = "38356")]
607 pub fn set_span(&mut self, span: Span) {
612 #[unstable(feature = "proc_macro", issue = "38356")]
613 impl fmt::Display for Group {
614 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
615 TokenStream::from(TokenTree::from(self.clone())).fmt(f)
619 /// An `Op` is an operator like `+` or `-`, and only represents one character.
621 /// Operators like `+=` are represented as two instance of `Op` with different
622 /// forms of `Spacing` returned.
623 #[unstable(feature = "proc_macro", issue = "38356")]
624 #[derive(Copy, Clone, Debug)]
631 /// Whether an `Op` is either followed immediately by another `Op` or followed by whitespace.
632 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
633 #[unstable(feature = "proc_macro", issue = "38356")]
635 /// e.g. `+` is `Alone` in `+ =`.
637 /// e.g. `+` is `Joint` in `+=`.
642 /// Creates a new `Op` from the given character and spacing.
644 /// The returned `Op` will have the default span of `Span::call_site()`
645 /// which can be further configured with the `set_span` method below.
646 #[unstable(feature = "proc_macro", issue = "38356")]
647 pub fn new(op: char, spacing: Spacing) -> Op {
651 span: Span::call_site(),
655 /// Returns the character this operation represents, for example `'+'`
656 #[unstable(feature = "proc_macro", issue = "38356")]
657 pub fn op(&self) -> char {
661 /// Returns the spacing of this operator, indicating whether it's a joint
662 /// operator with more operators coming next in the token stream or an
663 /// `Alone` meaning that the operator has ended.
664 #[unstable(feature = "proc_macro", issue = "38356")]
665 pub fn spacing(&self) -> Spacing {
669 /// Returns the span for this operator character
670 #[unstable(feature = "proc_macro", issue = "38356")]
671 pub fn span(&self) -> Span {
675 /// Configure the span for this operator's character
676 #[unstable(feature = "proc_macro", issue = "38356")]
677 pub fn set_span(&mut self, span: Span) {
682 #[unstable(feature = "proc_macro", issue = "38356")]
683 impl fmt::Display for Op {
684 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
685 TokenStream::from(TokenTree::from(self.clone())).fmt(f)
689 /// An interned string.
690 #[derive(Copy, Clone, Debug)]
691 #[unstable(feature = "proc_macro", issue = "38356")]
698 /// Creates a new `Term` with the given `string` as well as the specified
701 /// Note that `span`, currently in rustc, configures the hygiene information
702 /// for this identifier. As of this time `Span::call_site()` explicitly
703 /// opts-in to **non-hygienic** information (aka copy/pasted code) while
704 /// spans like `Span::def_site()` will opt-in to hygienic information,
705 /// meaning that code at the call site of the macro can't access this
708 /// Due to the current importance of hygiene this constructor, unlike other
709 /// tokens, requires a `Span` to be specified at construction.
710 #[unstable(feature = "proc_macro", issue = "38356")]
711 pub fn new(string: &str, span: Span) -> Term {
713 sym: Symbol::intern(string),
718 /// Get a reference to the interned string.
719 #[unstable(feature = "proc_macro", issue = "38356")]
720 pub fn as_str(&self) -> &str {
721 unsafe { &*(&*self.sym.as_str() as *const str) }
724 /// Returns the span of this `Term`, encompassing the entire string returned
726 #[unstable(feature = "proc_macro", issue = "38356")]
727 pub fn span(&self) -> Span {
731 /// Configures the span of this `Term`, possibly changing hygiene
733 #[unstable(feature = "proc_macro", issue = "38356")]
734 pub fn set_span(&mut self, span: Span) {
739 #[unstable(feature = "proc_macro", issue = "38356")]
740 impl fmt::Display for Term {
741 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
746 /// A literal character (`'a'`), string (`"hello"`), a number (`2.3`), etc.
747 #[derive(Clone, Debug)]
748 #[unstable(feature = "proc_macro", issue = "38356")]
751 suffix: Option<ast::Name>,
755 macro_rules! suffixed_int_literals {
756 ($($name:ident => $kind:ident,)*) => ($(
757 /// Creates a new suffixed integer literal with the specified value.
759 /// This function will create an integer like `1u32` where the integer
760 /// value specified is the first part of the token and the integral is
761 /// also suffixed at the end.
763 /// Literals created through this method have the `Span::call_site()`
764 /// span by default, which can be configured with the `set_span` method
766 #[unstable(feature = "proc_macro", issue = "38356")]
767 pub fn $name(n: $kind) -> Literal {
769 lit: token::Lit::Integer(Symbol::intern(&n.to_string())),
770 suffix: Some(Symbol::intern(stringify!($kind))),
771 span: Span::call_site(),
777 macro_rules! unsuffixed_int_literals {
778 ($($name:ident => $kind:ident,)*) => ($(
779 /// Creates a new unsuffixed integer literal with the specified value.
781 /// This function will create an integer like `1` where the integer
782 /// value specified is the first part of the token. No suffix is
783 /// specified on this token, meaning that invocations like
784 /// `Literal::i8_unsuffixed(1)` are equivalent to
785 /// `Literal::u32_unsuffixed(1)`.
787 /// Literals created through this method have the `Span::call_site()`
788 /// span by default, which can be configured with the `set_span` method
790 #[unstable(feature = "proc_macro", issue = "38356")]
791 pub fn $name(n: $kind) -> Literal {
793 lit: token::Lit::Integer(Symbol::intern(&n.to_string())),
795 span: Span::call_site(),
802 suffixed_int_literals! {
807 u128_suffixed => u128,
808 usize_suffixed => usize,
813 i128_suffixed => i128,
814 isize_suffixed => isize,
817 unsuffixed_int_literals! {
819 u16_unsuffixed => u16,
820 u32_unsuffixed => u32,
821 u64_unsuffixed => u64,
822 u128_unsuffixed => u128,
823 usize_unsuffixed => usize,
825 i16_unsuffixed => i16,
826 i32_unsuffixed => i32,
827 i64_unsuffixed => i64,
828 i128_unsuffixed => i128,
829 isize_unsuffixed => isize,
832 /// Creates a new unsuffixed floating-point literal.
834 /// This constructor is similar to those like `Literal::i8_unsuffixed` where
835 /// the float's value is emitted directly into the token but no suffix is
836 /// used, so it may be inferred to be a `f64` later in the compiler.
840 /// This function requires that the specified float is finite, for
841 /// example if it is infinity or NaN this function will panic.
842 #[unstable(feature = "proc_macro", issue = "38356")]
843 pub fn f32_unsuffixed(n: f32) -> Literal {
845 panic!("Invalid float literal {}", n);
848 lit: token::Lit::Float(Symbol::intern(&n.to_string())),
850 span: Span::call_site(),
854 /// Creates a new suffixed floating-point literal.
856 /// This consturctor will create a literal like `1.0f32` where the value
857 /// specified is the preceding part of the token and `f32` is the suffix of
858 /// the token. This token will always be inferred to be an `f32` in the
863 /// This function requires that the specified float is finite, for
864 /// example if it is infinity or NaN this function will panic.
865 #[unstable(feature = "proc_macro", issue = "38356")]
866 pub fn f32_suffixed(n: f32) -> Literal {
868 panic!("Invalid float literal {}", n);
871 lit: token::Lit::Float(Symbol::intern(&n.to_string())),
872 suffix: Some(Symbol::intern("f32")),
873 span: Span::call_site(),
877 /// Creates a new unsuffixed floating-point literal.
879 /// This constructor is similar to those like `Literal::i8_unsuffixed` where
880 /// the float's value is emitted directly into the token but no suffix is
881 /// used, so it may be inferred to be a `f64` later in the compiler.
885 /// This function requires that the specified float is finite, for
886 /// example if it is infinity or NaN this function will panic.
887 #[unstable(feature = "proc_macro", issue = "38356")]
888 pub fn f64_unsuffixed(n: f64) -> Literal {
890 panic!("Invalid float literal {}", n);
893 lit: token::Lit::Float(Symbol::intern(&n.to_string())),
895 span: Span::call_site(),
899 /// Creates a new suffixed floating-point literal.
901 /// This consturctor will create a literal like `1.0f64` where the value
902 /// specified is the preceding part of the token and `f64` is the suffix of
903 /// the token. This token will always be inferred to be an `f64` in the
908 /// This function requires that the specified float is finite, for
909 /// example if it is infinity or NaN this function will panic.
910 #[unstable(feature = "proc_macro", issue = "38356")]
911 pub fn f64_suffixed(n: f64) -> Literal {
913 panic!("Invalid float literal {}", n);
916 lit: token::Lit::Float(Symbol::intern(&n.to_string())),
917 suffix: Some(Symbol::intern("f64")),
918 span: Span::call_site(),
923 #[unstable(feature = "proc_macro", issue = "38356")]
924 pub fn string(string: &str) -> Literal {
925 let mut escaped = String::new();
926 for ch in string.chars() {
927 escaped.extend(ch.escape_debug());
930 lit: token::Lit::Str_(Symbol::intern(&escaped)),
932 span: Span::call_site(),
936 /// Character literal.
937 #[unstable(feature = "proc_macro", issue = "38356")]
938 pub fn character(ch: char) -> Literal {
939 let mut escaped = String::new();
940 escaped.extend(ch.escape_unicode());
942 lit: token::Lit::Char(Symbol::intern(&escaped)),
944 span: Span::call_site(),
948 /// Byte string literal.
949 #[unstable(feature = "proc_macro", issue = "38356")]
950 pub fn byte_string(bytes: &[u8]) -> Literal {
951 let string = bytes.iter().cloned().flat_map(ascii::escape_default)
952 .map(Into::<char>::into).collect::<String>();
954 lit: token::Lit::ByteStr(Symbol::intern(&string)),
956 span: Span::call_site(),
960 /// Returns the span encompassing this literal.
961 #[unstable(feature = "proc_macro", issue = "38356")]
962 pub fn span(&self) -> Span {
966 /// Configures the span associated for this literal.
967 #[unstable(feature = "proc_macro", issue = "38356")]
968 pub fn set_span(&mut self, span: Span) {
973 #[unstable(feature = "proc_macro", issue = "38356")]
974 impl fmt::Display for Literal {
975 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
976 TokenStream::from(TokenTree::from(self.clone())).fmt(f)
981 fn from_internal(delim: token::DelimToken) -> Delimiter {
983 token::Paren => Delimiter::Parenthesis,
984 token::Brace => Delimiter::Brace,
985 token::Bracket => Delimiter::Bracket,
986 token::NoDelim => Delimiter::None,
990 fn to_internal(self) -> token::DelimToken {
992 Delimiter::Parenthesis => token::Paren,
993 Delimiter::Brace => token::Brace,
994 Delimiter::Bracket => token::Bracket,
995 Delimiter::None => token::NoDelim,
1001 fn from_internal(stream: tokenstream::TokenStream, stack: &mut Vec<TokenTree>)
1003 use syntax::parse::token::*;
1005 let (tree, is_joint) = stream.as_tree();
1006 let (span, token) = match tree {
1007 tokenstream::TokenTree::Token(span, token) => (span, token),
1008 tokenstream::TokenTree::Delimited(span, delimed) => {
1009 let delimiter = Delimiter::from_internal(delimed.delim);
1010 let mut g = Group::new(delimiter, TokenStream(delimed.tts.into()));
1011 g.set_span(Span(span));
1016 let op_kind = if is_joint { Spacing::Joint } else { Spacing::Alone };
1019 let mut x = TokenTree::from($e);
1020 x.set_span(Span(span));
1025 ($a:expr) => (tt!(Op::new($a, op_kind)));
1026 ($a:expr, $b:expr) => ({
1027 stack.push(tt!(Op::new($b, op_kind)));
1028 tt!(Op::new($a, Spacing::Joint))
1030 ($a:expr, $b:expr, $c:expr) => ({
1031 stack.push(tt!(Op::new($c, op_kind)));
1032 stack.push(tt!(Op::new($b, Spacing::Joint)));
1033 tt!(Op::new($a, Spacing::Joint))
1040 Le => op!('<', '='),
1041 EqEq => op!('=', '='),
1042 Ne => op!('!', '='),
1043 Ge => op!('>', '='),
1045 AndAnd => op!('&', '&'),
1046 OrOr => op!('|', '|'),
1049 BinOp(Plus) => op!('+'),
1050 BinOp(Minus) => op!('-'),
1051 BinOp(Star) => op!('*'),
1052 BinOp(Slash) => op!('/'),
1053 BinOp(Percent) => op!('%'),
1054 BinOp(Caret) => op!('^'),
1055 BinOp(And) => op!('&'),
1056 BinOp(Or) => op!('|'),
1057 BinOp(Shl) => op!('<', '<'),
1058 BinOp(Shr) => op!('>', '>'),
1059 BinOpEq(Plus) => op!('+', '='),
1060 BinOpEq(Minus) => op!('-', '='),
1061 BinOpEq(Star) => op!('*', '='),
1062 BinOpEq(Slash) => op!('/', '='),
1063 BinOpEq(Percent) => op!('%', '='),
1064 BinOpEq(Caret) => op!('^', '='),
1065 BinOpEq(And) => op!('&', '='),
1066 BinOpEq(Or) => op!('|', '='),
1067 BinOpEq(Shl) => op!('<', '<', '='),
1068 BinOpEq(Shr) => op!('>', '>', '='),
1071 DotDot => op!('.', '.'),
1072 DotDotDot => op!('.', '.', '.'),
1073 DotDotEq => op!('.', '.', '='),
1077 ModSep => op!(':', ':'),
1078 RArrow => op!('-', '>'),
1079 LArrow => op!('<', '-'),
1080 FatArrow => op!('=', '>'),
1083 Question => op!('?'),
1085 Ident(ident, false) | Lifetime(ident) => {
1086 tt!(Term::new(&ident.name.as_str(), Span(span)))
1088 Ident(ident, true) => {
1089 tt!(Term::new(&format!("r#{}", ident), Span(span)))
1091 Literal(lit, suffix) => tt!(self::Literal { lit, suffix, span: Span(span) }),
1093 let style = comments::doc_comment_style(&c.as_str());
1094 let stripped = comments::strip_doc_comment_decoration(&c.as_str());
1096 tt!(Term::new("doc", Span(span))),
1097 tt!(Op::new('=', Spacing::Alone)),
1098 tt!(self::Literal::string(&stripped)),
1099 ].into_iter().collect();
1100 stack.push(tt!(Group::new(Delimiter::Bracket, stream)));
1101 if style == ast::AttrStyle::Inner {
1102 stack.push(tt!(Op::new('!', Spacing::Alone)));
1104 tt!(Op::new('#', Spacing::Alone))
1107 Interpolated(_) => {
1108 __internal::with_sess(|(sess, _)| {
1109 let tts = token.interpolated_to_tokenstream(sess, span);
1110 tt!(Group::new(Delimiter::None, TokenStream(tts)))
1114 DotEq => op!('.', '='),
1115 OpenDelim(..) | CloseDelim(..) => unreachable!(),
1116 Whitespace | Comment | Shebang(..) | Eof => unreachable!(),
1120 fn to_internal(self) -> tokenstream::TokenStream {
1121 use syntax::parse::token::*;
1122 use syntax::tokenstream::{TokenTree, Delimited};
1124 let (op, kind, span) = match self {
1125 self::TokenTree::Op(tt) => (tt.op(), tt.spacing(), tt.span()),
1126 self::TokenTree::Group(tt) => {
1127 return TokenTree::Delimited(tt.span.0, Delimited {
1128 delim: tt.delimiter.to_internal(),
1129 tts: tt.stream.0.into(),
1132 self::TokenTree::Term(tt) => {
1133 let ident = ast::Ident::new(tt.sym, tt.span.0);
1134 let sym_str = tt.sym.as_str();
1135 let token = if sym_str.starts_with("'") {
1137 } else if sym_str.starts_with("r#") {
1138 let name = Symbol::intern(&sym_str[2..]);
1139 let ident = ast::Ident::new(name, ident.span);
1144 return TokenTree::Token(tt.span.0, token).into();
1146 self::TokenTree::Literal(self::Literal {
1147 lit: Lit::Integer(ref a),
1151 if a.as_str().starts_with("-") =>
1153 let minus = BinOp(BinOpToken::Minus);
1154 let integer = Symbol::intern(&a.as_str()[1..]);
1155 let integer = Literal(Lit::Integer(integer), suffix);
1156 let a = TokenTree::Token(span.0, minus);
1157 let b = TokenTree::Token(span.0, integer);
1158 return vec![a, b].into_iter().collect()
1160 self::TokenTree::Literal(self::Literal {
1161 lit: Lit::Float(ref a),
1165 if a.as_str().starts_with("-") =>
1167 let minus = BinOp(BinOpToken::Minus);
1168 let float = Symbol::intern(&a.as_str()[1..]);
1169 let float = Literal(Lit::Float(float), suffix);
1170 let a = TokenTree::Token(span.0, minus);
1171 let b = TokenTree::Token(span.0, float);
1172 return vec![a, b].into_iter().collect()
1174 self::TokenTree::Literal(tt) => {
1175 let token = Literal(tt.lit, tt.suffix);
1176 return TokenTree::Token(tt.span.0, token).into()
1180 let token = match op {
1187 '-' => BinOp(Minus),
1189 '/' => BinOp(Slash),
1190 '%' => BinOp(Percent),
1191 '^' => BinOp(Caret),
1202 _ => panic!("unsupported character {}", op),
1205 let tree = TokenTree::Token(span.0, token);
1207 Spacing::Alone => tree.into(),
1208 Spacing::Joint => tree.joint(),
1213 /// Permanently unstable internal implementation details of this crate. This
1214 /// should not be used.
1216 /// These methods are used by the rest of the compiler to generate instances of
1217 /// `TokenStream` to hand to macro definitions, as well as consume the output.
1219 /// Note that this module is also intentionally separate from the rest of the
1220 /// crate. This allows the `#[unstable]` directive below to naturally apply to
1221 /// all of the contents.
1222 #[unstable(feature = "proc_macro_internals", issue = "27812")]
1224 pub mod __internal {
1225 pub use quote::{LiteralKind, Quoter, unquote};
1227 use std::cell::Cell;
1230 use syntax::ext::base::ExtCtxt;
1231 use syntax::ext::hygiene::Mark;
1233 use syntax::parse::{self, ParseSess};
1234 use syntax::parse::token::{self, Token};
1235 use syntax::tokenstream;
1236 use syntax_pos::{BytePos, Loc, DUMMY_SP};
1238 use super::{TokenStream, LexError};
1240 pub fn lookup_char_pos(pos: BytePos) -> Loc {
1241 with_sess(|(sess, _)| sess.codemap().lookup_char_pos(pos))
1244 pub fn new_token_stream(item: P<ast::Item>) -> TokenStream {
1245 let token = Token::interpolated(token::NtItem(item));
1246 TokenStream(tokenstream::TokenTree::Token(DUMMY_SP, token).into())
1249 pub fn token_stream_wrap(inner: tokenstream::TokenStream) -> TokenStream {
1253 pub fn token_stream_parse_items(stream: TokenStream) -> Result<Vec<P<ast::Item>>, LexError> {
1254 with_sess(move |(sess, _)| {
1255 let mut parser = parse::stream_to_parser(sess, stream.0);
1256 let mut items = Vec::new();
1258 while let Some(item) = try!(parser.parse_item().map_err(super::parse_to_lex_err)) {
1266 pub fn token_stream_inner(stream: TokenStream) -> tokenstream::TokenStream {
1270 pub trait Registry {
1271 fn register_custom_derive(&mut self,
1273 expand: fn(TokenStream) -> TokenStream,
1274 attributes: &[&'static str]);
1276 fn register_attr_proc_macro(&mut self,
1278 expand: fn(TokenStream, TokenStream) -> TokenStream);
1280 fn register_bang_proc_macro(&mut self,
1282 expand: fn(TokenStream) -> TokenStream);
1285 // Emulate scoped_thread_local!() here essentially
1287 static CURRENT_SESS: Cell<(*const ParseSess, Mark)> =
1288 Cell::new((0 as *const _, Mark::root()));
1291 pub fn set_sess<F, R>(cx: &ExtCtxt, f: F) -> R
1292 where F: FnOnce() -> R
1294 struct Reset { prev: (*const ParseSess, Mark) }
1296 impl Drop for Reset {
1297 fn drop(&mut self) {
1298 CURRENT_SESS.with(|p| p.set(self.prev));
1302 CURRENT_SESS.with(|p| {
1303 let _reset = Reset { prev: p.get() };
1304 p.set((cx.parse_sess, cx.current_expansion.mark));
1309 pub fn in_sess() -> bool
1311 let p = CURRENT_SESS.with(|p| p.get());
1315 pub fn with_sess<F, R>(f: F) -> R
1316 where F: FnOnce((&ParseSess, Mark)) -> R
1318 let p = CURRENT_SESS.with(|p| p.get());
1319 assert!(!p.0.is_null(), "proc_macro::__internal::with_sess() called \
1320 before set_parse_sess()!");
1321 f(unsafe { (&*p.0, p.1) })
1325 fn parse_to_lex_err(mut err: DiagnosticBuilder) -> LexError {
1327 LexError { _inner: () }