1 //! `tt` crate defines a `TokenTree` data structure: this is the interface (both
2 //! input and output) of macros. It closely mirrors `proc_macro` crate's
4 use std::{fmt, panic::RefUnwindSafe};
8 pub use smol_str::SmolStr;
10 /// Represents identity of the token.
12 /// For hygiene purposes, we need to track which expanded tokens originated from
13 /// which source tokens. We do it by assigning an distinct identity to each
14 /// source token and making sure that identities are preserved during macro
16 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
17 pub struct TokenId(pub u32);
20 pub const fn unspecified() -> TokenId {
25 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
30 impl_from!(Leaf, Subtree for TokenTree);
33 pub fn empty() -> Self {
34 TokenTree::Subtree(Subtree::default())
38 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
44 impl_from!(Literal, Punct, Ident for Leaf);
46 #[derive(Clone, PartialEq, Eq, Hash, Default)]
48 pub delimiter: Option<Delimiter>,
49 pub token_trees: Vec<TokenTree>,
52 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
53 pub struct Delimiter {
55 pub kind: DelimiterKind,
58 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
59 pub enum DelimiterKind {
65 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
71 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
78 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
84 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
90 fn print_debug_subtree(f: &mut fmt::Formatter<'_>, subtree: &Subtree, level: usize) -> fmt::Result {
91 let align = std::iter::repeat(" ").take(level).collect::<String>();
93 let aux = match subtree.delimiter.map(|it| (it.kind, it.id.0)) {
94 None => "$".to_string(),
95 Some((DelimiterKind::Parenthesis, id)) => format!("() {}", id),
96 Some((DelimiterKind::Brace, id)) => format!("{{}} {}", id),
97 Some((DelimiterKind::Bracket, id)) => format!("[] {}", id),
100 if subtree.token_trees.is_empty() {
101 write!(f, "{}SUBTREE {}", align, aux)?;
103 writeln!(f, "{}SUBTREE {}", align, aux)?;
104 for (idx, child) in subtree.token_trees.iter().enumerate() {
105 print_debug_token(f, child, level + 1)?;
106 if idx != subtree.token_trees.len() - 1 {
115 fn print_debug_token(f: &mut fmt::Formatter<'_>, tkn: &TokenTree, level: usize) -> fmt::Result {
116 let align = std::iter::repeat(" ").take(level).collect::<String>();
119 TokenTree::Leaf(leaf) => match leaf {
120 Leaf::Literal(lit) => write!(f, "{}LITERAL {} {}", align, lit.text, lit.id.0)?,
121 Leaf::Punct(punct) => write!(
123 "{}PUNCH {} [{}] {}",
126 if punct.spacing == Spacing::Alone { "alone" } else { "joint" },
129 Leaf::Ident(ident) => write!(f, "{}IDENT {} {}", align, ident.text, ident.id.0)?,
131 TokenTree::Subtree(subtree) => {
132 print_debug_subtree(f, subtree, level)?;
139 impl fmt::Debug for Subtree {
140 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
141 print_debug_subtree(f, self, 0)
145 impl fmt::Display for TokenTree {
146 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
148 TokenTree::Leaf(it) => fmt::Display::fmt(it, f),
149 TokenTree::Subtree(it) => fmt::Display::fmt(it, f),
154 impl fmt::Display for Subtree {
155 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
156 let (l, r) = match self.delimiter_kind() {
157 Some(DelimiterKind::Parenthesis) => ("(", ")"),
158 Some(DelimiterKind::Brace) => ("{", "}"),
159 Some(DelimiterKind::Bracket) => ("[", "]"),
163 let mut needs_space = false;
164 for tt in self.token_trees.iter() {
170 TokenTree::Leaf(Leaf::Punct(p)) => {
171 needs_space = p.spacing == Spacing::Alone;
172 fmt::Display::fmt(p, f)?
174 tt => fmt::Display::fmt(tt, f)?,
182 impl fmt::Display for Leaf {
183 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
185 Leaf::Ident(it) => fmt::Display::fmt(it, f),
186 Leaf::Literal(it) => fmt::Display::fmt(it, f),
187 Leaf::Punct(it) => fmt::Display::fmt(it, f),
192 impl fmt::Display for Ident {
193 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
194 fmt::Display::fmt(&self.text, f)
198 impl fmt::Display for Literal {
199 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
200 fmt::Display::fmt(&self.text, f)
204 impl fmt::Display for Punct {
205 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
206 fmt::Display::fmt(&self.char, f)
211 /// Count the number of tokens recursively
212 pub fn count(&self) -> usize {
213 let children_count = self
217 TokenTree::Subtree(c) => c.count(),
222 self.token_trees.len() + children_count
225 pub fn delimiter_kind(&self) -> Option<DelimiterKind> {
226 self.delimiter.map(|it| it.kind)
232 #[derive(Debug, PartialEq, Eq, Clone)]
233 pub enum ExpansionError {
237 ExpansionError(String),
240 impl fmt::Display for ExpansionError {
241 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
243 ExpansionError::IOError(e) => write!(f, "I/O error: {}", e),
244 ExpansionError::JsonError(e) => write!(f, "JSON decoding error: {}", e),
245 ExpansionError::Unknown(e) => e.fmt(f),
246 ExpansionError::ExpansionError(e) => write!(f, "proc macro returned error: {}", e),
251 pub trait TokenExpander: fmt::Debug + Send + Sync + RefUnwindSafe {
252 fn expand(&self, subtree: &Subtree, attrs: Option<&Subtree>)
253 -> Result<Subtree, ExpansionError>;