1 //! `tt` crate defines a `TokenTree` data structure: this is the interface (both
2 //! input and output) of macros. It closely mirrors `proc_macro` crate's
8 pub use smol_str::SmolStr;
10 /// Represents identity of the token.
12 /// For hygiene purposes, we need to track which expanded tokens originated from
13 /// which source tokens. We do it by assigning an distinct identity to each
14 /// source token and making sure that identities are preserved during macro
16 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
17 pub struct TokenId(pub u32);
20 pub const fn unspecified() -> TokenId {
25 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
30 impl_from!(Leaf, Subtree for TokenTree);
33 pub fn empty() -> Self {
34 TokenTree::Subtree(Subtree::default())
38 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
44 impl_from!(Literal, Punct, Ident for Leaf);
46 #[derive(Clone, PartialEq, Eq, Hash, Default)]
48 pub delimiter: Option<Delimiter>,
49 pub token_trees: Vec<TokenTree>,
52 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
53 pub struct Delimiter {
55 pub kind: DelimiterKind,
58 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
59 pub enum DelimiterKind {
65 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
71 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
78 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
84 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
90 fn print_debug_subtree(f: &mut fmt::Formatter<'_>, subtree: &Subtree, level: usize) -> fmt::Result {
91 let align = std::iter::repeat(" ").take(level).collect::<String>();
93 let aux = match subtree.delimiter.map(|it| (it.kind, it.id.0)) {
94 None => "$".to_string(),
95 Some((DelimiterKind::Parenthesis, id)) => format!("() {}", id),
96 Some((DelimiterKind::Brace, id)) => format!("{{}} {}", id),
97 Some((DelimiterKind::Bracket, id)) => format!("[] {}", id),
100 if subtree.token_trees.is_empty() {
101 write!(f, "{}SUBTREE {}", align, aux)?;
103 writeln!(f, "{}SUBTREE {}", align, aux)?;
104 for (idx, child) in subtree.token_trees.iter().enumerate() {
105 print_debug_token(f, child, level + 1)?;
106 if idx != subtree.token_trees.len() - 1 {
115 fn print_debug_token(f: &mut fmt::Formatter<'_>, tkn: &TokenTree, level: usize) -> fmt::Result {
116 let align = std::iter::repeat(" ").take(level).collect::<String>();
119 TokenTree::Leaf(leaf) => match leaf {
120 Leaf::Literal(lit) => write!(f, "{}LITERAL {} {}", align, lit.text, lit.id.0)?,
121 Leaf::Punct(punct) => write!(
123 "{}PUNCH {} [{}] {}",
126 if punct.spacing == Spacing::Alone { "alone" } else { "joint" },
129 Leaf::Ident(ident) => write!(f, "{}IDENT {} {}", align, ident.text, ident.id.0)?,
131 TokenTree::Subtree(subtree) => {
132 print_debug_subtree(f, subtree, level)?;
139 impl fmt::Debug for Subtree {
140 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
141 print_debug_subtree(f, self, 0)
145 impl fmt::Display for TokenTree {
146 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
148 TokenTree::Leaf(it) => fmt::Display::fmt(it, f),
149 TokenTree::Subtree(it) => fmt::Display::fmt(it, f),
154 impl fmt::Display for Subtree {
155 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
156 let (l, r) = match self.delimiter_kind() {
157 Some(DelimiterKind::Parenthesis) => ("(", ")"),
158 Some(DelimiterKind::Brace) => ("{", "}"),
159 Some(DelimiterKind::Bracket) => ("[", "]"),
163 let mut needs_space = false;
164 for tt in self.token_trees.iter() {
170 TokenTree::Leaf(Leaf::Punct(p)) => {
171 needs_space = p.spacing == Spacing::Alone;
172 fmt::Display::fmt(p, f)?
174 tt => fmt::Display::fmt(tt, f)?,
182 impl fmt::Display for Leaf {
183 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
185 Leaf::Ident(it) => fmt::Display::fmt(it, f),
186 Leaf::Literal(it) => fmt::Display::fmt(it, f),
187 Leaf::Punct(it) => fmt::Display::fmt(it, f),
192 impl fmt::Display for Ident {
193 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
194 fmt::Display::fmt(&self.text, f)
198 impl fmt::Display for Literal {
199 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
200 fmt::Display::fmt(&self.text, f)
204 impl fmt::Display for Punct {
205 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
206 fmt::Display::fmt(&self.char, f)
211 /// Count the number of tokens recursively
212 pub fn count(&self) -> usize {
213 let children_count = self
217 TokenTree::Subtree(c) => c.count(),
222 self.token_trees.len() + children_count
225 pub fn delimiter_kind(&self) -> Option<DelimiterKind> {
226 self.delimiter.map(|it| it.kind)
231 /// A simple line string used for debugging
232 pub fn as_debug_string(&self) -> String {
233 let delim = match self.delimiter_kind() {
234 Some(DelimiterKind::Brace) => ("{", "}"),
235 Some(DelimiterKind::Bracket) => ("[", "]"),
236 Some(DelimiterKind::Parenthesis) => ("(", ")"),
240 let mut res = String::new();
241 res.push_str(delim.0);
243 for child in &self.token_trees {
244 let s = match child {
245 TokenTree::Leaf(it) => {
247 Leaf::Literal(it) => it.text.to_string(),
248 Leaf::Punct(it) => it.char.to_string(),
249 Leaf::Ident(it) => it.text.to_string(),
252 (Leaf::Ident(_), Some(&TokenTree::Leaf(Leaf::Ident(_)))) => {
255 (Leaf::Punct(_), Some(&TokenTree::Leaf(Leaf::Punct(punct)))) => {
256 if punct.spacing == Spacing::Alone {
265 TokenTree::Subtree(it) => it.as_debug_string(),
271 res.push_str(delim.1);
278 #[derive(Debug, PartialEq, Eq, Clone)]
279 pub enum ExpansionError {
283 ExpansionError(String),
286 impl fmt::Display for ExpansionError {
287 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
289 ExpansionError::IOError(e) => write!(f, "I/O error: {}", e),
290 ExpansionError::JsonError(e) => write!(f, "JSON decoding error: {}", e),
291 ExpansionError::Unknown(e) => e.fmt(f),
292 ExpansionError::ExpansionError(e) => write!(f, "proc macro returned error: {}", e),