tt: &tt::Subtree,
fragment_kind: FragmentKind,
) -> Result<(Parse<SyntaxNode>, TokenMap), ExpandError> {
- let tmp;
- let tokens = match tt {
- tt::Subtree { delimiter: None, token_trees } => token_trees.as_slice(),
- _ => {
- tmp = [tt.clone().into()];
- &tmp[..]
+ let buffer = match tt {
+ tt::Subtree { delimiter: None, token_trees } => {
+ TokenBuffer::from_tokens(token_trees.as_slice())
}
+ _ => TokenBuffer::from_subtree(tt),
};
- let buffer = TokenBuffer::new(&tokens);
let mut token_source = SubtreeTokenSource::new(&buffer);
let mut tree_sink = TtTreeSink::new(buffer.begin());
parser::parse_fragment(&mut token_source, &mut tree_sink, fragment_kind);
}
last = self.cursor;
let text: &str = match self.cursor.token_tree() {
- Some(tt::TokenTree::Leaf(leaf)) => {
+ Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
// Mark the range if needed
let (text, id) = match leaf {
tt::Leaf::Ident(ident) => (&ident.text, ident.id),
self.cursor = self.cursor.bump();
text
}
- Some(tt::TokenTree::Subtree(subtree)) => {
+ Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
self.cursor = self.cursor.subtree().unwrap();
if let Some(id) = subtree.delimiter.map(|it| it.id) {
self.open_delims.insert(id, self.text_pos);
// Add whitespace between adjoint puncts
let next = last.bump();
if let (
- Some(tt::TokenTree::Leaf(tt::Leaf::Punct(curr))),
- Some(tt::TokenTree::Leaf(tt::Leaf::Punct(_))),
+ Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(curr), _)),
+ Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(_), _)),
) = (last.token_tree(), next.token_tree())
{
// Note: We always assume the semi-colon would be the last token in
)
.expand_tt("literals!(foo);");
let tts = &[expansion.into()];
- let buffer = tt::buffer::TokenBuffer::new(tts);
+ let buffer = tt::buffer::TokenBuffer::from_tokens(tts);
let mut tt_src = SubtreeTokenSource::new(&buffer);
let mut tokens = vec![];
while tt_src.current().kind != EOF {
//! FIXME: write short doc here
-use crate::{Subtree, TokenTree};
+use crate::{Leaf, Subtree, TokenTree};
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
struct EntryId(usize);
#[derive(Debug)]
enum Entry<'t> {
// Mimicking types from proc-macro.
- Subtree(&'t TokenTree, EntryId),
+ Subtree(Option<&'t TokenTree>, &'t Subtree, EntryId),
Leaf(&'t TokenTree),
// End entries contain a pointer to the entry from the containing
// token tree, or None if this is the outermost level.
buffers: Vec<Box<[Entry<'t>]>>,
}
-impl<'t> TokenBuffer<'t> {
- pub fn new(tokens: &'t [TokenTree]) -> TokenBuffer<'t> {
- let mut buffers = vec![];
-
- let idx = TokenBuffer::new_inner(tokens, &mut buffers, None);
- assert_eq!(idx, 0);
-
- TokenBuffer { buffers }
- }
+trait TokenList<'a> {
+ fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec<Entry<'a>>);
+}
- fn new_inner(
- tokens: &'t [TokenTree],
- buffers: &mut Vec<Box<[Entry<'t>]>>,
- next: Option<EntryPtr>,
- ) -> usize {
+impl<'a> TokenList<'a> for &'a [TokenTree] {
+ fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec<Entry<'a>>) {
// Must contain everything in tokens and then the Entry::End
- let start_capacity = tokens.len() + 1;
+ let start_capacity = self.len() + 1;
let mut entries = Vec::with_capacity(start_capacity);
let mut children = vec![];
-
- for (idx, tt) in tokens.iter().enumerate() {
+ for (idx, tt) in self.iter().enumerate() {
match tt {
TokenTree::Leaf(_) => {
entries.push(Entry::Leaf(tt));
}
TokenTree::Subtree(subtree) => {
entries.push(Entry::End(None));
- children.push((idx, (subtree, tt)));
+ children.push((idx, (subtree, Some(tt))));
}
}
}
+ (children, entries)
+ }
+}
+
+impl<'a> TokenList<'a> for &'a Subtree {
+ fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec<Entry<'a>>) {
+ // Must contain everything in tokens and then the Entry::End
+ let mut entries = vec![];
+ let mut children = vec![];
+ entries.push(Entry::End(None));
+ children.push((0usize, (*self, None)));
+ (children, entries)
+ }
+}
+
+impl<'t> TokenBuffer<'t> {
+ pub fn from_tokens(tokens: &'t [TokenTree]) -> TokenBuffer<'t> {
+ Self::new(tokens)
+ }
+
+ pub fn from_subtree(subtree: &'t Subtree) -> TokenBuffer<'t> {
+ Self::new(subtree)
+ }
+
+ fn new<T: TokenList<'t>>(tokens: T) -> TokenBuffer<'t> {
+ let mut buffers = vec![];
+ let idx = TokenBuffer::new_inner(tokens, &mut buffers, None);
+ assert_eq!(idx, 0);
+ TokenBuffer { buffers }
+ }
+
+ fn new_inner<T: TokenList<'t>>(
+ tokens: T,
+ buffers: &mut Vec<Box<[Entry<'t>]>>,
+ next: Option<EntryPtr>,
+ ) -> usize {
+ let (children, mut entries) = tokens.entries();
entries.push(Entry::End(next));
let res = buffers.len();
for (child_idx, (subtree, tt)) in children {
let idx = TokenBuffer::new_inner(
- &subtree.token_trees,
+ subtree.token_trees.as_slice(),
buffers,
Some(EntryPtr(EntryId(res), child_idx + 1)),
);
- buffers[res].as_mut()[child_idx] = Entry::Subtree(tt, EntryId(idx));
+ buffers[res].as_mut()[child_idx] = Entry::Subtree(tt, subtree, EntryId(idx));
}
res
}
}
+#[derive(Debug)]
+pub enum TokenTreeRef<'a> {
+ Subtree(&'a Subtree, Option<&'a TokenTree>),
+ Leaf(&'a Leaf, &'a TokenTree),
+}
+
+impl<'a> TokenTreeRef<'a> {
+ pub fn cloned(&self) -> TokenTree {
+ match &self {
+ TokenTreeRef::Subtree(subtree, tt) => match tt {
+ Some(it) => (*it).clone(),
+ None => (*subtree).clone().into(),
+ },
+ TokenTreeRef::Leaf(_, tt) => (*tt).clone(),
+ }
+ }
+}
+
/// A safe version of `Cursor` from `syn` crate https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L125
#[derive(Copy, Clone, Debug)]
pub struct Cursor<'a> {
match self.entry() {
Some(Entry::End(Some(ptr))) => {
let idx = ptr.1;
- if let Some(Entry::Subtree(TokenTree::Subtree(subtree), _)) =
+ if let Some(Entry::Subtree(_, subtree, _)) =
self.buffer.entry(&EntryPtr(ptr.0, idx - 1))
{
return Some(subtree);
}
-
None
}
_ => None,
/// a cursor into that subtree
pub fn subtree(self) -> Option<Cursor<'a>> {
match self.entry() {
- Some(Entry::Subtree(_, entry_id)) => {
+ Some(Entry::Subtree(_, _, entry_id)) => {
Some(Cursor::create(self.buffer, EntryPtr(*entry_id, 0)))
}
_ => None,
}
/// If the cursor is pointing at a `TokenTree`, returns it
- pub fn token_tree(self) -> Option<&'a TokenTree> {
+ pub fn token_tree(self) -> Option<TokenTreeRef<'a>> {
match self.entry() {
- Some(Entry::Leaf(tt)) => Some(tt),
- Some(Entry::Subtree(tt, _)) => Some(tt),
+ Some(Entry::Leaf(tt)) => match tt {
+ TokenTree::Leaf(leaf) => Some(TokenTreeRef::Leaf(leaf, *tt)),
+ TokenTree::Subtree(subtree) => Some(TokenTreeRef::Subtree(subtree, Some(tt))),
+ },
+ Some(Entry::Subtree(tt, subtree, _)) => Some(TokenTreeRef::Subtree(subtree, *tt)),
Some(Entry::End(_)) => None,
None => None,
}
/// a cursor into that subtree
pub fn bump_subtree(self) -> Cursor<'a> {
match self.entry() {
- Some(Entry::Subtree(_, _)) => self.subtree().unwrap(),
+ Some(Entry::Subtree(_, _, _)) => self.subtree().unwrap(),
_ => self.bump(),
}
}
projects / rust.git / commitdiff