- let text: &str = match self.cursor.token_tree() {
- Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
- // Mark the range if needed
- let (text, id) = match leaf {
- tt::Leaf::Ident(ident) => (&ident.text, ident.id),
- tt::Leaf::Punct(punct) => {
- assert!(punct.char.is_ascii());
- let char = &(punct.char as u8);
- tmp_str = SmolStr::new_inline(
- std::str::from_utf8(std::slice::from_ref(char)).unwrap(),
- );
- (&tmp_str, punct.id)
+ let text: &str = loop {
+ break match self.cursor.token_tree() {
+ Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
+ // Mark the range if needed
+ let (text, id) = match leaf {
+ tt::Leaf::Ident(ident) => (ident.text.as_str(), ident.id),
+ tt::Leaf::Punct(punct) => {
+ assert!(punct.char.is_ascii());
+ tmp = punct.char as u8;
+ (std::str::from_utf8(std::slice::from_ref(&tmp)).unwrap(), punct.id)
+ }
+ tt::Leaf::Literal(lit) => (lit.text.as_str(), lit.id),
+ };
+ let range = TextRange::at(self.text_pos, TextSize::of(text));
+ self.token_map.insert(id, range);
+ self.cursor = self.cursor.bump();
+ text
+ }
+ Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
+ self.cursor = self.cursor.subtree().unwrap();
+ match subtree.delimiter {
+ Some(d) => {
+ self.open_delims.insert(d.id, self.text_pos);
+ delim_to_str(d.kind, false)
+ }
+ None => continue,