-//! FIXME: write short doc here
+//! Conversions between [`SyntaxNode`] and [`tt::TokenTree`].
-use parser::{FragmentKind, ParseError, TreeSink};
use rustc_hash::FxHashMap;
+use stdx::{always, non_empty_vec::NonEmptyVec};
use syntax::{
ast::{self, make::tokens::doc_comment},
- tokenize, AstToken, Parse, SmolStr, SyntaxKind,
+ AstToken, Parse, PreorderWithTokens, SmolStr, SyntaxElement, SyntaxKind,
SyntaxKind::*,
- SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, Token as RawToken, T,
+ SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, WalkEvent, T,
};
use tt::buffer::{Cursor, TokenBuffer};
-use crate::ExpandError;
-use crate::{subtree_source::SubtreeTokenSource, tt_iter::TtIter};
+use crate::{to_parser_input::to_parser_input, tt_iter::TtIter, TokenMap};
-#[derive(Debug, PartialEq, Eq, Clone, Copy)]
-pub enum TokenTextRange {
- Token(TextRange),
- Delimiter(TextRange),
+/// Convert the syntax node to a `TokenTree` (what macro
+/// will consume).
+pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> (tt::Subtree, TokenMap) {
+ let (subtree, token_map, _) = syntax_node_to_token_tree_with_modifications(
+ node,
+ Default::default(),
+ 0,
+ Default::default(),
+ Default::default(),
+ );
+ (subtree, token_map)
}
-impl TokenTextRange {
- pub fn by_kind(self, kind: SyntaxKind) -> Option<TextRange> {
- match self {
- TokenTextRange::Token(it) => Some(it),
- TokenTextRange::Delimiter(it) => match kind {
- T!['{'] | T!['('] | T!['['] => Some(TextRange::at(it.start(), 1.into())),
- T!['}'] | T![')'] | T![']'] => {
- Some(TextRange::at(it.end() - TextSize::of('}'), 1.into()))
- }
- _ => None,
- },
- }
- }
+/// Convert the syntax node to a `TokenTree` (what macro will consume)
+/// with the censored range excluded.
+pub fn syntax_node_to_token_tree_with_modifications(
+ node: &SyntaxNode,
+ existing_token_map: TokenMap,
+ next_id: u32,
+ replace: FxHashMap<SyntaxNode, Vec<SyntheticToken>>,
+ append: FxHashMap<SyntaxNode, Vec<SyntheticToken>>,
+) -> (tt::Subtree, TokenMap, u32) {
+ let global_offset = node.text_range().start();
+ let mut c = Convertor::new(node, global_offset, existing_token_map, next_id, replace, append);
+ let subtree = convert_tokens(&mut c);
+ c.id_alloc.map.shrink_to_fit();
+ always!(c.replace.is_empty(), "replace: {:?}", c.replace);
+ always!(c.append.is_empty(), "append: {:?}", c.append);
+ (subtree, c.id_alloc.map, c.id_alloc.next_id)
}
-/// Maps `tt::TokenId` to the relative range of the original token.
-#[derive(Debug, PartialEq, Eq, Clone, Default)]
-pub struct TokenMap {
- /// Maps `tt::TokenId` to the *relative* source range.
- entries: Vec<(tt::TokenId, TokenTextRange)>,
-}
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct SyntheticTokenId(pub u32);
-/// Convert the syntax tree (what user has written) to a `TokenTree` (what macro
-/// will consume).
-pub fn ast_to_token_tree(ast: &impl ast::AstNode) -> Option<(tt::Subtree, TokenMap)> {
- syntax_node_to_token_tree(ast.syntax())
-}
-
-/// Convert the syntax node to a `TokenTree` (what macro
-/// will consume).
-pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, TokenMap)> {
- let global_offset = node.text_range().start();
- let mut c = Convertor::new(node, global_offset);
- let subtree = c.go()?;
- c.id_alloc.map.entries.shrink_to_fit();
- Some((subtree, c.id_alloc.map))
+#[derive(Debug, Clone)]
+pub struct SyntheticToken {
+ pub kind: SyntaxKind,
+ pub text: SmolStr,
+ pub range: TextRange,
+ pub id: SyntheticTokenId,
}
// The following items are what `rustc` macro can be parsed into :
pub fn token_tree_to_syntax_node(
tt: &tt::Subtree,
- fragment_kind: FragmentKind,
-) -> Result<(Parse<SyntaxNode>, TokenMap), ExpandError> {
+ entry_point: parser::TopEntryPoint,
+) -> (Parse<SyntaxNode>, TokenMap) {
let buffer = match tt {
tt::Subtree { delimiter: None, token_trees } => {
TokenBuffer::from_tokens(token_trees.as_slice())
}
_ => TokenBuffer::from_subtree(tt),
};
- let mut token_source = SubtreeTokenSource::new(&buffer);
+ let parser_input = to_parser_input(&buffer);
+ let parser_output = entry_point.parse(&parser_input);
let mut tree_sink = TtTreeSink::new(buffer.begin());
- parser::parse_fragment(&mut token_source, &mut tree_sink, fragment_kind);
- if tree_sink.roots.len() != 1 {
- return Err(ExpandError::ConversionError);
+ for event in parser_output.iter() {
+ match event {
+ parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => {
+ tree_sink.token(kind, n_raw_tokens)
+ }
+ parser::Step::Enter { kind } => tree_sink.start_node(kind),
+ parser::Step::Exit => tree_sink.finish_node(),
+ parser::Step::Error { msg } => tree_sink.error(msg.to_string()),
+ }
}
- //FIXME: would be cool to report errors
let (parse, range_map) = tree_sink.finish();
- Ok((parse, range_map))
+ (parse, range_map)
}
/// Convert a string to a `TokenTree`
pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> {
- let (tokens, errors) = tokenize(text);
- if !errors.is_empty() {
+ let lexed = parser::LexedStr::new(text);
+ if lexed.errors().next().is_some() {
return None;
}
let mut conv = RawConvertor {
- text,
- offset: TextSize::default(),
- inner: tokens.iter(),
+ lexed,
+ pos: 0,
id_alloc: TokenIdAlloc {
map: Default::default(),
global_offset: TextSize::default(),
},
};
- let subtree = conv.go()?;
+ let subtree = convert_tokens(&mut conv);
Some((subtree, conv.id_alloc.map))
}
-/// Split token tree with seperate expr: $($e:expr)SEP*
+/// Split token tree with separate expr: $($e:expr)SEP*
pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> {
if tt.token_trees.is_empty() {
return Vec::new();
let mut res = Vec::new();
while iter.peek_n(0).is_some() {
- let expanded = iter.expect_fragment(FragmentKind::Expr);
- if expanded.err.is_some() {
- break;
- }
+ let expanded = iter.expect_fragment(parser::PrefixEntryPoint::Expr);
res.push(match expanded.value {
None => break,
res
}
-impl TokenMap {
- pub fn token_by_range(&self, relative_range: TextRange) -> Option<tt::TokenId> {
- let &(token_id, _) = self.entries.iter().find(|(_, range)| match range {
- TokenTextRange::Token(it) => *it == relative_range,
- TokenTextRange::Delimiter(it) => {
- let open = TextRange::at(it.start(), 1.into());
- let close = TextRange::at(it.end() - TextSize::of('}'), 1.into());
- open == relative_range || close == relative_range
- }
- })?;
- Some(token_id)
+fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree {
+ struct StackEntry {
+ subtree: tt::Subtree,
+ idx: usize,
+ open_range: TextRange,
}
- pub fn range_by_token(&self, token_id: tt::TokenId) -> Option<TokenTextRange> {
- let &(_, range) = self.entries.iter().find(|(tid, _)| *tid == token_id)?;
- Some(range)
- }
+ let entry = StackEntry {
+ subtree: tt::Subtree { delimiter: None, ..Default::default() },
+ // never used (delimiter is `None`)
+ idx: !0,
+ open_range: TextRange::empty(TextSize::of('.')),
+ };
+ let mut stack = NonEmptyVec::new(entry);
- fn insert(&mut self, token_id: tt::TokenId, relative_range: TextRange) {
- self.entries.push((token_id, TokenTextRange::Token(relative_range)));
- }
+ loop {
+ let StackEntry { subtree, .. } = stack.last_mut();
+ let result = &mut subtree.token_trees;
+ let (token, range) = match conv.bump() {
+ Some(it) => it,
+ None => break,
+ };
+ let synth_id = token.synthetic_id(&conv);
+
+ let kind = token.kind(&conv);
+ if kind == COMMENT {
+ if let Some(tokens) = conv.convert_doc_comment(&token) {
+ // FIXME: There has to be a better way to do this
+ // Add the comments token id to the converted doc string
+ let id = conv.id_alloc().alloc(range, synth_id);
+ result.extend(tokens.into_iter().map(|mut tt| {
+ if let tt::TokenTree::Subtree(sub) = &mut tt {
+ if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(lit))) =
+ sub.token_trees.get_mut(2)
+ {
+ lit.id = id
+ }
+ }
+ tt
+ }));
+ }
+ continue;
+ }
+ let tt = if kind.is_punct() && kind != UNDERSCORE {
+ if synth_id.is_none() {
+ assert_eq!(range.len(), TextSize::of('.'));
+ }
- fn insert_delim(
- &mut self,
- token_id: tt::TokenId,
- open_relative_range: TextRange,
- close_relative_range: TextRange,
- ) -> usize {
- let res = self.entries.len();
- let cover = open_relative_range.cover(close_relative_range);
-
- self.entries.push((token_id, TokenTextRange::Delimiter(cover)));
- res
- }
-
- fn update_close_delim(&mut self, idx: usize, close_relative_range: TextRange) {
- let (_, token_text_range) = &mut self.entries[idx];
- if let TokenTextRange::Delimiter(dim) = token_text_range {
- let cover = dim.cover(close_relative_range);
- *token_text_range = TokenTextRange::Delimiter(cover);
+ if let Some(delim) = subtree.delimiter {
+ let expected = match delim.kind {
+ tt::DelimiterKind::Parenthesis => T![')'],
+ tt::DelimiterKind::Brace => T!['}'],
+ tt::DelimiterKind::Bracket => T![']'],
+ };
+
+ if kind == expected {
+ if let Some(entry) = stack.pop() {
+ conv.id_alloc().close_delim(entry.idx, Some(range));
+ stack.last_mut().subtree.token_trees.push(entry.subtree.into());
+ }
+ continue;
+ }
+ }
+
+ let delim = match kind {
+ T!['('] => Some(tt::DelimiterKind::Parenthesis),
+ T!['{'] => Some(tt::DelimiterKind::Brace),
+ T!['['] => Some(tt::DelimiterKind::Bracket),
+ _ => None,
+ };
+
+ if let Some(kind) = delim {
+ let mut subtree = tt::Subtree::default();
+ let (id, idx) = conv.id_alloc().open_delim(range);
+ subtree.delimiter = Some(tt::Delimiter { id, kind });
+ stack.push(StackEntry { subtree, idx, open_range: range });
+ continue;
+ }
+
+ let spacing = match conv.peek().map(|next| next.kind(&conv)) {
+ Some(kind)
+ if !kind.is_trivia()
+ && kind.is_punct()
+ && kind != T!['[']
+ && kind != T!['{']
+ && kind != T!['(']
+ && kind != UNDERSCORE =>
+ {
+ tt::Spacing::Joint
+ }
+ _ => tt::Spacing::Alone,
+ };
+ let char = match token.to_char(&conv) {
+ Some(c) => c,
+ None => {
+ panic!("Token from lexer must be single char: token = {:#?}", token);
+ }
+ };
+ tt::Leaf::from(tt::Punct { char, spacing, id: conv.id_alloc().alloc(range, synth_id) })
+ .into()
+ } else {
+ macro_rules! make_leaf {
+ ($i:ident) => {
+ tt::$i { id: conv.id_alloc().alloc(range, synth_id), text: token.to_text(conv) }
+ .into()
+ };
+ }
+ let leaf: tt::Leaf = match kind {
+ T![true] | T![false] => make_leaf!(Ident),
+ IDENT => make_leaf!(Ident),
+ UNDERSCORE => make_leaf!(Ident),
+ k if k.is_keyword() => make_leaf!(Ident),
+ k if k.is_literal() => make_leaf!(Literal),
+ LIFETIME_IDENT => {
+ let char_unit = TextSize::of('\'');
+ let r = TextRange::at(range.start(), char_unit);
+ let apostrophe = tt::Leaf::from(tt::Punct {
+ char: '\'',
+ spacing: tt::Spacing::Joint,
+ id: conv.id_alloc().alloc(r, synth_id),
+ });
+ result.push(apostrophe.into());
+
+ let r = TextRange::at(range.start() + char_unit, range.len() - char_unit);
+ let ident = tt::Leaf::from(tt::Ident {
+ text: SmolStr::new(&token.to_text(conv)[1..]),
+ id: conv.id_alloc().alloc(r, synth_id),
+ });
+ result.push(ident.into());
+ continue;
+ }
+ _ => continue,
+ };
+
+ leaf.into()
+ };
+ result.push(tt);
+ }
+
+ // If we get here, we've consumed all input tokens.
+ // We might have more than one subtree in the stack, if the delimiters are improperly balanced.
+ // Merge them so we're left with one.
+ while let Some(entry) = stack.pop() {
+ let parent = stack.last_mut();
+
+ conv.id_alloc().close_delim(entry.idx, None);
+ let leaf: tt::Leaf = tt::Punct {
+ id: conv.id_alloc().alloc(entry.open_range, None),
+ char: match entry.subtree.delimiter.unwrap().kind {
+ tt::DelimiterKind::Parenthesis => '(',
+ tt::DelimiterKind::Brace => '{',
+ tt::DelimiterKind::Bracket => '[',
+ },
+ spacing: tt::Spacing::Alone,
}
+ .into();
+ parent.subtree.token_trees.push(leaf.into());
+ parent.subtree.token_trees.extend(entry.subtree.token_trees);
}
- fn remove_delim(&mut self, idx: usize) {
- // FIXME: This could be accidentally quadratic
- self.entries.remove(idx);
+ let subtree = stack.into_last().subtree;
+ if let [tt::TokenTree::Subtree(first)] = &*subtree.token_trees {
+ first.clone()
+ } else {
+ subtree
}
}
// Quote the string
// Note that `tt::Literal` expect an escaped string
- let text = format!("{:?}", text.escape_debug().to_string());
+ let text = format!("\"{}\"", text.escape_debug());
text.into()
}
fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option<Vec<tt::TokenTree>> {
+ cov_mark::hit!(test_meta_doc_comments);
let comment = ast::Comment::cast(token.clone())?;
let doc = comment.kind().doc?;
let meta_tkns = vec![mk_ident("doc"), mk_punct('='), mk_doc_literal(&comment)];
// Make `#![]`
- let mut token_trees = vec![mk_punct('#')];
+ let mut token_trees = Vec::with_capacity(3);
+ token_trees.push(mk_punct('#'));
if let ast::CommentPlacement::Inner = doc {
token_trees.push(mk_punct('!'));
}
}
impl TokenIdAlloc {
- fn alloc(&mut self, absolute_range: TextRange) -> tt::TokenId {
+ fn alloc(
+ &mut self,
+ absolute_range: TextRange,
+ synthetic_id: Option<SyntheticTokenId>,
+ ) -> tt::TokenId {
let relative_range = absolute_range - self.global_offset;
let token_id = tt::TokenId(self.next_id);
self.next_id += 1;
self.map.insert(token_id, relative_range);
+ if let Some(id) = synthetic_id {
+ self.map.insert_synthetic(token_id, id);
+ }
token_id
}
}
}
-/// A Raw Token (straightly from lexer) convertor
+/// A raw token (straight from lexer) convertor
struct RawConvertor<'a> {
- text: &'a str,
- offset: TextSize,
+ lexed: parser::LexedStr<'a>,
+ pos: usize,
id_alloc: TokenIdAlloc,
- inner: std::slice::Iter<'a, RawToken>,
-}
-
-trait SrcToken: std::fmt::Debug {
- fn kind(&self) -> SyntaxKind;
-
- fn to_char(&self) -> Option<char>;
-
- fn to_text(&self) -> SmolStr;
}
-trait TokenConvertor {
- type Token: SrcToken;
-
- fn go(&mut self) -> Option<tt::Subtree> {
- let mut subtree = tt::Subtree::default();
- subtree.delimiter = None;
- while self.peek().is_some() {
- self.collect_leaf(&mut subtree.token_trees);
- }
- if subtree.token_trees.is_empty() {
- return None;
- }
- if subtree.token_trees.len() == 1 {
- if let tt::TokenTree::Subtree(first) = &subtree.token_trees[0] {
- return Some(first.clone());
- }
- }
- Some(subtree)
- }
-
- fn collect_leaf(&mut self, result: &mut Vec<tt::TokenTree>) {
- let (token, range) = match self.bump() {
- None => return,
- Some(it) => it,
- };
+trait SrcToken<Ctx>: std::fmt::Debug {
+ fn kind(&self, ctx: &Ctx) -> SyntaxKind;
- let k: SyntaxKind = token.kind();
- if k == COMMENT {
- if let Some(tokens) = self.convert_doc_comment(&token) {
- result.extend(tokens);
- }
- return;
- }
+ fn to_char(&self, ctx: &Ctx) -> Option<char>;
- result.push(if k.is_punct() {
- assert_eq!(range.len(), TextSize::of('.'));
- let delim = match k {
- T!['('] => Some((tt::DelimiterKind::Parenthesis, T![')'])),
- T!['{'] => Some((tt::DelimiterKind::Brace, T!['}'])),
- T!['['] => Some((tt::DelimiterKind::Bracket, T![']'])),
- _ => None,
- };
-
- if let Some((kind, closed)) = delim {
- let mut subtree = tt::Subtree::default();
- let (id, idx) = self.id_alloc().open_delim(range);
- subtree.delimiter = Some(tt::Delimiter { kind, id });
-
- while self.peek().map(|it| it.kind() != closed).unwrap_or(false) {
- self.collect_leaf(&mut subtree.token_trees);
- }
- let last_range = match self.bump() {
- None => {
- // For error resilience, we insert an char punct for the opening delim here
- self.id_alloc().close_delim(idx, None);
- let leaf: tt::Leaf = tt::Punct {
- id: self.id_alloc().alloc(range),
- char: token.to_char().unwrap(),
- spacing: tt::Spacing::Alone,
- }
- .into();
- result.push(leaf.into());
- result.extend(subtree.token_trees);
- return;
- }
- Some(it) => it.1,
- };
- self.id_alloc().close_delim(idx, Some(last_range));
- subtree.into()
- } else {
- let spacing = match self.peek() {
- Some(next)
- if next.kind().is_trivia()
- || next.kind() == T!['[']
- || next.kind() == T!['{']
- || next.kind() == T!['('] =>
- {
- tt::Spacing::Alone
- }
- Some(next) if next.kind().is_punct() => tt::Spacing::Joint,
- _ => tt::Spacing::Alone,
- };
- let char = match token.to_char() {
- Some(c) => c,
- None => {
- panic!("Token from lexer must be single char: token = {:#?}", token);
- }
- };
- tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc().alloc(range) }).into()
- }
- } else {
- macro_rules! make_leaf {
- ($i:ident) => {
- tt::$i { id: self.id_alloc().alloc(range), text: token.to_text() }.into()
- };
- }
- let leaf: tt::Leaf = match k {
- T![true] | T![false] => make_leaf!(Ident),
- IDENT => make_leaf!(Ident),
- k if k.is_keyword() => make_leaf!(Ident),
- k if k.is_literal() => make_leaf!(Literal),
- LIFETIME_IDENT => {
- let char_unit = TextSize::of('\'');
- let r = TextRange::at(range.start(), char_unit);
- let apostrophe = tt::Leaf::from(tt::Punct {
- char: '\'',
- spacing: tt::Spacing::Joint,
- id: self.id_alloc().alloc(r),
- });
- result.push(apostrophe.into());
+ fn to_text(&self, ctx: &Ctx) -> SmolStr;
- let r = TextRange::at(range.start() + char_unit, range.len() - char_unit);
- let ident = tt::Leaf::from(tt::Ident {
- text: SmolStr::new(&token.to_text()[1..]),
- id: self.id_alloc().alloc(r),
- });
- result.push(ident.into());
- return;
- }
- _ => return,
- };
+ fn synthetic_id(&self, ctx: &Ctx) -> Option<SyntheticTokenId>;
+}
- leaf.into()
- });
- }
+trait TokenConvertor: Sized {
+ type Token: SrcToken<Self>;
fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>>;
fn id_alloc(&mut self) -> &mut TokenIdAlloc;
}
-impl<'a> SrcToken for (&'a RawToken, &'a str) {
- fn kind(&self) -> SyntaxKind {
- self.0.kind
+impl<'a> SrcToken<RawConvertor<'a>> for usize {
+ fn kind(&self, ctx: &RawConvertor<'a>) -> SyntaxKind {
+ ctx.lexed.kind(*self)
}
- fn to_char(&self) -> Option<char> {
- self.1.chars().next()
+ fn to_char(&self, ctx: &RawConvertor<'a>) -> Option<char> {
+ ctx.lexed.text(*self).chars().next()
}
- fn to_text(&self) -> SmolStr {
- self.1.into()
+ fn to_text(&self, ctx: &RawConvertor<'_>) -> SmolStr {
+ ctx.lexed.text(*self).into()
}
-}
-impl RawConvertor<'_> {}
+ fn synthetic_id(&self, _ctx: &RawConvertor<'a>) -> Option<SyntheticTokenId> {
+ None
+ }
+}
impl<'a> TokenConvertor for RawConvertor<'a> {
- type Token = (&'a RawToken, &'a str);
+ type Token = usize;
- fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>> {
- convert_doc_comment(&doc_comment(token.1))
+ fn convert_doc_comment(&self, &token: &usize) -> Option<Vec<tt::TokenTree>> {
+ let text = self.lexed.text(token);
+ convert_doc_comment(&doc_comment(text))
}
fn bump(&mut self) -> Option<(Self::Token, TextRange)> {
- let token = self.inner.next()?;
- let range = TextRange::at(self.offset, token.len);
- self.offset += token.len;
+ if self.pos == self.lexed.len() {
+ return None;
+ }
+ let token = self.pos;
+ self.pos += 1;
+ let range = self.lexed.text_range(token);
+ let range = TextRange::new(range.start.try_into().unwrap(), range.end.try_into().unwrap());
- Some(((token, &self.text[range]), range))
+ Some((token, range))
}
fn peek(&self) -> Option<Self::Token> {
- let token = self.inner.as_slice().get(0);
-
- token.map(|it| {
- let range = TextRange::at(self.offset, it.len);
- (it, &self.text[range])
- })
+ if self.pos == self.lexed.len() {
+ return None;
+ }
+ Some(self.pos)
}
fn id_alloc(&mut self) -> &mut TokenIdAlloc {
struct Convertor {
id_alloc: TokenIdAlloc,
current: Option<SyntaxToken>,
+ current_synthetic: Vec<SyntheticToken>,
+ preorder: PreorderWithTokens,
+ replace: FxHashMap<SyntaxNode, Vec<SyntheticToken>>,
+ append: FxHashMap<SyntaxNode, Vec<SyntheticToken>>,
range: TextRange,
punct_offset: Option<(SyntaxToken, TextSize)>,
}
impl Convertor {
- fn new(node: &SyntaxNode, global_offset: TextSize) -> Convertor {
+ fn new(
+ node: &SyntaxNode,
+ global_offset: TextSize,
+ existing_token_map: TokenMap,
+ next_id: u32,
+ mut replace: FxHashMap<SyntaxNode, Vec<SyntheticToken>>,
+ mut append: FxHashMap<SyntaxNode, Vec<SyntheticToken>>,
+ ) -> Convertor {
+ let range = node.text_range();
+ let mut preorder = node.preorder_with_tokens();
+ let (first, synthetic) = Self::next_token(&mut preorder, &mut replace, &mut append);
Convertor {
- id_alloc: { TokenIdAlloc { map: TokenMap::default(), global_offset, next_id: 0 } },
- current: node.first_token(),
- range: node.text_range(),
+ id_alloc: { TokenIdAlloc { map: existing_token_map, global_offset, next_id } },
+ current: first,
+ current_synthetic: synthetic,
+ preorder,
+ range,
+ replace,
+ append,
punct_offset: None,
}
}
+
+ fn next_token(
+ preorder: &mut PreorderWithTokens,
+ replace: &mut FxHashMap<SyntaxNode, Vec<SyntheticToken>>,
+ append: &mut FxHashMap<SyntaxNode, Vec<SyntheticToken>>,
+ ) -> (Option<SyntaxToken>, Vec<SyntheticToken>) {
+ while let Some(ev) = preorder.next() {
+ let ele = match ev {
+ WalkEvent::Enter(ele) => ele,
+ WalkEvent::Leave(SyntaxElement::Node(node)) => {
+ if let Some(mut v) = append.remove(&node) {
+ if !v.is_empty() {
+ v.reverse();
+ return (None, v);
+ }
+ }
+ continue;
+ }
+ _ => continue,
+ };
+ match ele {
+ SyntaxElement::Token(t) => return (Some(t), Vec::new()),
+ SyntaxElement::Node(node) => {
+ if let Some(mut v) = replace.remove(&node) {
+ preorder.skip_subtree();
+ if !v.is_empty() {
+ v.reverse();
+ return (None, v);
+ }
+ }
+ }
+ }
+ }
+ (None, Vec::new())
+ }
}
#[derive(Debug)]
enum SynToken {
Ordinary(SyntaxToken),
+ // FIXME is this supposed to be `Punct`?
Punch(SyntaxToken, TextSize),
+ Synthetic(SyntheticToken),
}
impl SynToken {
- fn token(&self) -> &SyntaxToken {
+ fn token(&self) -> Option<&SyntaxToken> {
match self {
- SynToken::Ordinary(it) => it,
- SynToken::Punch(it, _) => it,
+ SynToken::Ordinary(it) | SynToken::Punch(it, _) => Some(it),
+ SynToken::Synthetic(_) => None,
}
}
}
-impl SrcToken for SynToken {
- fn kind(&self) -> SyntaxKind {
- self.token().kind()
+impl SrcToken<Convertor> for SynToken {
+ fn kind(&self, _ctx: &Convertor) -> SyntaxKind {
+ match self {
+ SynToken::Ordinary(token) => token.kind(),
+ SynToken::Punch(token, _) => token.kind(),
+ SynToken::Synthetic(token) => token.kind,
+ }
}
- fn to_char(&self) -> Option<char> {
+ fn to_char(&self, _ctx: &Convertor) -> Option<char> {
match self {
SynToken::Ordinary(_) => None,
SynToken::Punch(it, i) => it.text().chars().nth((*i).into()),
+ SynToken::Synthetic(token) if token.text.len() == 1 => token.text.chars().next(),
+ SynToken::Synthetic(_) => None,
}
}
- fn to_text(&self) -> SmolStr {
- self.token().text().into()
+ fn to_text(&self, _ctx: &Convertor) -> SmolStr {
+ match self {
+ SynToken::Ordinary(token) => token.text().into(),
+ SynToken::Punch(token, _) => token.text().into(),
+ SynToken::Synthetic(token) => token.text.clone(),
+ }
+ }
+
+ fn synthetic_id(&self, _ctx: &Convertor) -> Option<SyntheticTokenId> {
+ match self {
+ SynToken::Synthetic(token) => Some(token.id),
+ _ => None,
+ }
}
}
impl TokenConvertor for Convertor {
type Token = SynToken;
fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>> {
- convert_doc_comment(token.token())
+ convert_doc_comment(token.token()?)
}
fn bump(&mut self) -> Option<(Self::Token, TextRange)> {
}
}
+ if let Some(synth_token) = self.current_synthetic.pop() {
+ if self.current_synthetic.is_empty() {
+ let (new_current, new_synth) =
+ Self::next_token(&mut self.preorder, &mut self.replace, &mut self.append);
+ self.current = new_current;
+ self.current_synthetic = new_synth;
+ }
+ let range = synth_token.range;
+ return Some((SynToken::Synthetic(synth_token), range));
+ }
+
let curr = self.current.clone()?;
if !&self.range.contains_range(curr.text_range()) {
return None;
}
- self.current = curr.next_token();
-
+ let (new_current, new_synth) =
+ Self::next_token(&mut self.preorder, &mut self.replace, &mut self.append);
+ self.current = new_current;
+ self.current_synthetic = new_synth;
let token = if curr.kind().is_punct() {
+ self.punct_offset = Some((curr.clone(), 0.into()));
let range = curr.text_range();
let range = TextRange::at(range.start(), TextSize::of('.'));
- self.punct_offset = Some((curr.clone(), 0.into()));
(SynToken::Punch(curr, 0.into()), range)
} else {
self.punct_offset = None;
fn peek(&self) -> Option<Self::Token> {
if let Some((punct, mut offset)) = self.punct_offset.clone() {
- offset = offset + TextSize::of('.');
+ offset += TextSize::of('.');
if usize::from(offset) < punct.text().len() {
return Some(SynToken::Punch(punct, offset));
}
}
+ if let Some(synth_token) = self.current_synthetic.last() {
+ return Some(SynToken::Synthetic(synth_token.clone()));
+ }
+
let curr = self.current.clone()?;
if !self.range.contains_range(curr.text_range()) {
return None;
text_pos: TextSize,
inner: SyntaxTreeBuilder,
token_map: TokenMap,
-
- // Number of roots
- // Use for detect ill-form tree which is not single root
- roots: smallvec::SmallVec<[usize; 1]>,
}
impl<'a> TtTreeSink<'a> {
open_delims: FxHashMap::default(),
text_pos: 0.into(),
inner: SyntaxTreeBuilder::default(),
- roots: smallvec::SmallVec::new(),
token_map: TokenMap::default(),
}
}
fn finish(mut self) -> (Parse<SyntaxNode>, TokenMap) {
- self.token_map.entries.shrink_to_fit();
+ self.token_map.shrink_to_fit();
(self.inner.finish(), self.token_map)
}
}
-fn delim_to_str(d: Option<tt::DelimiterKind>, closing: bool) -> &'static str {
+fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> &'static str {
let texts = match d {
- Some(tt::DelimiterKind::Parenthesis) => "()",
- Some(tt::DelimiterKind::Brace) => "{}",
- Some(tt::DelimiterKind::Bracket) => "[]",
- None => return "",
+ tt::DelimiterKind::Parenthesis => "()",
+ tt::DelimiterKind::Brace => "{}",
+ tt::DelimiterKind::Bracket => "[]",
};
let idx = closing as usize;
&texts[idx..texts.len() - (1 - idx)]
}
-impl<'a> TreeSink for TtTreeSink<'a> {
+impl<'a> TtTreeSink<'a> {
fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
- if kind == L_DOLLAR || kind == R_DOLLAR {
- self.cursor = self.cursor.bump_subtree();
- return;
- }
if kind == LIFETIME_IDENT {
n_tokens = 2;
}
let mut last = self.cursor;
for _ in 0..n_tokens {
- let tmp_str: SmolStr;
+ let tmp: u8;
if self.cursor.eof() {
break;
}
last = self.cursor;
- let text: &str = match self.cursor.token_tree() {
- Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
- // Mark the range if needed
- let (text, id) = match leaf {
- tt::Leaf::Ident(ident) => (&ident.text, ident.id),
- tt::Leaf::Punct(punct) => {
- assert!(punct.char.is_ascii());
- let char = &(punct.char as u8);
- tmp_str = SmolStr::new_inline(
- std::str::from_utf8(std::slice::from_ref(char)).unwrap(),
- );
- (&tmp_str, punct.id)
+ let text: &str = loop {
+ break match self.cursor.token_tree() {
+ Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
+ // Mark the range if needed
+ let (text, id) = match leaf {
+ tt::Leaf::Ident(ident) => (ident.text.as_str(), ident.id),
+ tt::Leaf::Punct(punct) => {
+ assert!(punct.char.is_ascii());
+ tmp = punct.char as u8;
+ (std::str::from_utf8(std::slice::from_ref(&tmp)).unwrap(), punct.id)
+ }
+ tt::Leaf::Literal(lit) => (lit.text.as_str(), lit.id),
+ };
+ let range = TextRange::at(self.text_pos, TextSize::of(text));
+ self.token_map.insert(id, range);
+ self.cursor = self.cursor.bump();
+ text
+ }
+ Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
+ self.cursor = self.cursor.subtree().unwrap();
+ match subtree.delimiter {
+ Some(d) => {
+ self.open_delims.insert(d.id, self.text_pos);
+ delim_to_str(d.kind, false)
+ }
+ None => continue,
}
- tt::Leaf::Literal(lit) => (&lit.text, lit.id),
- };
- let range = TextRange::at(self.text_pos, TextSize::of(text.as_str()));
- self.token_map.insert(id, range);
- self.cursor = self.cursor.bump();
- text
- }
- Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
- self.cursor = self.cursor.subtree().unwrap();
- if let Some(id) = subtree.delimiter.map(|it| it.id) {
- self.open_delims.insert(id, self.text_pos);
}
- delim_to_str(subtree.delimiter_kind(), false)
- }
- None => {
- if let Some(parent) = self.cursor.end() {
+ None => {
+ let parent = self.cursor.end().unwrap();
self.cursor = self.cursor.bump();
- if let Some(id) = parent.delimiter.map(|it| it.id) {
- if let Some(open_delim) = self.open_delims.get(&id) {
- let open_range = TextRange::at(*open_delim, TextSize::of('('));
- let close_range = TextRange::at(self.text_pos, TextSize::of('('));
- self.token_map.insert_delim(id, open_range, close_range);
+ match parent.delimiter {
+ Some(d) => {
+ if let Some(open_delim) = self.open_delims.get(&d.id) {
+ let open_range = TextRange::at(*open_delim, TextSize::of('('));
+ let close_range =
+ TextRange::at(self.text_pos, TextSize::of('('));
+ self.token_map.insert_delim(d.id, open_range, close_range);
+ }
+ delim_to_str(d.kind, true)
}
+ None => continue,
}
- delim_to_str(parent.delimiter_kind(), true)
- } else {
- continue;
}
- }
+ };
};
- self.buf += &text;
+ self.buf += text;
self.text_pos += TextSize::of(text);
}
fn start_node(&mut self, kind: SyntaxKind) {
self.inner.start_node(kind);
-
- match self.roots.last_mut() {
- None | Some(0) => self.roots.push(1),
- Some(ref mut n) => **n += 1,
- };
}
fn finish_node(&mut self) {
self.inner.finish_node();
- *self.roots.last_mut().unwrap() -= 1;
}
- fn error(&mut self, error: ParseError) {
+ fn error(&mut self, error: String) {
self.inner.error(error, self.text_pos)
}
}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::tests::parse_macro;
- use parser::TokenSource;
- use syntax::{
- algo::{insert_children, InsertPosition},
- ast::AstNode,
- };
-
- #[test]
- fn convert_tt_token_source() {
- let expansion = parse_macro(
- r#"
- macro_rules! literals {
- ($i:ident) => {
- {
- let a = 'c';
- let c = 1000;
- let f = 12E+99_f64;
- let s = "rust1";
- }
- }
- }
- "#,
- )
- .expand_tt("literals!(foo);");
- let tts = &[expansion.into()];
- let buffer = tt::buffer::TokenBuffer::from_tokens(tts);
- let mut tt_src = SubtreeTokenSource::new(&buffer);
- let mut tokens = vec![];
- while tt_src.current().kind != EOF {
- tokens.push((tt_src.current().kind, tt_src.text()));
- tt_src.bump();
- }
-
- // [${]
- // [let] [a] [=] ['c'] [;]
- assert_eq!(tokens[2 + 3].1, "'c'");
- assert_eq!(tokens[2 + 3].0, CHAR);
- // [let] [c] [=] [1000] [;]
- assert_eq!(tokens[2 + 5 + 3].1, "1000");
- assert_eq!(tokens[2 + 5 + 3].0, INT_NUMBER);
- // [let] [f] [=] [12E+99_f64] [;]
- assert_eq!(tokens[2 + 10 + 3].1, "12E+99_f64");
- assert_eq!(tokens[2 + 10 + 3].0, FLOAT_NUMBER);
-
- // [let] [s] [=] ["rust1"] [;]
- assert_eq!(tokens[2 + 15 + 3].1, "\"rust1\"");
- assert_eq!(tokens[2 + 15 + 3].0, STRING);
- }
-
- #[test]
- fn stmts_token_trees_to_expr_is_err() {
- let expansion = parse_macro(
- r#"
- macro_rules! stmts {
- () => {
- let a = 0;
- let b = 0;
- let c = 0;
- let d = 0;
- }
- }
- "#,
- )
- .expand_tt("stmts!();");
- assert!(token_tree_to_syntax_node(&expansion, FragmentKind::Expr).is_err());
- }
-
- #[test]
- fn test_token_tree_last_child_is_white_space() {
- let source_file = ast::SourceFile::parse("f!({} );").ok().unwrap();
- let macro_call = source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
- let token_tree = macro_call.token_tree().unwrap();
-
- // Token Tree now is :
- // TokenTree
- // - T!['(']
- // - TokenTree
- // - T!['{']
- // - T!['}']
- // - WHITE_SPACE
- // - T![')']
-
- let rbrace =
- token_tree.syntax().descendants_with_tokens().find(|it| it.kind() == T!['}']).unwrap();
- let space = token_tree
- .syntax()
- .descendants_with_tokens()
- .find(|it| it.kind() == SyntaxKind::WHITESPACE)
- .unwrap();
-
- // reorder th white space, such that the white is inside the inner token-tree.
- let token_tree = insert_children(
- &rbrace.parent().unwrap(),
- InsertPosition::Last,
- std::iter::once(space),
- );
-
- // Token Tree now is :
- // TokenTree
- // - T!['{']
- // - T!['}']
- // - WHITE_SPACE
- let token_tree = ast::TokenTree::cast(token_tree).unwrap();
- let tt = ast_to_token_tree(&token_tree).unwrap().0;
-
- assert_eq!(tt.delimiter_kind(), Some(tt::DelimiterKind::Brace));
- }
-
- #[test]
- fn test_token_tree_multi_char_punct() {
- let source_file = ast::SourceFile::parse("struct Foo { a: x::Y }").ok().unwrap();
- let struct_def = source_file.syntax().descendants().find_map(ast::Struct::cast).unwrap();
- let tt = ast_to_token_tree(&struct_def).unwrap().0;
- token_tree_to_syntax_node(&tt, FragmentKind::Item).unwrap();
- }
-}