crates/syntax/src/parsing/text_tree_sink.rs

   1 //! FIXME: write short doc here
   2
   3 use std::mem;
   4
   5 use parser::{ParseError, TreeSink};
   6
   7 use crate::{
   8     ast,
   9     parsing::Token,
  10     syntax_node::GreenNode,
  11     SyntaxError,
  12     SyntaxKind::{self, *},
  13     SyntaxTreeBuilder, TextRange, TextSize,
  14 };
  15
  16 /// Bridges the parser with our specific syntax tree representation.
  17 ///
  18 /// `TextTreeSink` also handles attachment of trivia (whitespace) to nodes.
  19 pub(crate) struct TextTreeSink<'a> {
  20     text: &'a str,
  21     tokens: &'a [Token],
  22     text_pos: TextSize,
  23     token_pos: usize,
  24     state: State,
  25     inner: SyntaxTreeBuilder,
  26 }
  27
  28 enum State {
  29     PendingStart,
  30     Normal,
  31     PendingFinish,
  32 }
  33
  34 impl<'a> TreeSink for TextTreeSink<'a> {
  35     fn token(&mut self, kind: SyntaxKind, n_tokens: u8) {
  36         match mem::replace(&mut self.state, State::Normal) {
  37             State::PendingStart => unreachable!(),
  38             State::PendingFinish => self.inner.finish_node(),
  39             State::Normal => (),
  40         }
  41         self.eat_trivias();
  42         let n_tokens = n_tokens as usize;
  43         let len = self.tokens[self.token_pos..self.token_pos + n_tokens]
  44             .iter()
  45             .map(|it| it.len)
  46             .sum::<TextSize>();
  47         self.do_token(kind, len, n_tokens);
  48     }
  49
  50     fn start_node(&mut self, kind: SyntaxKind) {
  51         match mem::replace(&mut self.state, State::Normal) {
  52             State::PendingStart => {
  53                 self.inner.start_node(kind);
  54                 // No need to attach trivias to previous node: there is no
  55                 // previous node.
  56                 return;
  57             }
  58             State::PendingFinish => self.inner.finish_node(),
  59             State::Normal => (),
  60         }
  61
  62         let n_trivias =
  63             self.tokens[self.token_pos..].iter().take_while(|it| it.kind.is_trivia()).count();
  64         let leading_trivias = &self.tokens[self.token_pos..self.token_pos + n_trivias];
  65         let mut trivia_end =
  66             self.text_pos + leading_trivias.iter().map(|it| it.len).sum::<TextSize>();
  67
  68         let n_attached_trivias = {
  69             let leading_trivias = leading_trivias.iter().rev().map(|it| {
  70                 let next_end = trivia_end - it.len;
  71                 let range = TextRange::new(next_end, trivia_end);
  72                 trivia_end = next_end;
  73                 (it.kind, &self.text[range])
  74             });
  75             n_attached_trivias(kind, leading_trivias)
  76         };
  77         self.eat_n_trivias(n_trivias - n_attached_trivias);
  78         self.inner.start_node(kind);
  79         self.eat_n_trivias(n_attached_trivias);
  80     }
  81
  82     fn finish_node(&mut self) {
  83         match mem::replace(&mut self.state, State::PendingFinish) {
  84             State::PendingStart => unreachable!(),
  85             State::PendingFinish => self.inner.finish_node(),
  86             State::Normal => (),
  87         }
  88     }
  89
  90     fn error(&mut self, error: ParseError) {
  91         self.inner.error(error, self.text_pos)
  92     }
  93 }
  94
  95 impl<'a> TextTreeSink<'a> {
  96     pub(super) fn new(text: &'a str, tokens: &'a [Token]) -> Self {
  97         Self {
  98             text,
  99             tokens,
 100             text_pos: 0.into(),
 101             token_pos: 0,
 102             state: State::PendingStart,
 103             inner: SyntaxTreeBuilder::default(),
 104         }
 105     }
 106
 107     pub(super) fn finish(mut self) -> (GreenNode, Vec<SyntaxError>) {
 108         match mem::replace(&mut self.state, State::Normal) {
 109             State::PendingFinish => {
 110                 self.eat_trivias();
 111                 self.inner.finish_node()
 112             }
 113             State::PendingStart | State::Normal => unreachable!(),
 114         }
 115
 116         self.inner.finish_raw()
 117     }
 118
 119     fn eat_trivias(&mut self) {
 120         while let Some(&token) = self.tokens.get(self.token_pos) {
 121             if !token.kind.is_trivia() {
 122                 break;
 123             }
 124             self.do_token(token.kind, token.len, 1);
 125         }
 126     }
 127
 128     fn eat_n_trivias(&mut self, n: usize) {
 129         for _ in 0..n {
 130             let token = self.tokens[self.token_pos];
 131             assert!(token.kind.is_trivia());
 132             self.do_token(token.kind, token.len, 1);
 133         }
 134     }
 135
 136     fn do_token(&mut self, kind: SyntaxKind, len: TextSize, n_tokens: usize) {
 137         let range = TextRange::at(self.text_pos, len);
 138         let text = &self.text[range];
 139         self.text_pos += len;
 140         self.token_pos += n_tokens;
 141         self.inner.token(kind, text);
 142     }
 143 }
 144
 145 fn n_attached_trivias<'a>(
 146     kind: SyntaxKind,
 147     trivias: impl Iterator<Item = (SyntaxKind, &'a str)>,
 148 ) -> usize {
 149     match kind {
 150         MACRO_CALL | MACRO_RULES | CONST | TYPE_ALIAS | STRUCT | ENUM | VARIANT | FN | TRAIT
 151         | MODULE | RECORD_FIELD | STATIC | USE => {
 152             let mut res = 0;
 153             let mut trivias = trivias.enumerate().peekable();
 154
 155             while let Some((i, (kind, text))) = trivias.next() {
 156                 match kind {
 157                     WHITESPACE if text.contains("\n\n") => {
 158                         // we check whether the next token is a doc-comment
 159                         // and skip the whitespace in this case
 160                         if let Some((COMMENT, peek_text)) = trivias.peek().map(|(_, pair)| pair) {
 161                             let comment_kind = ast::CommentKind::from_text(peek_text);
 162                             if comment_kind.doc == Some(ast::CommentPlacement::Outer) {
 163                                 continue;
 164                             }
 165                         }
 166                         break;
 167                     }
 168                     COMMENT => {
 169                         let comment_kind = ast::CommentKind::from_text(text);
 170                         if comment_kind.doc == Some(ast::CommentPlacement::Inner) {
 171                             break;
 172                         }
 173                         res = i + 1;
 174                     }
 175                     _ => (),
 176                 }
 177             }
 178             res
 179         }
 180         _ => 0,
 181     }
 182 }