mod event;
-mod input;
+pub(crate) mod input;
use std::cell::Cell;
parser_api::Parser,
parser_impl::{
event::{Event, EventProcessor},
- input::{InputPosition, ParserInput},
+ input::InputPosition,
},
},
};
fn finish(self) -> Self::Tree;
}
+pub(super) trait TokenSource {
+ fn token_kind(&self, pos: InputPosition) -> SyntaxKind;
+ fn is_token_joint_to_next(&self, pos: InputPosition) -> bool;
+ fn is_keyword(&self, pos: InputPosition, kw: &str) -> bool;
+}
+
/// Parse a sequence of tokens into the representative node tree
pub(super) fn parse_with<S: TreeSink>(
sink: S,
) -> S::Tree {
let mut events = {
let input = input::ParserInput::new(text, tokens);
- let parser_impl = ParserImpl::new(&input);
+ let parser_impl = ParserImpl::new(input);
let mut parser_api = Parser(parser_impl);
parser(&mut parser_api);
parser_api.0.into_events()
/// Implementation details of `Parser`, extracted
/// to a separate struct in order not to pollute
/// the public API of the `Parser`.
-pub(super) struct ParserImpl<'t> {
- parser_input: &'t ParserInput<'t>,
+pub(super) struct ParserImpl<S> {
+ token_source: S,
pos: InputPosition,
events: Vec<Event>,
steps: Cell<u32>,
}
-impl<'t> ParserImpl<'t> {
- fn new(inp: &'t ParserInput<'t>) -> ParserImpl<'t> {
+impl<S: TokenSource> ParserImpl<S> {
+ fn new(token_source: S) -> ParserImpl<S> {
ParserImpl {
- parser_input: inp,
+ token_source,
pos: InputPosition::new(),
events: Vec::new(),
steps: Cell::new(0),
}
pub(super) fn current2(&self) -> Option<(SyntaxKind, SyntaxKind)> {
- let c1 = self.parser_input.kind(self.pos);
- let c2 = self.parser_input.kind(self.pos + 1);
- if self.parser_input.token_start_at(self.pos + 1)
- == self.parser_input.token_start_at(self.pos) + self.parser_input.token_len(self.pos)
- {
+ let c1 = self.token_source.token_kind(self.pos);
+ let c2 = self.token_source.token_kind(self.pos + 1);
+ if self.token_source.is_token_joint_to_next(self.pos) {
Some((c1, c2))
} else {
None
}
pub(super) fn current3(&self) -> Option<(SyntaxKind, SyntaxKind, SyntaxKind)> {
- let c1 = self.parser_input.kind(self.pos);
- let c2 = self.parser_input.kind(self.pos + 1);
- let c3 = self.parser_input.kind(self.pos + 2);
- if self.parser_input.token_start_at(self.pos + 1)
- == self.parser_input.token_start_at(self.pos) + self.parser_input.token_len(self.pos)
- && self.parser_input.token_start_at(self.pos + 2)
- == self.parser_input.token_start_at(self.pos + 1)
- + self.parser_input.token_len(self.pos + 1)
+ let c1 = self.token_source.token_kind(self.pos);
+ let c2 = self.token_source.token_kind(self.pos + 1);
+ let c3 = self.token_source.token_kind(self.pos + 2);
+ if self.token_source.is_token_joint_to_next(self.pos)
+ && self.token_source.is_token_joint_to_next(self.pos + 1)
{
Some((c1, c2, c3))
} else {
let steps = self.steps.get();
assert!(steps <= 10_000_000, "the parser seems stuck");
self.steps.set(steps + 1);
-
- self.parser_input.kind(self.pos + n)
+ self.token_source.token_kind(self.pos + n)
}
- pub(super) fn at_kw(&self, t: &str) -> bool {
- self.parser_input.token_text(self.pos) == t
+ pub(super) fn at_kw(&self, kw: &str) -> bool {
+ self.token_source.is_keyword(self.pos, kw)
}
/// Start parsing right behind the last event.
use crate::{
SyntaxKind, SyntaxKind::EOF, TextRange, TextUnit,
- parsing::lexer::Token,
+ parsing::{
+ parser_impl::TokenSource,
+ lexer::Token,
+ },
};
use std::ops::{Add, AddAssign};
+impl<'t> TokenSource for ParserInput<'t> {
+ fn token_kind(&self, pos: InputPosition) -> SyntaxKind {
+ let idx = pos.0 as usize;
+ if !(idx < self.tokens.len()) {
+ return EOF;
+ }
+ self.tokens[idx].kind
+ }
+ fn is_token_joint_to_next(&self, pos: InputPosition) -> bool {
+ let idx_curr = pos.0 as usize;
+ let idx_next = pos.0 as usize;
+ if !(idx_next < self.tokens.len()) {
+ return true;
+ }
+ self.start_offsets[idx_curr] + self.tokens[idx_curr].len == self.start_offsets[idx_next]
+ }
+ fn is_keyword(&self, pos: InputPosition, kw: &str) -> bool {
+ let idx = pos.0 as usize;
+ if !(idx < self.tokens.len()) {
+ return false;
+ }
+ let range = TextRange::offset_len(self.start_offsets[idx], self.tokens[idx].len);
+
+ self.text[range] == *kw
+ }
+}
+
pub(crate) struct ParserInput<'t> {
text: &'t str,
/// start position of each token(expect whitespace and comment)
ParserInput { text, start_offsets, tokens }
}
-
- /// Get the syntax kind of token at given input position.
- pub fn kind(&self, pos: InputPosition) -> SyntaxKind {
- let idx = pos.0 as usize;
- if !(idx < self.tokens.len()) {
- return EOF;
- }
- self.tokens[idx].kind
- }
-
- /// Get the length of a token at given input position.
- pub fn token_len(&self, pos: InputPosition) -> TextUnit {
- let idx = pos.0 as usize;
- if !(idx < self.tokens.len()) {
- return 0.into();
- }
- self.tokens[idx].len
- }
-
- /// Get the start position of a taken at given input position.
- pub fn token_start_at(&self, pos: InputPosition) -> TextUnit {
- let idx = pos.0 as usize;
- if !(idx < self.tokens.len()) {
- return 0.into();
- }
- self.start_offsets[idx]
- }
-
- /// Get the raw text of a token at given input position.
- pub fn token_text(&self, pos: InputPosition) -> &'t str {
- let idx = pos.0 as usize;
- if !(idx < self.tokens.len()) {
- return "";
- }
- let range = TextRange::offset_len(self.start_offsets[idx], self.tokens[idx].len);
- &self.text[range]
- }
}
#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]