10995: internal: switch from trait-based TokenSource to simple struct of arrays r=matklad a=matklad
cc #10765
The idea here is to try to simplify the interface as best as we can. The original trait-based approach is a bit over-engineered and hard to debug. Here, we replace callback with just data. The next PR in series will replace the output `TreeSink` trait with data as well.
The biggest drawback here is that we now require to materialize all parser's input up-front. This is a bad fit for macro by example: when you parse `$e:expr`, you might consume only part of the input. However, today's trait-based solution doesn't really help -- we were already materializing the whole thing! So, let's keep it simple!
Co-authored-by: Aleksey Kladov <aleksey.kladov@gmail.com>
mod expander;
mod syntax_bridge;
mod tt_iter;
-mod subtree_source;
+mod to_parser_tokens;
#[cfg(test)]
mod benchmark;
+++ /dev/null
-//! Our parser is generic over the source of tokens it parses.
-//!
-//! This module defines tokens sourced from declarative macros.
-
-use parser::{Token, TokenSource};
-use syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T};
-use tt::buffer::TokenBuffer;
-
-#[derive(Debug, Clone, Eq, PartialEq)]
-struct TtToken {
- tt: Token,
- text: SmolStr,
-}
-
-pub(crate) struct SubtreeTokenSource {
- cached: Vec<TtToken>,
- curr: (Token, usize),
-}
-
-impl<'a> SubtreeTokenSource {
- pub(crate) fn new(buffer: &TokenBuffer) -> SubtreeTokenSource {
- let mut current = buffer.begin();
- let mut cached = Vec::with_capacity(100);
-
- while !current.eof() {
- let cursor = current;
- let tt = cursor.token_tree();
-
- // Check if it is lifetime
- if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(punct), _)) = tt {
- if punct.char == '\'' {
- let next = cursor.bump();
- if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Ident(ident), _)) =
- next.token_tree()
- {
- let text = SmolStr::new("'".to_string() + &ident.text);
- cached.push(TtToken {
- tt: Token { kind: LIFETIME_IDENT, is_jointed_to_next: false },
- text,
- });
- current = next.bump();
- continue;
- } else {
- panic!("Next token must be ident : {:#?}", next.token_tree());
- }
- }
- }
-
- current = match tt {
- Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
- cached.push(convert_leaf(leaf));
- cursor.bump()
- }
- Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
- if let Some(d) = subtree.delimiter_kind() {
- cached.push(convert_delim(d, false));
- }
- cursor.subtree().unwrap()
- }
- None => match cursor.end() {
- Some(subtree) => {
- if let Some(d) = subtree.delimiter_kind() {
- cached.push(convert_delim(d, true));
- }
- cursor.bump()
- }
- None => continue,
- },
- };
- }
-
- let mut res = SubtreeTokenSource {
- curr: (Token { kind: EOF, is_jointed_to_next: false }, 0),
- cached,
- };
- res.curr = (res.token(0), 0);
- res
- }
-
- fn token(&self, pos: usize) -> Token {
- match self.cached.get(pos) {
- Some(it) => it.tt,
- None => Token { kind: EOF, is_jointed_to_next: false },
- }
- }
-}
-
-impl<'a> TokenSource for SubtreeTokenSource {
- fn current(&self) -> Token {
- self.curr.0
- }
-
- /// Lookahead n token
- fn lookahead_nth(&self, n: usize) -> Token {
- self.token(self.curr.1 + n)
- }
-
- /// bump cursor to next token
- fn bump(&mut self) {
- if self.current().kind == EOF {
- return;
- }
- self.curr = (self.token(self.curr.1 + 1), self.curr.1 + 1);
- }
-
- /// Is the current token a specified keyword?
- fn is_keyword(&self, kw: &str) -> bool {
- match self.cached.get(self.curr.1) {
- Some(t) => t.text == *kw,
- None => false,
- }
- }
-}
-
-fn convert_delim(d: tt::DelimiterKind, closing: bool) -> TtToken {
- let (kinds, texts) = match d {
- tt::DelimiterKind::Parenthesis => ([T!['('], T![')']], "()"),
- tt::DelimiterKind::Brace => ([T!['{'], T!['}']], "{}"),
- tt::DelimiterKind::Bracket => ([T!['['], T![']']], "[]"),
- };
-
- let idx = closing as usize;
- let kind = kinds[idx];
- let text = &texts[idx..texts.len() - (1 - idx)];
- TtToken { tt: Token { kind, is_jointed_to_next: false }, text: SmolStr::new(text) }
-}
-
-fn convert_literal(l: &tt::Literal) -> TtToken {
- let is_negated = l.text.starts_with('-');
- let inner_text = &l.text[if is_negated { 1 } else { 0 }..];
-
- let kind = lex_single_syntax_kind(inner_text)
- .map(|(kind, _error)| kind)
- .filter(|kind| {
- kind.is_literal() && (!is_negated || matches!(kind, FLOAT_NUMBER | INT_NUMBER))
- })
- .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l));
-
- TtToken { tt: Token { kind, is_jointed_to_next: false }, text: l.text.clone() }
-}
-
-fn convert_ident(ident: &tt::Ident) -> TtToken {
- let kind = match ident.text.as_ref() {
- "true" => T![true],
- "false" => T![false],
- "_" => UNDERSCORE,
- i if i.starts_with('\'') => LIFETIME_IDENT,
- _ => SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT),
- };
-
- TtToken { tt: Token { kind, is_jointed_to_next: false }, text: ident.text.clone() }
-}
-
-fn convert_punct(p: tt::Punct) -> TtToken {
- let kind = match SyntaxKind::from_char(p.char) {
- None => panic!("{:#?} is not a valid punct", p),
- Some(kind) => kind,
- };
-
- let text = {
- let mut buf = [0u8; 4];
- let s: &str = p.char.encode_utf8(&mut buf);
- SmolStr::new(s)
- };
- TtToken { tt: Token { kind, is_jointed_to_next: p.spacing == tt::Spacing::Joint }, text }
-}
-
-fn convert_leaf(leaf: &tt::Leaf) -> TtToken {
- match leaf {
- tt::Leaf::Literal(l) => convert_literal(l),
- tt::Leaf::Ident(ident) => convert_ident(ident),
- tt::Leaf::Punct(punct) => convert_punct(*punct),
- }
-}
use tt::buffer::{Cursor, TokenBuffer};
use crate::{
- subtree_source::SubtreeTokenSource, tt_iter::TtIter, ExpandError, ParserEntryPoint, TokenMap,
+ to_parser_tokens::to_parser_tokens, tt_iter::TtIter, ExpandError, ParserEntryPoint, TokenMap,
};
/// Convert the syntax node to a `TokenTree` (what macro
}
_ => TokenBuffer::from_subtree(tt),
};
- let mut token_source = SubtreeTokenSource::new(&buffer);
+ let parser_tokens = to_parser_tokens(&buffer);
let mut tree_sink = TtTreeSink::new(buffer.begin());
- parser::parse(&mut token_source, &mut tree_sink, entry_point);
+ parser::parse(&parser_tokens, &mut tree_sink, entry_point);
if tree_sink.roots.len() != 1 {
return Err(ExpandError::ConversionError);
}
--- /dev/null
+//! Convert macro-by-example tokens which are specific to macro expansion into a
+//! format that works for our parser.
+
+use syntax::{lex_single_syntax_kind, SyntaxKind, SyntaxKind::*, T};
+use tt::buffer::TokenBuffer;
+
+pub(crate) fn to_parser_tokens(buffer: &TokenBuffer) -> parser::Tokens {
+ let mut res = parser::Tokens::default();
+
+ let mut current = buffer.begin();
+
+ while !current.eof() {
+ let cursor = current;
+ let tt = cursor.token_tree();
+
+ // Check if it is lifetime
+ if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(punct), _)) = tt {
+ if punct.char == '\'' {
+ let next = cursor.bump();
+ match next.token_tree() {
+ Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Ident(_ident), _)) => {
+ res.push(LIFETIME_IDENT);
+ current = next.bump();
+ continue;
+ }
+ _ => panic!("Next token must be ident : {:#?}", next.token_tree()),
+ }
+ }
+ }
+
+ current = match tt {
+ Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
+ match leaf {
+ tt::Leaf::Literal(lit) => {
+ let is_negated = lit.text.starts_with('-');
+ let inner_text = &lit.text[if is_negated { 1 } else { 0 }..];
+
+ let kind = lex_single_syntax_kind(inner_text)
+ .map(|(kind, _error)| kind)
+ .filter(|kind| {
+ kind.is_literal()
+ && (!is_negated || matches!(kind, FLOAT_NUMBER | INT_NUMBER))
+ })
+ .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &lit));
+
+ res.push(kind);
+ }
+ tt::Leaf::Ident(ident) => match ident.text.as_ref() {
+ "_" => res.push(T![_]),
+ i if i.starts_with('\'') => res.push(LIFETIME_IDENT),
+ _ => match SyntaxKind::from_keyword(&ident.text) {
+ Some(kind) => res.push(kind),
+ None => {
+ let contextual_keyword =
+ SyntaxKind::from_contextual_keyword(&ident.text)
+ .unwrap_or(SyntaxKind::IDENT);
+ res.push_ident(contextual_keyword);
+ }
+ },
+ },
+ tt::Leaf::Punct(punct) => {
+ let kind = SyntaxKind::from_char(punct.char)
+ .unwrap_or_else(|| panic!("{:#?} is not a valid punct", punct));
+ res.push(kind);
+ if punct.spacing == tt::Spacing::Joint {
+ res.was_joint();
+ }
+ }
+ }
+ cursor.bump()
+ }
+ Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
+ if let Some(d) = subtree.delimiter_kind() {
+ res.push(match d {
+ tt::DelimiterKind::Parenthesis => T!['('],
+ tt::DelimiterKind::Brace => T!['{'],
+ tt::DelimiterKind::Bracket => T!['['],
+ });
+ }
+ cursor.subtree().unwrap()
+ }
+ None => match cursor.end() {
+ Some(subtree) => {
+ if let Some(d) = subtree.delimiter_kind() {
+ res.push(match d {
+ tt::DelimiterKind::Parenthesis => T![')'],
+ tt::DelimiterKind::Brace => T!['}'],
+ tt::DelimiterKind::Bracket => T![']'],
+ })
+ }
+ cursor.bump()
+ }
+ None => continue,
+ },
+ };
+ }
+
+ res
+}
//! A "Parser" structure for token trees. We use this when parsing a declarative
//! macro definition into a list of patterns and templates.
-use crate::{subtree_source::SubtreeTokenSource, ExpandError, ExpandResult, ParserEntryPoint};
+use crate::{to_parser_tokens::to_parser_tokens, ExpandError, ExpandResult, ParserEntryPoint};
use parser::TreeSink;
use syntax::SyntaxKind;
}
let buffer = TokenBuffer::from_tokens(self.inner.as_slice());
- let mut src = SubtreeTokenSource::new(&buffer);
+ let parser_tokens = to_parser_tokens(&buffer);
let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
- parser::parse(&mut src, &mut sink, entry_point);
+ parser::parse(&parser_tokens, &mut sink, entry_point);
let mut err = if !sink.cursor.is_root() || sink.error {
Some(err!("expected {:?}", entry_point))
T![&] => {
m = p.start();
p.bump(T![&]);
- if p.at(IDENT)
- && p.at_contextual_kw("raw")
- && (p.nth_at(1, T![mut]) || p.nth_at(1, T![const]))
- {
+ if p.at_contextual_kw(T![raw]) && (p.nth_at(1, T![mut]) || p.nth_at(1, T![const])) {
p.bump_remap(T![raw]);
p.bump_any();
} else {
has_mods = true;
abi(p);
}
- if p.at(IDENT) && p.at_contextual_kw("auto") && p.nth(1) == T![trait] {
+ if p.at_contextual_kw(T![auto]) && p.nth(1) == T![trait] {
p.bump_remap(T![auto]);
has_mods = true;
}
// test default_item
// default impl T for Foo {}
- if p.at(IDENT) && p.at_contextual_kw("default") {
+ if p.at_contextual_kw(T![default]) {
match p.nth(1) {
T![fn] | T![type] | T![const] | T![impl] => {
p.bump_remap(T![default]);
// test existential_type
// existential type Foo: Fn() -> usize;
- if p.at(IDENT) && p.at_contextual_kw("existential") && p.nth(1) == T![type] {
+ if p.at_contextual_kw(T![existential]) && p.nth(1) == T![type] {
p.bump_remap(T![existential]);
has_mods = true;
}
T![type] => type_alias(p, m),
T![struct] => adt::strukt(p, m),
T![enum] => adt::enum_(p, m),
- IDENT if p.at_contextual_kw("union") && p.nth(1) == IDENT => adt::union(p, m),
+ IDENT if p.at_contextual_kw(T![union]) && p.nth(1) == IDENT => adt::union(p, m),
T![macro] => macro_def(p, m),
- IDENT if p.at_contextual_kw("macro_rules") && p.nth(1) == BANG => macro_rules(p, m),
+ IDENT if p.at_contextual_kw(T![macro_rules]) && p.nth(1) == BANG => macro_rules(p, m),
T![const] if (la == IDENT || la == T![_] || la == T![mut]) => consts::konst(p, m),
T![static] => consts::static_(p, m),
}
fn macro_rules(p: &mut Parser, m: Marker) {
- assert!(p.at_contextual_kw("macro_rules"));
+ assert!(p.at_contextual_kw(T![macro_rules]));
p.bump_remap(T![macro_rules]);
p.expect(T![!]);
// test union_item
// struct U { i: i32, f: f32 }
pub(super) fn union(p: &mut Parser, m: Marker) {
- assert!(p.at_contextual_kw("union"));
+ assert!(p.at_contextual_kw(T![union]));
p.bump_remap(T![union]);
struct_or_union(p, m, false);
}
//! The Rust parser.
//!
+//! NOTE: The crate is undergoing refactors, don't believe everything the docs
+//! say :-)
+//!
//! The parser doesn't know about concrete representation of tokens and syntax
-//! trees. Abstract [`TokenSource`] and [`TreeSink`] traits are used instead.
-//! As a consequence, this crate does not contain a lexer.
+//! trees. Abstract [`TokenSource`] and [`TreeSink`] traits are used instead. As
+//! a consequence, this crate does not contain a lexer.
//!
//! The [`Parser`] struct from the [`parser`] module is a cursor into the
//! sequence of tokens. Parsing routines use [`Parser`] to inspect current
mod event;
mod parser;
mod grammar;
+mod tokens;
pub(crate) use token_set::TokenSet;
-pub use syntax_kind::SyntaxKind;
+pub use crate::{syntax_kind::SyntaxKind, tokens::Tokens};
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct ParseError(pub Box<String>);
-/// `TokenSource` abstracts the source of the tokens parser operates on.
-///
-/// Hopefully this will allow us to treat text and token trees in the same way!
-pub trait TokenSource {
- fn current(&self) -> Token;
-
- /// Lookahead n token
- fn lookahead_nth(&self, n: usize) -> Token;
-
- /// bump cursor to next token
- fn bump(&mut self);
-
- /// Is the current token a specified keyword?
- fn is_keyword(&self, kw: &str) -> bool;
-}
-
-/// `Token` abstracts the cursor of `TokenSource` operates on.
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
-pub struct Token {
- /// What is the current token?
- pub kind: SyntaxKind,
-
- /// Is the current token joined to the next one (`> >` vs `>>`).
- pub is_jointed_to_next: bool,
-}
-
/// `TreeSink` abstracts details of a particular syntax tree implementation.
pub trait TreeSink {
/// Adds new token to the current branch.
}
/// Parse given tokens into the given sink as a rust file.
-pub fn parse_source_file(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) {
- parse(token_source, tree_sink, ParserEntryPoint::SourceFile);
+pub fn parse_source_file(tokens: &Tokens, tree_sink: &mut dyn TreeSink) {
+ parse(tokens, tree_sink, ParserEntryPoint::SourceFile);
}
-pub fn parse(
- token_source: &mut dyn TokenSource,
- tree_sink: &mut dyn TreeSink,
- entry_point: ParserEntryPoint,
-) {
+pub fn parse(tokens: &Tokens, tree_sink: &mut dyn TreeSink, entry_point: ParserEntryPoint) {
let entry_point: fn(&'_ mut parser::Parser) = match entry_point {
ParserEntryPoint::SourceFile => grammar::entry_points::source_file,
ParserEntryPoint::Path => grammar::entry_points::path,
ParserEntryPoint::Attr => grammar::entry_points::attr,
};
- let mut p = parser::Parser::new(token_source);
+ let mut p = parser::Parser::new(tokens);
entry_point(&mut p);
let events = p.finish();
event::process(tree_sink, events);
///
/// Tokens must start with `{`, end with `}` and form a valid brace
/// sequence.
- pub fn parse(self, token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+ pub fn parse(self, tokens: &Tokens, tree_sink: &mut dyn TreeSink) {
let Reparser(r) = self;
- let mut p = parser::Parser::new(token_source);
+ let mut p = parser::Parser::new(tokens);
r(&mut p);
let events = p.finish();
event::process(tree_sink, events);
use crate::{
event::Event,
+ tokens::Tokens,
ParseError,
SyntaxKind::{self, EOF, ERROR, TOMBSTONE},
- TokenSet, TokenSource, T,
+ TokenSet, T,
};
/// `Parser` struct provides the low-level API for
/// "start expression, consume number literal,
/// finish expression". See `Event` docs for more.
pub(crate) struct Parser<'t> {
- token_source: &'t mut dyn TokenSource,
+ tokens: &'t Tokens,
+ pos: usize,
events: Vec<Event>,
steps: Cell<u32>,
}
static PARSER_STEP_LIMIT: Limit = Limit::new(15_000_000);
impl<'t> Parser<'t> {
- pub(super) fn new(token_source: &'t mut dyn TokenSource) -> Parser<'t> {
- Parser { token_source, events: Vec::new(), steps: Cell::new(0) }
+ pub(super) fn new(tokens: &'t Tokens) -> Parser<'t> {
+ Parser { tokens, pos: 0, events: Vec::new(), steps: Cell::new(0) }
}
pub(crate) fn finish(self) -> Vec<Event> {
assert!(PARSER_STEP_LIMIT.check(steps as usize).is_ok(), "the parser seems stuck");
self.steps.set(steps + 1);
- self.token_source.lookahead_nth(n).kind
+ self.tokens.kind(self.pos + n)
}
/// Checks if the current token is `kind`.
T![<<=] => self.at_composite3(n, T![<], T![<], T![=]),
T![>>=] => self.at_composite3(n, T![>], T![>], T![=]),
- _ => self.token_source.lookahead_nth(n).kind == kind,
+ _ => self.tokens.kind(self.pos + n) == kind,
}
}
}
fn at_composite2(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind) -> bool {
- let t1 = self.token_source.lookahead_nth(n);
- if t1.kind != k1 || !t1.is_jointed_to_next {
- return false;
- }
- let t2 = self.token_source.lookahead_nth(n + 1);
- t2.kind == k2
+ self.tokens.kind(self.pos + n) == k1
+ && self.tokens.kind(self.pos + n + 1) == k2
+ && self.tokens.is_joint(self.pos + n)
}
fn at_composite3(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind, k3: SyntaxKind) -> bool {
- let t1 = self.token_source.lookahead_nth(n);
- if t1.kind != k1 || !t1.is_jointed_to_next {
- return false;
- }
- let t2 = self.token_source.lookahead_nth(n + 1);
- if t2.kind != k2 || !t2.is_jointed_to_next {
- return false;
- }
- let t3 = self.token_source.lookahead_nth(n + 2);
- t3.kind == k3
+ self.tokens.kind(self.pos + n) == k1
+ && self.tokens.kind(self.pos + n + 1) == k2
+ && self.tokens.kind(self.pos + n + 2) == k3
+ && self.tokens.is_joint(self.pos + n)
+ && self.tokens.is_joint(self.pos + n + 1)
}
/// Checks if the current token is in `kinds`.
}
/// Checks if the current token is contextual keyword with text `t`.
- pub(crate) fn at_contextual_kw(&self, kw: &str) -> bool {
- self.token_source.is_keyword(kw)
+ pub(crate) fn at_contextual_kw(&self, kw: SyntaxKind) -> bool {
+ self.tokens.contextual_kind(self.pos) == kw
}
/// Starts a new node in the syntax tree. All nodes and tokens
}
fn do_bump(&mut self, kind: SyntaxKind, n_raw_tokens: u8) {
- for _ in 0..n_raw_tokens {
- self.token_source.bump();
- }
-
+ self.pos += n_raw_tokens as usize;
self.push_event(Event::Token { kind, n_raw_tokens });
}
};
Some(kw)
}
+ pub fn from_contextual_keyword(ident: &str) -> Option<SyntaxKind> {
+ let kw = match ident {
+ "auto" => AUTO_KW,
+ "default" => DEFAULT_KW,
+ "existential" => EXISTENTIAL_KW,
+ "union" => UNION_KW,
+ "raw" => RAW_KW,
+ "macro_rules" => MACRO_RULES_KW,
+ _ => return None,
+ };
+ Some(kw)
+ }
pub fn from_char(c: char) -> Option<SyntaxKind> {
let tok = match c {
';' => SEMICOLON,
--- /dev/null
+//! Input for the parser -- a sequence of tokens.
+//!
+//! As of now, parser doesn't have access to the *text* of the tokens, and makes
+//! decisions based solely on their classification.
+
+use crate::SyntaxKind;
+
+#[allow(non_camel_case_types)]
+type bits = u64;
+
+/// Main input to the parser.
+///
+/// A sequence of tokens represented internally as a struct of arrays.
+#[derive(Default)]
+pub struct Tokens {
+ kind: Vec<SyntaxKind>,
+ joint: Vec<bits>,
+ contextual_kind: Vec<SyntaxKind>,
+}
+
+/// `pub` impl used by callers to create `Tokens`.
+impl Tokens {
+ #[inline]
+ pub fn push(&mut self, kind: SyntaxKind) {
+ self.push_impl(kind, SyntaxKind::EOF)
+ }
+ #[inline]
+ pub fn push_ident(&mut self, contextual_kind: SyntaxKind) {
+ self.push_impl(SyntaxKind::IDENT, contextual_kind)
+ }
+ /// Sets jointness for the last token we've pushed.
+ ///
+ /// This is a separate API rather than an argument to the `push` to make it
+ /// convenient both for textual and mbe tokens. With text, you know whether
+ /// the *previous* token was joint, with mbe, you know whether the *current*
+ /// one is joint. This API allows for styles of usage:
+ ///
+ /// ```
+ /// // In text:
+ /// tokens.was_joint(prev_joint);
+ /// tokens.push(curr);
+ ///
+ /// // In MBE:
+ /// token.push(curr);
+ /// tokens.push(curr_joint)
+ /// ```
+ #[inline]
+ pub fn was_joint(&mut self) {
+ let n = self.len() - 1;
+ let (idx, b_idx) = self.bit_index(n);
+ self.joint[idx] |= 1 << b_idx;
+ }
+ #[inline]
+ fn push_impl(&mut self, kind: SyntaxKind, contextual_kind: SyntaxKind) {
+ let idx = self.len();
+ if idx % (bits::BITS as usize) == 0 {
+ self.joint.push(0);
+ }
+ self.kind.push(kind);
+ self.contextual_kind.push(contextual_kind);
+ }
+}
+
+/// pub(crate) impl used by the parser to consume `Tokens`.
+impl Tokens {
+ pub(crate) fn kind(&self, idx: usize) -> SyntaxKind {
+ self.kind.get(idx).copied().unwrap_or(SyntaxKind::EOF)
+ }
+ pub(crate) fn contextual_kind(&self, idx: usize) -> SyntaxKind {
+ self.contextual_kind.get(idx).copied().unwrap_or(SyntaxKind::EOF)
+ }
+ pub(crate) fn is_joint(&self, n: usize) -> bool {
+ let (idx, b_idx) = self.bit_index(n);
+ self.joint[idx] & 1 << b_idx != 0
+ }
+}
+
+impl Tokens {
+ fn bit_index(&self, n: usize) -> (usize, usize) {
+ let idx = n / (bits::BITS as usize);
+ let b_idx = n % (bits::BITS as usize);
+ (idx, b_idx)
+ }
+ fn len(&self) -> usize {
+ self.kind.len()
+ }
+}
//! incremental reparsing.
pub(crate) mod lexer;
-mod text_token_source;
mod text_tree_sink;
mod reparsing;
use parser::SyntaxKind;
-use text_token_source::TextTokenSource;
use text_tree_sink::TextTreeSink;
use crate::{syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode};
pub(crate) use crate::parsing::{lexer::*, reparsing::incremental_reparse};
pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) {
- let (tokens, lexer_errors) = tokenize(text);
+ let (lexer_tokens, lexer_errors) = tokenize(text);
+ let parser_tokens = to_parser_tokens(text, &lexer_tokens);
- let mut token_source = TextTokenSource::new(text, &tokens);
- let mut tree_sink = TextTreeSink::new(text, &tokens);
+ let mut tree_sink = TextTreeSink::new(text, &lexer_tokens);
- parser::parse_source_file(&mut token_source, &mut tree_sink);
+ parser::parse_source_file(&parser_tokens, &mut tree_sink);
let (tree, mut parser_errors) = tree_sink.finish();
parser_errors.extend(lexer_errors);
text: &str,
entry_point: parser::ParserEntryPoint,
) -> Result<T, ()> {
- let (tokens, lexer_errors) = tokenize(text);
+ let (lexer_tokens, lexer_errors) = tokenize(text);
if !lexer_errors.is_empty() {
return Err(());
}
- let mut token_source = TextTokenSource::new(text, &tokens);
- let mut tree_sink = TextTreeSink::new(text, &tokens);
+ let parser_tokens = to_parser_tokens(text, &lexer_tokens);
+
+ let mut tree_sink = TextTreeSink::new(text, &lexer_tokens);
// TextTreeSink assumes that there's at least some root node to which it can attach errors and
// tokens. We arbitrarily give it a SourceFile.
use parser::TreeSink;
tree_sink.start_node(SyntaxKind::SOURCE_FILE);
- parser::parse(&mut token_source, &mut tree_sink, entry_point);
+ parser::parse(&parser_tokens, &mut tree_sink, entry_point);
tree_sink.finish_node();
- let (tree, parser_errors) = tree_sink.finish();
- use parser::TokenSource;
- if !parser_errors.is_empty() || token_source.current().kind != SyntaxKind::EOF {
+ let (tree, parser_errors, eof) = tree_sink.finish_eof();
+ if !parser_errors.is_empty() || !eof {
return Err(());
}
SyntaxNode::new_root(tree).first_child().and_then(T::cast).ok_or(())
}
+
+pub(crate) fn to_parser_tokens(text: &str, lexer_tokens: &[lexer::Token]) -> ::parser::Tokens {
+ let mut off = 0;
+ let mut res = parser::Tokens::default();
+ let mut was_joint = false;
+ for t in lexer_tokens {
+ if t.kind.is_trivia() {
+ was_joint = false;
+ } else {
+ if t.kind == SyntaxKind::IDENT {
+ let token_text = &text[off..][..usize::from(t.len)];
+ let contextual_kw =
+ SyntaxKind::from_contextual_keyword(token_text).unwrap_or(SyntaxKind::IDENT);
+ res.push_ident(contextual_kw);
+ } else {
+ if was_joint {
+ res.was_joint();
+ }
+ res.push(t.kind);
+ }
+ was_joint = true;
+ }
+ off += usize::from(t.len);
+ }
+ res
+}
use crate::{
parsing::{
lexer::{lex_single_syntax_kind, tokenize, Token},
- text_token_source::TextTokenSource,
text_tree_sink::TextTreeSink,
+ to_parser_tokens,
},
syntax_node::{GreenNode, GreenToken, NodeOrToken, SyntaxElement, SyntaxNode},
SyntaxError,
let (node, reparser) = find_reparsable_node(root, edit.delete)?;
let text = get_text_after_edit(node.clone().into(), edit);
- let (tokens, new_lexer_errors) = tokenize(&text);
- if !is_balanced(&tokens) {
+ let (lexer_tokens, new_lexer_errors) = tokenize(&text);
+ if !is_balanced(&lexer_tokens) {
return None;
}
+ let parser_tokens = to_parser_tokens(&text, &lexer_tokens);
- let mut token_source = TextTokenSource::new(&text, &tokens);
- let mut tree_sink = TextTreeSink::new(&text, &tokens);
- reparser.parse(&mut token_source, &mut tree_sink);
+ let mut tree_sink = TextTreeSink::new(&text, &lexer_tokens);
+ reparser.parse(&parser_tokens, &mut tree_sink);
let (green, mut new_parser_errors) = tree_sink.finish();
new_parser_errors.extend(new_lexer_errors);
+++ /dev/null
-//! See `TextTokenSource` docs.
-
-use parser::TokenSource;
-
-use crate::{parsing::lexer::Token, SyntaxKind::EOF, TextRange, TextSize};
-
-/// Implementation of `parser::TokenSource` that takes tokens from source code text.
-pub(crate) struct TextTokenSource<'t> {
- text: &'t str,
- /// token and its start position (non-whitespace/comment tokens)
- /// ```non-rust
- /// struct Foo;
- /// ^------^--^-
- /// | | \________
- /// | \____ \
- /// | \ |
- /// (struct, 0) (Foo, 7) (;, 10)
- /// ```
- /// `[(struct, 0), (Foo, 7), (;, 10)]`
- token_offset_pairs: Vec<(Token, TextSize)>,
-
- /// Current token and position
- curr: (parser::Token, usize),
-}
-
-impl<'t> TokenSource for TextTokenSource<'t> {
- fn current(&self) -> parser::Token {
- self.curr.0
- }
-
- fn lookahead_nth(&self, n: usize) -> parser::Token {
- mk_token(self.curr.1 + n, &self.token_offset_pairs)
- }
-
- fn bump(&mut self) {
- if self.curr.0.kind == EOF {
- return;
- }
-
- let pos = self.curr.1 + 1;
- self.curr = (mk_token(pos, &self.token_offset_pairs), pos);
- }
-
- fn is_keyword(&self, kw: &str) -> bool {
- self.token_offset_pairs
- .get(self.curr.1)
- .map_or(false, |(token, offset)| &self.text[TextRange::at(*offset, token.len)] == kw)
- }
-}
-
-fn mk_token(pos: usize, token_offset_pairs: &[(Token, TextSize)]) -> parser::Token {
- let (kind, is_jointed_to_next) = match token_offset_pairs.get(pos) {
- Some((token, offset)) => (
- token.kind,
- token_offset_pairs
- .get(pos + 1)
- .map_or(false, |(_, next_offset)| offset + token.len == *next_offset),
- ),
- None => (EOF, false),
- };
- parser::Token { kind, is_jointed_to_next }
-}
-
-impl<'t> TextTokenSource<'t> {
- /// Generate input from tokens(expect comment and whitespace).
- pub(crate) fn new(text: &'t str, raw_tokens: &'t [Token]) -> TextTokenSource<'t> {
- let token_offset_pairs: Vec<_> = raw_tokens
- .iter()
- .filter_map({
- let mut len = 0.into();
- move |token| {
- let pair = if token.kind.is_trivia() { None } else { Some((*token, len)) };
- len += token.len;
- pair
- }
- })
- .collect();
-
- let first = mk_token(0, &token_offset_pairs);
- TextTokenSource { text, token_offset_pairs, curr: (first, 0) }
- }
-}
}
}
- pub(super) fn finish(mut self) -> (GreenNode, Vec<SyntaxError>) {
+ pub(super) fn finish_eof(mut self) -> (GreenNode, Vec<SyntaxError>, bool) {
match mem::replace(&mut self.state, State::Normal) {
State::PendingFinish => {
self.eat_trivias();
State::PendingStart | State::Normal => unreachable!(),
}
- self.inner.finish_raw()
+ let (node, errors) = self.inner.finish_raw();
+ let is_eof = self.token_pos == self.tokens.len();
+
+ (node, errors, is_eof)
+ }
+
+ pub(super) fn finish(self) -> (GreenNode, Vec<SyntaxError>) {
+ let (node, errors, _eof) = self.finish_eof();
+ (node, errors)
}
fn eat_trivias(&mut self) {
let full_keywords =
full_keywords_values.iter().map(|kw| format_ident!("{}_KW", to_upper_snake_case(kw)));
+ let contextual_keywords_values = &grammar.contextual_keywords;
+ let contextual_keywords =
+ contextual_keywords_values.iter().map(|kw| format_ident!("{}_KW", to_upper_snake_case(kw)));
+
let all_keywords_values =
grammar.keywords.iter().chain(grammar.contextual_keywords.iter()).collect::<Vec<_>>();
let all_keywords_idents = all_keywords_values.iter().map(|kw| format_ident!("{}", kw));
Some(kw)
}
+ pub fn from_contextual_keyword(ident: &str) -> Option<SyntaxKind> {
+ let kw = match ident {
+ #(#contextual_keywords_values => #contextual_keywords,)*
+ _ => return None,
+ };
+ Some(kw)
+ }
+
pub fn from_char(c: char) -> Option<SyntaxKind> {
let tok = match c {
#(#single_byte_tokens_values => #single_byte_tokens,)*
projects / rust.git / commitdiff