}
pub(crate) fn expect_tt(&mut self) -> Result<tt::TokenTree, ()> {
+ match self.peek_n(0) {
+ Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '\'' => {
+ return self.expect_lifetime();
+ }
+ _ => (),
+ }
+
let tt = self.next().ok_or_else(|| ())?.clone();
let punct = match tt {
tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if punct.spacing == tt::Spacing::Joint => {
}
}
- pub(crate) fn expect_lifetime(&mut self) -> Result<&tt::Ident, ()> {
- let ident = self.expect_ident()?;
- // check if it start from "`"
- if !ident.text.starts_with('\'') {
+ pub(crate) fn expect_lifetime(&mut self) -> Result<tt::TokenTree, ()> {
+ let punct = self.expect_punct()?;
+ if punct.char != '\'' {
return Err(());
}
- Ok(ident)
+ let ident = self.expect_ident()?;
+
+ Ok(tt::Subtree {
+ delimiter: None,
+ token_trees: vec![
+ tt::Leaf::Punct(punct.clone()).into(),
+ tt::Leaf::Ident(ident.clone()).into(),
+ ],
+ }
+ .into())
}
pub(crate) fn expect_fragment(
}
impl<'a> TreeSink for OffsetTokenSink<'a> {
- fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) {
+ fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
+ if kind == SyntaxKind::LIFETIME {
+ n_tokens = 2;
+ }
for _ in 0..n_tokens {
self.cursor = self.cursor.bump_subtree();
}
}
}
- let buffer = TokenBuffer::new(self.inner.as_slice());
+ let buffer = TokenBuffer::new(&self.inner.as_slice());
let mut src = SubtreeTokenSource::new(&buffer);
let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
"tt" => input.expect_tt().map(Some).map_err(|()| err!()),
"lifetime" => input
.expect_lifetime()
- .map(|ident| Some(tt::Leaf::Ident(ident.clone()).into()))
+ .map(|tt| Some(tt))
.map_err(|()| err!("expected lifetime")),
"literal" => input
.expect_literal()
}
fn get(&self, pos: usize) -> Ref<Option<TtToken>> {
+ fn is_lifetime(c: Cursor) -> Option<(Cursor, SmolStr)> {
+ let tkn = c.token_tree();
+
+ if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = tkn {
+ if punct.char == '\'' {
+ let next = c.bump();
+ if let Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) = next.token_tree() {
+ let res_cursor = next.bump();
+ let text = SmolStr::new("'".to_string() + &ident.to_string());
+
+ return Some((res_cursor, text));
+ } else {
+ panic!("Next token must be ident : {:#?}", next.token_tree());
+ }
+ }
+ }
+
+ None
+ }
+
if pos < self.cached.borrow().len() {
return Ref::map(self.cached.borrow(), |c| &c[pos]);
}
continue;
}
+ if let Some((curr, text)) = is_lifetime(cursor) {
+ cached.push(Some(TtToken { kind: LIFETIME, is_joint_to_next: false, text }));
+ self.cached_cursor.set(curr);
+ continue;
+ }
+
match cursor.token_tree() {
Some(tt::TokenTree::Leaf(leaf)) => {
cached.push(Some(convert_leaf(&leaf)));
}
fn convert_punct(p: tt::Punct) -> TtToken {
- let kind = SyntaxKind::from_char(p.char).unwrap();
+ let kind = match SyntaxKind::from_char(p.char) {
+ None => panic!("{:#?} is not a valid punct", p),
+ Some(kind) => kind,
+ };
+
let text = {
let mut buf = [0u8; 4];
let s: &str = p.char.encode_utf8(&mut buf);
inner: std::slice::Iter<'a, RawToken>,
}
-trait SrcToken {
+trait SrcToken: std::fmt::Debug {
fn kind(&self) -> SyntaxKind;
fn to_char(&self) -> Option<char>;
Some(next) if next.kind().is_punct() => tt::Spacing::Joint,
_ => tt::Spacing::Alone,
};
- let char = token.to_char().expect("Token from lexer must be single char");
-
+ let char = match token.to_char() {
+ Some(c) => c,
+ None => {
+ panic!("Token from lexer must be single char: token = {:#?}", token);
+ }
+ };
tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc().alloc(range) }).into()
}
} else {
}
let leaf: tt::Leaf = match k {
T![true] | T![false] => make_leaf!(Literal),
- IDENT | LIFETIME => make_leaf!(Ident),
+ IDENT => make_leaf!(Ident),
k if k.is_keyword() => make_leaf!(Ident),
k if k.is_literal() => make_leaf!(Literal),
+ LIFETIME => {
+ let char_unit = TextUnit::from_usize(1);
+ let r = TextRange::offset_len(range.start(), char_unit);
+ let apostrophe = tt::Leaf::from(tt::Punct {
+ char: '\'',
+ spacing: tt::Spacing::Joint,
+ id: self.id_alloc().alloc(r),
+ });
+ result.push(apostrophe.into());
+
+ let r =
+ TextRange::offset_len(range.start() + char_unit, range.len() - char_unit);
+ let ident = tt::Leaf::from(tt::Ident {
+ text: SmolStr::new(&token.to_text()[1..]),
+ id: self.id_alloc().alloc(r),
+ });
+ result.push(ident.into());
+ return;
+ }
_ => return,
};
}
}
+#[derive(Debug)]
enum SynToken {
Ordiniary(SyntaxToken),
Punch(SyntaxToken, TextUnit),
}
impl<'a> TreeSink for TtTreeSink<'a> {
- fn token(&mut self, kind: SyntaxKind, n_tokens: u8) {
+ fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
if kind == L_DOLLAR || kind == R_DOLLAR {
self.cursor = self.cursor.bump_subtree();
return;
}
+ if kind == LIFETIME {
+ n_tokens = 2;
+ }
let mut last = self.cursor;
for _ in 0..n_tokens {