use rustc_ast as ast;
use rustc_ast::token;
-use rustc_ast::tokenstream::{self, CanSynthesizeMissingTokens};
-use rustc_ast::tokenstream::{DelimSpan, Spacing::*, TokenStream, TreeAndSpacing};
+use rustc_ast::tokenstream::{self, DelimSpan, Spacing::*, TokenStream, TreeAndSpacing};
use rustc_ast_pretty::pprust;
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::sync::Lrc;
use rustc_errors::{Diagnostic, MultiSpan, PResult};
use rustc_parse::lexer::nfc_normalize;
-use rustc_parse::{nt_to_tokenstream, parse_stream_from_source_str};
+use rustc_parse::parse_stream_from_source_str;
use rustc_session::parse::ParseSess;
use rustc_span::def_id::CrateNum;
use rustc_span::symbol::{self, kw, sym, Symbol};
TokenTree::Ident(Ident::new(rustc.sess(), ident.name, is_raw, ident.span))
}
Interpolated(nt) => {
- let stream = nt_to_tokenstream(&nt, rustc.sess(), CanSynthesizeMissingTokens::No);
TokenTree::Group(Group {
delimiter: pm::Delimiter::None,
- stream,
+ stream: TokenStream::from_nonterminal_ast(&nt),
span: DelimSpan::from_single(span),
flatten: crate::base::nt_pretty_printing_compatibility_hack(&nt, rustc.sess()),
})
pub struct FreeFunctions;
-#[derive(Clone)]
-pub struct TokenStreamIter {
- cursor: tokenstream::Cursor,
- stack: Vec<TokenTree<Group, Punct, Ident, Literal>>,
-}
-
#[derive(Clone)]
pub struct Group {
delimiter: Delimiter,
impl server::Types for Rustc<'_, '_> {
type FreeFunctions = FreeFunctions;
type TokenStream = TokenStream;
- type TokenStreamBuilder = tokenstream::TokenStreamBuilder;
- type TokenStreamIter = TokenStreamIter;
type Group = Group;
type Punct = Punct;
type Ident = Ident;
}
impl server::TokenStream for Rustc<'_, '_> {
- fn new(&mut self) -> Self::TokenStream {
- TokenStream::default()
- }
fn is_empty(&mut self, stream: &Self::TokenStream) -> bool {
stream.is_empty()
}
// NOTE: For now, limit `expand_expr` to exclusively expand to literals.
// This may be relaxed in the future.
- // We don't use `nt_to_tokenstream` as the tokenstream currently cannot
+ // We don't use `TokenStream::from_ast` as the tokenstream currently cannot
// be recovered in the general case.
match &expr.kind {
ast::ExprKind::Lit(l) => {
) -> Self::TokenStream {
tree.to_internal()
}
- fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter {
- TokenStreamIter { cursor: stream.trees(), stack: vec![] }
- }
-}
-
-impl server::TokenStreamBuilder for Rustc<'_, '_> {
- fn new(&mut self) -> Self::TokenStreamBuilder {
- tokenstream::TokenStreamBuilder::new()
- }
- fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) {
- builder.push(stream);
+ fn concat_trees(
+ &mut self,
+ base: Option<Self::TokenStream>,
+ trees: Vec<TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>>,
+ ) -> Self::TokenStream {
+ let mut builder = tokenstream::TokenStreamBuilder::new();
+ if let Some(base) = base {
+ builder.push(base);
+ }
+ for tree in trees {
+ builder.push(tree.to_internal());
+ }
+ builder.build()
}
- fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream {
+ fn concat_streams(
+ &mut self,
+ base: Option<Self::TokenStream>,
+ streams: Vec<Self::TokenStream>,
+ ) -> Self::TokenStream {
+ let mut builder = tokenstream::TokenStreamBuilder::new();
+ if let Some(base) = base {
+ builder.push(base);
+ }
+ for stream in streams {
+ builder.push(stream);
+ }
builder.build()
}
-}
-
-impl server::TokenStreamIter for Rustc<'_, '_> {
- fn next(
+ fn into_trees(
&mut self,
- iter: &mut Self::TokenStreamIter,
- ) -> Option<TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>> {
+ stream: Self::TokenStream,
+ ) -> Vec<TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>> {
+ // FIXME: This is a raw port of the previous approach (which had a
+ // `TokenStreamIter` server-side object with a single `next` method),
+ // and can probably be optimized (for bulk conversion).
+ let mut cursor = stream.into_trees();
+ let mut stack = Vec::new();
+ let mut tts = Vec::new();
loop {
- let tree = iter.stack.pop().or_else(|| {
- let next = iter.cursor.next_with_spacing()?;
- Some(TokenTree::from_internal((next, &mut iter.stack, self)))
- })?;
- // A hack used to pass AST fragments to attribute and derive macros
- // as a single nonterminal token instead of a token stream.
- // Such token needs to be "unwrapped" and not represented as a delimited group.
- // FIXME: It needs to be removed, but there are some compatibility issues (see #73345).
- if let TokenTree::Group(ref group) = tree {
- if group.flatten {
- iter.cursor.append(group.stream.clone());
- continue;
+ let next = stack.pop().or_else(|| {
+ let next = cursor.next_with_spacing()?;
+ Some(TokenTree::from_internal((next, &mut stack, self)))
+ });
+ match next {
+ Some(TokenTree::Group(group)) => {
+ // A hack used to pass AST fragments to attribute and derive
+ // macros as a single nonterminal token instead of a token
+ // stream. Such token needs to be "unwrapped" and not
+ // represented as a delimited group.
+ // FIXME: It needs to be removed, but there are some
+ // compatibility issues (see #73345).
+ if group.flatten {
+ cursor.append(group.stream);
+ continue;
+ }
+ tts.push(TokenTree::Group(group));
}
+ Some(tt) => tts.push(tt),
+ None => return tts,
}
- return Some(tree);
}
}
}
impl server::Group for Rustc<'_, '_> {
- fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group {
+ fn new(&mut self, delimiter: Delimiter, stream: Option<Self::TokenStream>) -> Self::Group {
Group {
delimiter,
- stream,
+ stream: stream.unwrap_or_default(),
span: DelimSpan::from_single(server::Span::call_site(self)),
flatten: false,
}
projects / rust.git / blobdiff