1 //! The main parser interface.
3 #![feature(bool_to_option)]
4 #![feature(crate_visibility_modifier)]
5 #![feature(slice_patterns)]
8 use syntax::print::pprust;
9 use syntax::sess::ParseSess;
10 use syntax::token::{self, Nonterminal};
11 use syntax::tokenstream::{self, TokenStream, TokenTree};
13 use rustc_data_structures::sync::Lrc;
14 use rustc_errors::{Diagnostic, FatalError, Level, PResult};
15 use rustc_span::{FileName, SourceFile, Span};
23 pub const MACRO_ARGUMENTS: Option<&'static str> = Some("macro arguments");
27 use parser::{emit_unclosed_delims, make_unclosed_delims_error, Parser};
29 pub mod validate_attr;
34 pub struct Directory<'a> {
35 pub path: Cow<'a, Path>,
36 pub ownership: DirectoryOwnership,
39 #[derive(Copy, Clone)]
40 pub enum DirectoryOwnership {
42 // None if `mod.rs`, `Some("foo")` if we're in `foo.rs`.
43 relative: Option<ast::Ident>,
49 // A bunch of utility functions of the form `parse_<thing>_from_<source>`
50 // where <thing> includes crate, expr, item, stmt, tts, and one that
51 // uses a HOF to parse anything, and <source> includes file and
54 /// A variant of 'panictry!' that works on a Vec<Diagnostic> instead of a single DiagnosticBuilder.
55 macro_rules! panictry_buffer {
56 ($handler:expr, $e:expr) => {{
57 use rustc_errors::FatalError;
58 use std::result::Result::{Err, Ok};
63 $handler.emit_diagnostic(&e);
71 pub fn parse_crate_from_file<'a>(input: &Path, sess: &'a ParseSess) -> PResult<'a, ast::Crate> {
72 let mut parser = new_parser_from_file(sess, input);
73 parser.parse_crate_mod()
76 pub fn parse_crate_attrs_from_file<'a>(
79 ) -> PResult<'a, Vec<ast::Attribute>> {
80 let mut parser = new_parser_from_file(sess, input);
81 parser.parse_inner_attributes()
84 pub fn parse_crate_from_source_str(
88 ) -> PResult<'_, ast::Crate> {
89 new_parser_from_source_str(sess, name, source).parse_crate_mod()
92 pub fn parse_crate_attrs_from_source_str(
96 ) -> PResult<'_, Vec<ast::Attribute>> {
97 new_parser_from_source_str(sess, name, source).parse_inner_attributes()
100 pub fn parse_stream_from_source_str(
104 override_span: Option<Span>,
106 let (stream, mut errors) =
107 source_file_to_stream(sess, sess.source_map().new_source_file(name, source), override_span);
108 emit_unclosed_delims(&mut errors, &sess);
112 /// Creates a new parser from a source string.
113 pub fn new_parser_from_source_str(sess: &ParseSess, name: FileName, source: String) -> Parser<'_> {
114 panictry_buffer!(&sess.span_diagnostic, maybe_new_parser_from_source_str(sess, name, source))
117 /// Creates a new parser from a source string. Returns any buffered errors from lexing the initial
119 pub fn maybe_new_parser_from_source_str(
123 ) -> Result<Parser<'_>, Vec<Diagnostic>> {
125 maybe_source_file_to_parser(sess, sess.source_map().new_source_file(name, source))?;
126 parser.recurse_into_file_modules = false;
130 /// Creates a new parser, handling errors as appropriate if the file doesn't exist.
131 pub fn new_parser_from_file<'a>(sess: &'a ParseSess, path: &Path) -> Parser<'a> {
132 source_file_to_parser(sess, file_to_source_file(sess, path, None))
135 /// Creates a new parser, returning buffered diagnostics if the file doesn't exist,
136 /// or from lexing the initial token stream.
137 pub fn maybe_new_parser_from_file<'a>(
140 ) -> Result<Parser<'a>, Vec<Diagnostic>> {
141 let file = try_file_to_source_file(sess, path, None).map_err(|db| vec![db])?;
142 maybe_source_file_to_parser(sess, file)
145 /// Given a session, a crate config, a path, and a span, add
146 /// the file at the given path to the `source_map`, and returns a parser.
147 /// On an error, uses the given span as the source of the problem.
148 pub fn new_sub_parser_from_file<'a>(
151 directory_ownership: DirectoryOwnership,
152 module_name: Option<String>,
155 let mut p = source_file_to_parser(sess, file_to_source_file(sess, path, Some(sp)));
156 p.directory.ownership = directory_ownership;
157 p.root_module_name = module_name;
161 /// Given a `source_file` and config, returns a parser.
162 fn source_file_to_parser(sess: &ParseSess, source_file: Lrc<SourceFile>) -> Parser<'_> {
163 panictry_buffer!(&sess.span_diagnostic, maybe_source_file_to_parser(sess, source_file))
166 /// Given a `source_file` and config, return a parser. Returns any buffered errors from lexing the
167 /// initial token stream.
168 fn maybe_source_file_to_parser(
170 source_file: Lrc<SourceFile>,
171 ) -> Result<Parser<'_>, Vec<Diagnostic>> {
172 let end_pos = source_file.end_pos;
173 let (stream, unclosed_delims) = maybe_file_to_stream(sess, source_file, None)?;
174 let mut parser = stream_to_parser(sess, stream, None);
175 parser.unclosed_delims = unclosed_delims;
176 if parser.token == token::Eof && parser.token.span.is_dummy() {
177 parser.token.span = Span::new(end_pos, end_pos, parser.token.span.ctxt());
183 // Must preserve old name for now, because `quote!` from the *existing*
184 // compiler expands into it.
185 pub fn new_parser_from_tts(sess: &ParseSess, tts: Vec<TokenTree>) -> Parser<'_> {
186 stream_to_parser(sess, tts.into_iter().collect(), crate::MACRO_ARGUMENTS)
191 /// Given a session and a path and an optional span (for error reporting),
192 /// add the path to the session's source_map and return the new source_file or
193 /// error when a file can't be read.
194 fn try_file_to_source_file(
197 spanopt: Option<Span>,
198 ) -> Result<Lrc<SourceFile>, Diagnostic> {
199 sess.source_map().load_file(path).map_err(|e| {
200 let msg = format!("couldn't read {}: {}", path.display(), e);
201 let mut diag = Diagnostic::new(Level::Fatal, &msg);
202 if let Some(sp) = spanopt {
209 /// Given a session and a path and an optional span (for error reporting),
210 /// adds the path to the session's `source_map` and returns the new `source_file`.
211 fn file_to_source_file(sess: &ParseSess, path: &Path, spanopt: Option<Span>) -> Lrc<SourceFile> {
212 match try_file_to_source_file(sess, path, spanopt) {
213 Ok(source_file) => source_file,
215 sess.span_diagnostic.emit_diagnostic(&d);
221 /// Given a `source_file`, produces a sequence of token trees.
222 pub fn source_file_to_stream(
224 source_file: Lrc<SourceFile>,
225 override_span: Option<Span>,
226 ) -> (TokenStream, Vec<lexer::UnmatchedBrace>) {
227 panictry_buffer!(&sess.span_diagnostic, maybe_file_to_stream(sess, source_file, override_span))
230 /// Given a source file, produces a sequence of token trees. Returns any buffered errors from
231 /// parsing the token stream.
232 pub fn maybe_file_to_stream(
234 source_file: Lrc<SourceFile>,
235 override_span: Option<Span>,
236 ) -> Result<(TokenStream, Vec<lexer::UnmatchedBrace>), Vec<Diagnostic>> {
237 let srdr = lexer::StringReader::new(sess, source_file, override_span);
238 let (token_trees, unmatched_braces) = srdr.into_token_trees();
241 Ok(stream) => Ok((stream, unmatched_braces)),
243 let mut buffer = Vec::with_capacity(1);
244 err.buffer(&mut buffer);
245 // Not using `emit_unclosed_delims` to use `db.buffer`
246 for unmatched in unmatched_braces {
247 if let Some(err) = make_unclosed_delims_error(unmatched, &sess) {
248 err.buffer(&mut buffer);
256 /// Given a stream and the `ParseSess`, produces a parser.
257 pub fn stream_to_parser<'a>(
260 subparser_name: Option<&'static str>,
262 Parser::new(sess, stream, None, true, false, subparser_name)
265 /// Given a stream, the `ParseSess` and the base directory, produces a parser.
267 /// Use this function when you are creating a parser from the token stream
268 /// and also care about the current working directory of the parser (e.g.,
269 /// you are trying to resolve modules defined inside a macro invocation).
273 /// The main usage of this function is outside of rustc, for those who uses
274 /// libsyntax as a library. Please do not remove this function while refactoring
275 /// just because it is not used in rustc codebase!
276 pub fn stream_to_parser_with_base_dir<'a>(
279 base_dir: Directory<'a>,
281 Parser::new(sess, stream, Some(base_dir), true, false, None)
284 /// Runs the given subparser `f` on the tokens of the given `attr`'s item.
285 pub fn parse_in<'a, T>(
289 mut f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
290 ) -> PResult<'a, T> {
291 let mut parser = Parser::new(sess, tts, None, false, false, Some(name));
292 let result = f(&mut parser)?;
293 if parser.token != token::Eof {
294 parser.unexpected()?;
299 // NOTE(Centril): The following probably shouldn't be here but it acknowledges the
300 // fact that architecturally, we are using parsing (read on below to understand why).
302 pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> TokenStream {
303 // A `Nonterminal` is often a parsed AST item. At this point we now
304 // need to convert the parsed AST to an actual token stream, e.g.
305 // un-parse it basically.
307 // Unfortunately there's not really a great way to do that in a
308 // guaranteed lossless fashion right now. The fallback here is to just
309 // stringify the AST node and reparse it, but this loses all span
312 // As a result, some AST nodes are annotated with the token stream they
313 // came from. Here we attempt to extract these lossless token streams
314 // before we fall back to the stringification.
315 let tokens = match *nt {
316 Nonterminal::NtItem(ref item) => {
317 prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span)
319 Nonterminal::NtTraitItem(ref item) | Nonterminal::NtImplItem(ref item) => {
320 prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span)
322 Nonterminal::NtIdent(ident, is_raw) => {
323 Some(tokenstream::TokenTree::token(token::Ident(ident.name, is_raw), ident.span).into())
325 Nonterminal::NtLifetime(ident) => {
326 Some(tokenstream::TokenTree::token(token::Lifetime(ident.name), ident.span).into())
328 Nonterminal::NtTT(ref tt) => Some(tt.clone().into()),
332 // FIXME(#43081): Avoid this pretty-print + reparse hack
333 let source = pprust::nonterminal_to_string(nt);
334 let filename = FileName::macro_expansion_source_code(&source);
335 let tokens_for_real = parse_stream_from_source_str(filename, source, sess, Some(span));
337 // During early phases of the compiler the AST could get modified
338 // directly (e.g., attributes added or removed) and the internal cache
339 // of tokens my not be invalidated or updated. Consequently if the
340 // "lossless" token stream disagrees with our actual stringification
341 // (which has historically been much more battle-tested) then we go
342 // with the lossy stream anyway (losing span information).
344 // Note that the comparison isn't `==` here to avoid comparing spans,
345 // but it *also* is a "probable" equality which is a pretty weird
346 // definition. We mostly want to catch actual changes to the AST
347 // like a `#[cfg]` being processed or some weird `macro_rules!`
350 // What we *don't* want to catch is the fact that a user-defined
351 // literal like `0xf` is stringified as `15`, causing the cached token
352 // stream to not be literal `==` token-wise (ignoring spans) to the
353 // token stream we got from stringification.
355 // Instead the "probably equal" check here is "does each token
356 // recursively have the same discriminant?" We basically don't look at
357 // the token values here and assume that such fine grained token stream
358 // modifications, including adding/removing typically non-semantic
359 // tokens such as extra braces and commas, don't happen.
360 if let Some(tokens) = tokens {
361 if tokens.probably_equal_for_proc_macro(&tokens_for_real) {
365 "cached tokens found, but they're not \"probably equal\", \
366 going with stringified version"
369 return tokens_for_real;
374 attrs: &[ast::Attribute],
375 tokens: Option<&tokenstream::TokenStream>,
376 span: rustc_span::Span,
377 ) -> Option<tokenstream::TokenStream> {
378 let tokens = tokens?;
379 if attrs.len() == 0 {
380 return Some(tokens.clone());
382 let mut builder = tokenstream::TokenStreamBuilder::new();
386 ast::AttrStyle::Outer,
387 "inner attributes should prevent cached tokens from existing"
390 let source = pprust::attribute_to_string(attr);
391 let macro_filename = FileName::macro_expansion_source_code(&source);
393 let item = match attr.kind {
394 ast::AttrKind::Normal(ref item) => item,
395 ast::AttrKind::DocComment(_) => {
396 let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span));
397 builder.push(stream);
402 // synthesize # [ $path $tokens ] manually here
403 let mut brackets = tokenstream::TokenStreamBuilder::new();
405 // For simple paths, push the identifier directly
406 if item.path.segments.len() == 1 && item.path.segments[0].args.is_none() {
407 let ident = item.path.segments[0].ident;
408 let token = token::Ident(ident.name, ident.as_str().starts_with("r#"));
409 brackets.push(tokenstream::TokenTree::token(token, ident.span));
411 // ... and for more complicated paths, fall back to a reparse hack that
412 // should eventually be removed.
414 let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span));
415 brackets.push(stream);
418 brackets.push(item.args.outer_tokens());
420 // The span we list here for `#` and for `[ ... ]` are both wrong in
421 // that it encompasses more than each token, but it hopefully is "good
422 // enough" for now at least.
423 builder.push(tokenstream::TokenTree::token(token::Pound, attr.span));
424 let delim_span = tokenstream::DelimSpan::from_single(attr.span);
425 builder.push(tokenstream::TokenTree::Delimited(
427 token::DelimToken::Bracket,
428 brackets.build().into(),
431 builder.push(tokens.clone());
432 Some(builder.build())