1 // Copyright 2012 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! The main parser interface
15 use codemap::{Span, CodeMap, FileMap};
17 use diagnostic::{SpanHandler, mk_span_handler, mk_handler, Emitter};
18 use parse::attr::ParserAttr;
19 use parse::parser::Parser;
21 use std::cell::RefCell;
32 /// Common routines shared by parser mods
35 /// Routines the parser uses to classify AST nodes
38 /// Reporting obsolete syntax
41 // info about a parsing session.
42 pub struct ParseSess {
43 cm: @codemap::CodeMap, // better be the same as the one in the reader!
44 span_diagnostic: @SpanHandler, // better be the same as the one in the reader!
45 /// Used to determine and report recursive mod inclusions
46 included_mod_stack: RefCell<~[Path]>,
49 pub fn new_parse_sess(demitter: Option<@Emitter>) -> @ParseSess {
50 let cm = @CodeMap::new();
53 span_diagnostic: mk_span_handler(mk_handler(demitter), cm),
54 included_mod_stack: RefCell::new(~[]),
58 pub fn new_parse_sess_special_handler(sh: @SpanHandler,
59 cm: @codemap::CodeMap)
64 included_mod_stack: RefCell::new(~[]),
68 // a bunch of utility functions of the form parse_<thing>_from_<source>
69 // where <thing> includes crate, expr, item, stmt, tts, and one that
70 // uses a HOF to parse anything, and <source> includes file and
73 pub fn parse_crate_from_file(
75 cfg: ast::CrateConfig,
78 new_parser_from_file(sess, /*bad*/ cfg.clone(), input).parse_crate_mod()
79 // why is there no p.abort_if_errors here?
82 pub fn parse_crate_attrs_from_file(
84 cfg: ast::CrateConfig,
86 ) -> ~[ast::Attribute] {
87 let mut parser = new_parser_from_file(sess, cfg, input);
88 let (inner, _) = parser.parse_inner_attrs_and_next();
92 pub fn parse_crate_from_source_str(
95 cfg: ast::CrateConfig,
98 let mut p = new_parser_from_source_str(sess,
102 maybe_aborted(p.parse_crate_mod(),p)
105 pub fn parse_crate_attrs_from_source_str(
108 cfg: ast::CrateConfig,
110 ) -> ~[ast::Attribute] {
111 let mut p = new_parser_from_source_str(sess,
115 let (inner, _) = maybe_aborted(p.parse_inner_attrs_and_next(),p);
119 pub fn parse_expr_from_source_str(
122 cfg: ast::CrateConfig,
125 let mut p = new_parser_from_source_str(sess, cfg, name, source);
126 maybe_aborted(p.parse_expr(), p)
129 pub fn parse_item_from_source_str(
132 cfg: ast::CrateConfig,
133 attrs: ~[ast::Attribute],
135 ) -> Option<@ast::Item> {
136 let mut p = new_parser_from_source_str(sess, cfg, name, source);
137 maybe_aborted(p.parse_item(attrs),p)
140 pub fn parse_meta_from_source_str(
143 cfg: ast::CrateConfig,
145 ) -> @ast::MetaItem {
146 let mut p = new_parser_from_source_str(sess, cfg, name, source);
147 maybe_aborted(p.parse_meta_item(),p)
150 pub fn parse_stmt_from_source_str(
153 cfg: ast::CrateConfig,
154 attrs: ~[ast::Attribute],
157 let mut p = new_parser_from_source_str(
163 maybe_aborted(p.parse_stmt(attrs),p)
166 pub fn parse_tts_from_source_str(
169 cfg: ast::CrateConfig,
171 ) -> ~[ast::TokenTree] {
172 let mut p = new_parser_from_source_str(
179 // right now this is re-creating the token trees from ... token trees.
180 maybe_aborted(p.parse_all_token_trees(),p)
183 // Create a new parser from a source string
184 pub fn new_parser_from_source_str(sess: @ParseSess,
185 cfg: ast::CrateConfig,
189 filemap_to_parser(sess,string_to_filemap(sess,source,name),cfg)
192 /// Create a new parser, handling errors as appropriate
193 /// if the file doesn't exist
194 pub fn new_parser_from_file(
196 cfg: ast::CrateConfig,
199 filemap_to_parser(sess,file_to_filemap(sess,path,None),cfg)
202 /// Given a session, a crate config, a path, and a span, add
203 /// the file at the given path to the codemap, and return a parser.
204 /// On an error, use the given span as the source of the problem.
205 pub fn new_sub_parser_from_file(
207 cfg: ast::CrateConfig,
211 filemap_to_parser(sess,file_to_filemap(sess,path,Some(sp)),cfg)
214 /// Given a filemap and config, return a parser
215 pub fn filemap_to_parser(sess: @ParseSess,
217 cfg: ast::CrateConfig) -> Parser {
218 tts_to_parser(sess,filemap_to_tts(sess,filemap),cfg)
221 // must preserve old name for now, because quote! from the *existing*
222 // compiler expands into it
223 pub fn new_parser_from_tts(sess: @ParseSess,
224 cfg: ast::CrateConfig,
225 tts: ~[ast::TokenTree]) -> Parser {
226 tts_to_parser(sess,tts,cfg)
232 /// Given a session and a path and an optional span (for error reporting),
233 /// add the path to the session's codemap and return the new filemap.
234 pub fn file_to_filemap(sess: @ParseSess, path: &Path, spanopt: Option<Span>)
236 let err = |msg: &str| {
238 Some(sp) => sess.span_diagnostic.span_fatal(sp, msg),
239 None => sess.span_diagnostic.handler().fatal(msg),
242 let bytes = match io::result(|| File::open(path).read_to_end()) {
245 err(format!("couldn't read {}: {}", path.display(), e.desc));
249 match str::from_utf8_owned_opt(bytes) {
251 return string_to_filemap(sess, s.to_managed(),
252 path.as_str().unwrap().to_managed());
255 err(format!("{} is not UTF-8 encoded", path.display()))
261 // given a session and a string, add the string to
262 // the session's codemap and return the new filemap
263 pub fn string_to_filemap(sess: @ParseSess, source: @str, path: @str)
265 sess.cm.new_filemap(path, source)
268 // given a filemap, produce a sequence of token-trees
269 pub fn filemap_to_tts(sess: @ParseSess, filemap: @FileMap)
270 -> ~[ast::TokenTree] {
271 // it appears to me that the cfg doesn't matter here... indeed,
272 // parsing tt's probably shouldn't require a parser at all.
274 let srdr = lexer::new_string_reader(sess.span_diagnostic, filemap);
275 let mut p1 = Parser(sess, cfg, srdr as @lexer::Reader);
276 p1.parse_all_token_trees()
279 // given tts and cfg, produce a parser
280 pub fn tts_to_parser(sess: @ParseSess,
281 tts: ~[ast::TokenTree],
282 cfg: ast::CrateConfig) -> Parser {
283 let trdr = lexer::new_tt_reader(sess.span_diagnostic, None, tts);
284 Parser(sess, cfg, trdr as @lexer::Reader)
287 // abort if necessary
288 pub fn maybe_aborted<T>(result: T, mut p: Parser) -> T {
298 use extra::serialize::Encodable;
301 use std::io::MemWriter;
303 use codemap::{Span, BytePos, Spanned};
307 use parse::parser::Parser;
308 use parse::token::{str_to_ident};
309 use util::parser_testing::{string_to_tts, string_to_parser};
310 use util::parser_testing::{string_to_expr, string_to_item};
311 use util::parser_testing::string_to_stmt;
314 fn to_json_str<'a, E: Encodable<extra::json::Encoder<'a>>>(val: &E) -> ~str {
315 let mut writer = MemWriter::new();
316 let mut encoder = extra::json::Encoder::new(&mut writer as &mut io::Writer);
317 val.encode(&mut encoder);
318 str::from_utf8_owned(writer.unwrap())
321 // produce a codemap::span
322 fn sp(a: u32, b: u32) -> Span {
323 Span{lo:BytePos(a),hi:BytePos(b),expn_info:None}
326 #[test] fn path_exprs_1() {
327 assert_eq!(string_to_expr(@"a"),
329 id: ast::DUMMY_NODE_ID,
330 node: ast::ExprPath(ast::Path {
335 identifier: str_to_ident("a"),
336 lifetimes: opt_vec::Empty,
337 types: opt_vec::Empty,
345 #[test] fn path_exprs_2 () {
346 assert_eq!(string_to_expr(@"::a::b"),
348 id: ast::DUMMY_NODE_ID,
349 node: ast::ExprPath(ast::Path {
354 identifier: str_to_ident("a"),
355 lifetimes: opt_vec::Empty,
356 types: opt_vec::Empty,
359 identifier: str_to_ident("b"),
360 lifetimes: opt_vec::Empty,
361 types: opt_vec::Empty,
370 #[test] fn bad_path_expr_1() {
371 string_to_expr(@"::abc::def::return");
374 // check the token-tree-ization of macros
375 #[test] fn string_to_tts_macro () {
376 let tts = string_to_tts(@"macro_rules! zip (($a)=>($a))");
379 ast::TTTok(_,token::NOT),
381 ast::TTDelim(delim_elts)] =>
383 [ast::TTTok(_,token::LPAREN),
384 ast::TTDelim(first_set),
385 ast::TTTok(_,token::FAT_ARROW),
386 ast::TTDelim(second_set),
387 ast::TTTok(_,token::RPAREN)] =>
389 [ast::TTTok(_,token::LPAREN),
390 ast::TTTok(_,token::DOLLAR),
392 ast::TTTok(_,token::RPAREN)] =>
394 [ast::TTTok(_,token::LPAREN),
395 ast::TTTok(_,token::DOLLAR),
397 ast::TTTok(_,token::RPAREN)] =>
398 assert_eq!("correct","correct"),
399 _ => assert_eq!("wrong 4","correct")
402 error!("failing value 3: {:?}",first_set);
403 assert_eq!("wrong 3","correct")
407 error!("failing value 2: {:?}",delim_elts);
408 assert_eq!("wrong","correct");
413 error!("failing value: {:?}",tts);
414 assert_eq!("wrong 1","correct");
419 #[test] fn string_to_tts_1 () {
420 let tts = string_to_tts(@"fn a (b : int) { b; }");
421 assert_eq!(to_json_str(&tts),
424 \"variant\":\"TTTok\",\
428 \"variant\":\"IDENT\",\
437 \"variant\":\"TTTok\",\
441 \"variant\":\"IDENT\",\
450 \"variant\":\"TTDelim\",\
454 \"variant\":\"TTTok\",\
461 \"variant\":\"TTTok\",\
465 \"variant\":\"IDENT\",\
474 \"variant\":\"TTTok\",\
481 \"variant\":\"TTTok\",\
485 \"variant\":\"IDENT\",\
494 \"variant\":\"TTTok\",\
504 \"variant\":\"TTDelim\",\
508 \"variant\":\"TTTok\",\
515 \"variant\":\"TTTok\",\
519 \"variant\":\"IDENT\",\
528 \"variant\":\"TTTok\",\
535 \"variant\":\"TTTok\",\
548 #[test] fn ret_expr() {
549 assert_eq!(string_to_expr(@"return d"),
551 id: ast::DUMMY_NODE_ID,
552 node:ast::ExprRet(Some(@ast::Expr{
553 id: ast::DUMMY_NODE_ID,
554 node:ast::ExprPath(ast::Path{
559 identifier: str_to_ident("d"),
560 lifetimes: opt_vec::Empty,
561 types: opt_vec::Empty,
571 #[test] fn parse_stmt_1 () {
572 assert_eq!(string_to_stmt(@"b;"),
574 node: ast::StmtExpr(@ast::Expr {
575 id: ast::DUMMY_NODE_ID,
576 node: ast::ExprPath(ast::Path {
581 identifier: str_to_ident("b"),
582 lifetimes: opt_vec::Empty,
583 types: opt_vec::Empty,
593 fn parser_done(p: Parser){
594 assert_eq!(p.token.clone(), token::EOF);
597 #[test] fn parse_ident_pat () {
598 let mut parser = string_to_parser(@"b");
599 assert_eq!(parser.parse_pat(),
600 @ast::Pat{id: ast::DUMMY_NODE_ID,
602 ast::BindByValue(ast::MutImmutable),
608 identifier: str_to_ident("b"),
609 lifetimes: opt_vec::Empty,
610 types: opt_vec::Empty,
619 // check the contents of the tt manually:
620 #[test] fn parse_fundecl () {
621 // this test depends on the intern order of "fn" and "int"
622 assert_eq!(string_to_item(@"fn a (b : int) { b; }"),
624 @ast::Item{ident:str_to_ident("a"),
626 id: ast::DUMMY_NODE_ID,
627 node: ast::ItemFn(ast::P(ast::FnDecl {
629 ty: ast::P(ast::Ty{id: ast::DUMMY_NODE_ID,
630 node: ast::TyPath(ast::Path{
637 lifetimes: opt_vec::Empty,
638 types: opt_vec::Empty,
641 }, None, ast::DUMMY_NODE_ID),
645 id: ast::DUMMY_NODE_ID,
647 ast::BindByValue(ast::MutImmutable),
655 lifetimes: opt_vec::Empty,
656 types: opt_vec::Empty,
664 id: ast::DUMMY_NODE_ID
666 output: ast::P(ast::Ty{id: ast::DUMMY_NODE_ID,
668 span:sp(15,15)}), // not sure
674 ast::Generics{ // no idea on either of these:
675 lifetimes: opt_vec::Empty,
676 ty_params: opt_vec::Empty,
681 node: ast::StmtSemi(@ast::Expr{
682 id: ast::DUMMY_NODE_ID,
703 id: ast::DUMMY_NODE_ID,
704 rules: ast::DefaultBlock, // no idea
712 #[test] fn parse_exprs () {
713 // just make sure that they parse....
714 string_to_expr(@"3 + 4");
715 string_to_expr(@"a::z.froob(b,@(987+3))");
718 #[test] fn attrs_fix_bug () {
719 string_to_item(@"pub fn mk_file_writer(path: &Path, flags: &[FileFlag])
720 -> Result<@Writer, ~str> {
723 (O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int
727 fn wb() -> c_int { O_WRONLY as c_int }
729 let mut fflags: c_int = wb();