1 // Copyright 2012 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! The main parser interface
15 use codemap::{Span, CodeMap, FileMap};
17 use diagnostic::{SpanHandler, mk_span_handler, mk_handler, Emitter};
18 use parse::attr::ParserAttr;
19 use parse::parser::Parser;
21 use std::cell::RefCell;
32 /// Common routines shared by parser mods
35 /// Routines the parser uses to classify AST nodes
38 /// Reporting obsolete syntax
41 // info about a parsing session.
42 pub struct ParseSess {
43 cm: @codemap::CodeMap, // better be the same as the one in the reader!
44 span_diagnostic: @SpanHandler, // better be the same as the one in the reader!
45 /// Used to determine and report recursive mod inclusions
46 included_mod_stack: RefCell<~[Path]>,
49 pub fn new_parse_sess(demitter: Option<@Emitter>) -> @ParseSess {
50 let cm = @CodeMap::new();
53 span_diagnostic: mk_span_handler(mk_handler(demitter), cm),
54 included_mod_stack: RefCell::new(~[]),
58 pub fn new_parse_sess_special_handler(sh: @SpanHandler,
59 cm: @codemap::CodeMap)
64 included_mod_stack: RefCell::new(~[]),
68 // a bunch of utility functions of the form parse_<thing>_from_<source>
69 // where <thing> includes crate, expr, item, stmt, tts, and one that
70 // uses a HOF to parse anything, and <source> includes file and
73 pub fn parse_crate_from_file(
75 cfg: ast::CrateConfig,
78 new_parser_from_file(sess, /*bad*/ cfg.clone(), input).parse_crate_mod()
79 // why is there no p.abort_if_errors here?
82 pub fn parse_crate_attrs_from_file(
84 cfg: ast::CrateConfig,
86 ) -> ~[ast::Attribute] {
87 let mut parser = new_parser_from_file(sess, cfg, input);
88 let (inner, _) = parser.parse_inner_attrs_and_next();
92 pub fn parse_crate_from_source_str(name: ~str,
94 cfg: ast::CrateConfig,
97 let mut p = new_parser_from_source_str(sess,
101 maybe_aborted(p.parse_crate_mod(),p)
104 pub fn parse_crate_attrs_from_source_str(name: ~str,
106 cfg: ast::CrateConfig,
108 -> ~[ast::Attribute] {
109 let mut p = new_parser_from_source_str(sess,
113 let (inner, _) = maybe_aborted(p.parse_inner_attrs_and_next(),p);
117 pub fn parse_expr_from_source_str(name: ~str,
119 cfg: ast::CrateConfig,
122 let mut p = new_parser_from_source_str(sess, cfg, name, source);
123 maybe_aborted(p.parse_expr(), p)
126 pub fn parse_item_from_source_str(name: ~str,
128 cfg: ast::CrateConfig,
130 -> Option<@ast::Item> {
131 let mut p = new_parser_from_source_str(sess, cfg, name, source);
132 let attrs = p.parse_outer_attributes();
133 maybe_aborted(p.parse_item(attrs),p)
136 pub fn parse_meta_from_source_str(name: ~str,
138 cfg: ast::CrateConfig,
141 let mut p = new_parser_from_source_str(sess, cfg, name, source);
142 maybe_aborted(p.parse_meta_item(),p)
145 pub fn parse_stmt_from_source_str(name: ~str,
147 cfg: ast::CrateConfig,
148 attrs: ~[ast::Attribute],
151 let mut p = new_parser_from_source_str(
157 maybe_aborted(p.parse_stmt(attrs),p)
160 pub fn parse_tts_from_source_str(name: ~str,
162 cfg: ast::CrateConfig,
164 -> ~[ast::TokenTree] {
165 let mut p = new_parser_from_source_str(
172 // right now this is re-creating the token trees from ... token trees.
173 maybe_aborted(p.parse_all_token_trees(),p)
176 // Create a new parser from a source string
177 pub fn new_parser_from_source_str(sess: @ParseSess,
178 cfg: ast::CrateConfig,
182 filemap_to_parser(sess,string_to_filemap(sess,source,name),cfg)
185 /// Create a new parser, handling errors as appropriate
186 /// if the file doesn't exist
187 pub fn new_parser_from_file(
189 cfg: ast::CrateConfig,
192 filemap_to_parser(sess,file_to_filemap(sess,path,None),cfg)
195 /// Given a session, a crate config, a path, and a span, add
196 /// the file at the given path to the codemap, and return a parser.
197 /// On an error, use the given span as the source of the problem.
198 pub fn new_sub_parser_from_file(
200 cfg: ast::CrateConfig,
204 filemap_to_parser(sess,file_to_filemap(sess,path,Some(sp)),cfg)
207 /// Given a filemap and config, return a parser
208 pub fn filemap_to_parser(sess: @ParseSess,
210 cfg: ast::CrateConfig) -> Parser {
211 tts_to_parser(sess,filemap_to_tts(sess,filemap),cfg)
214 // must preserve old name for now, because quote! from the *existing*
215 // compiler expands into it
216 pub fn new_parser_from_tts(sess: @ParseSess,
217 cfg: ast::CrateConfig,
218 tts: ~[ast::TokenTree]) -> Parser {
219 tts_to_parser(sess,tts,cfg)
225 /// Given a session and a path and an optional span (for error reporting),
226 /// add the path to the session's codemap and return the new filemap.
227 pub fn file_to_filemap(sess: @ParseSess, path: &Path, spanopt: Option<Span>)
229 let err = |msg: &str| {
231 Some(sp) => sess.span_diagnostic.span_fatal(sp, msg),
232 None => sess.span_diagnostic.handler().fatal(msg),
235 let bytes = match io::result(|| File::open(path).read_to_end()) {
238 err(format!("couldn't read {}: {}", path.display(), e.desc));
242 match str::from_utf8_owned(bytes) {
244 return string_to_filemap(sess, s, path.as_str().unwrap().to_str())
246 None => err(format!("{} is not UTF-8 encoded", path.display())),
251 // given a session and a string, add the string to
252 // the session's codemap and return the new filemap
253 pub fn string_to_filemap(sess: @ParseSess, source: ~str, path: ~str)
255 sess.cm.new_filemap(path, source)
258 // given a filemap, produce a sequence of token-trees
259 pub fn filemap_to_tts(sess: @ParseSess, filemap: @FileMap)
260 -> ~[ast::TokenTree] {
261 // it appears to me that the cfg doesn't matter here... indeed,
262 // parsing tt's probably shouldn't require a parser at all.
264 let srdr = lexer::new_string_reader(sess.span_diagnostic, filemap);
265 let mut p1 = Parser(sess, cfg, srdr as @lexer::Reader);
266 p1.parse_all_token_trees()
269 // given tts and cfg, produce a parser
270 pub fn tts_to_parser(sess: @ParseSess,
271 tts: ~[ast::TokenTree],
272 cfg: ast::CrateConfig) -> Parser {
273 let trdr = lexer::new_tt_reader(sess.span_diagnostic, None, tts);
274 Parser(sess, cfg, trdr as @lexer::Reader)
277 // abort if necessary
278 pub fn maybe_aborted<T>(result: T, mut p: Parser) -> T {
288 use extra::serialize::Encodable;
291 use std::io::MemWriter;
293 use codemap::{Span, BytePos, Spanned};
297 use parse::parser::Parser;
298 use parse::token::{str_to_ident};
299 use util::parser_testing::{string_to_tts, string_to_parser};
300 use util::parser_testing::{string_to_expr, string_to_item};
301 use util::parser_testing::string_to_stmt;
304 fn to_json_str<'a, E: Encodable<extra::json::Encoder<'a>>>(val: &E) -> ~str {
305 let mut writer = MemWriter::new();
306 let mut encoder = extra::json::Encoder::new(&mut writer as &mut io::Writer);
307 val.encode(&mut encoder);
308 str::from_utf8_owned(writer.unwrap()).unwrap()
311 // produce a codemap::span
312 fn sp(a: u32, b: u32) -> Span {
313 Span{lo:BytePos(a),hi:BytePos(b),expn_info:None}
316 #[test] fn path_exprs_1() {
317 assert_eq!(string_to_expr(@"a"),
319 id: ast::DUMMY_NODE_ID,
320 node: ast::ExprPath(ast::Path {
325 identifier: str_to_ident("a"),
326 lifetimes: opt_vec::Empty,
327 types: opt_vec::Empty,
335 #[test] fn path_exprs_2 () {
336 assert_eq!(string_to_expr(@"::a::b"),
338 id: ast::DUMMY_NODE_ID,
339 node: ast::ExprPath(ast::Path {
344 identifier: str_to_ident("a"),
345 lifetimes: opt_vec::Empty,
346 types: opt_vec::Empty,
349 identifier: str_to_ident("b"),
350 lifetimes: opt_vec::Empty,
351 types: opt_vec::Empty,
360 #[test] fn bad_path_expr_1() {
361 string_to_expr(@"::abc::def::return");
364 // check the token-tree-ization of macros
365 #[test] fn string_to_tts_macro () {
366 let tts = string_to_tts(@"macro_rules! zip (($a)=>($a))");
369 ast::TTTok(_,token::NOT),
371 ast::TTDelim(delim_elts)] =>
373 [ast::TTTok(_,token::LPAREN),
374 ast::TTDelim(first_set),
375 ast::TTTok(_,token::FAT_ARROW),
376 ast::TTDelim(second_set),
377 ast::TTTok(_,token::RPAREN)] =>
379 [ast::TTTok(_,token::LPAREN),
380 ast::TTTok(_,token::DOLLAR),
382 ast::TTTok(_,token::RPAREN)] =>
384 [ast::TTTok(_,token::LPAREN),
385 ast::TTTok(_,token::DOLLAR),
387 ast::TTTok(_,token::RPAREN)] =>
388 assert_eq!("correct","correct"),
389 _ => assert_eq!("wrong 4","correct")
392 error!("failing value 3: {:?}",first_set);
393 assert_eq!("wrong 3","correct")
397 error!("failing value 2: {:?}",delim_elts);
398 assert_eq!("wrong","correct");
403 error!("failing value: {:?}",tts);
404 assert_eq!("wrong 1","correct");
409 #[test] fn string_to_tts_1 () {
410 let tts = string_to_tts(@"fn a (b : int) { b; }");
411 assert_eq!(to_json_str(&tts),
414 \"variant\":\"TTTok\",\
418 \"variant\":\"IDENT\",\
427 \"variant\":\"TTTok\",\
431 \"variant\":\"IDENT\",\
440 \"variant\":\"TTDelim\",\
444 \"variant\":\"TTTok\",\
451 \"variant\":\"TTTok\",\
455 \"variant\":\"IDENT\",\
464 \"variant\":\"TTTok\",\
471 \"variant\":\"TTTok\",\
475 \"variant\":\"IDENT\",\
484 \"variant\":\"TTTok\",\
494 \"variant\":\"TTDelim\",\
498 \"variant\":\"TTTok\",\
505 \"variant\":\"TTTok\",\
509 \"variant\":\"IDENT\",\
518 \"variant\":\"TTTok\",\
525 \"variant\":\"TTTok\",\
538 #[test] fn ret_expr() {
539 assert_eq!(string_to_expr(@"return d"),
541 id: ast::DUMMY_NODE_ID,
542 node:ast::ExprRet(Some(@ast::Expr{
543 id: ast::DUMMY_NODE_ID,
544 node:ast::ExprPath(ast::Path{
549 identifier: str_to_ident("d"),
550 lifetimes: opt_vec::Empty,
551 types: opt_vec::Empty,
561 #[test] fn parse_stmt_1 () {
562 assert_eq!(string_to_stmt(@"b;"),
564 node: ast::StmtExpr(@ast::Expr {
565 id: ast::DUMMY_NODE_ID,
566 node: ast::ExprPath(ast::Path {
571 identifier: str_to_ident("b"),
572 lifetimes: opt_vec::Empty,
573 types: opt_vec::Empty,
583 fn parser_done(p: Parser){
584 assert_eq!(p.token.clone(), token::EOF);
587 #[test] fn parse_ident_pat () {
588 let mut parser = string_to_parser(@"b");
589 assert_eq!(parser.parse_pat(),
590 @ast::Pat{id: ast::DUMMY_NODE_ID,
592 ast::BindByValue(ast::MutImmutable),
598 identifier: str_to_ident("b"),
599 lifetimes: opt_vec::Empty,
600 types: opt_vec::Empty,
609 // check the contents of the tt manually:
610 #[test] fn parse_fundecl () {
611 // this test depends on the intern order of "fn" and "int"
612 assert_eq!(string_to_item(@"fn a (b : int) { b; }"),
614 @ast::Item{ident:str_to_ident("a"),
616 id: ast::DUMMY_NODE_ID,
617 node: ast::ItemFn(ast::P(ast::FnDecl {
619 ty: ast::P(ast::Ty{id: ast::DUMMY_NODE_ID,
620 node: ast::TyPath(ast::Path{
627 lifetimes: opt_vec::Empty,
628 types: opt_vec::Empty,
631 }, None, ast::DUMMY_NODE_ID),
635 id: ast::DUMMY_NODE_ID,
637 ast::BindByValue(ast::MutImmutable),
645 lifetimes: opt_vec::Empty,
646 types: opt_vec::Empty,
654 id: ast::DUMMY_NODE_ID
656 output: ast::P(ast::Ty{id: ast::DUMMY_NODE_ID,
658 span:sp(15,15)}), // not sure
664 ast::Generics{ // no idea on either of these:
665 lifetimes: opt_vec::Empty,
666 ty_params: opt_vec::Empty,
671 node: ast::StmtSemi(@ast::Expr{
672 id: ast::DUMMY_NODE_ID,
693 id: ast::DUMMY_NODE_ID,
694 rules: ast::DefaultBlock, // no idea
702 #[test] fn parse_exprs () {
703 // just make sure that they parse....
704 string_to_expr(@"3 + 4");
705 string_to_expr(@"a::z.froob(b,@(987+3))");
708 #[test] fn attrs_fix_bug () {
709 string_to_item(@"pub fn mk_file_writer(path: &Path, flags: &[FileFlag])
710 -> Result<@Writer, ~str> {
713 (O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int
717 fn wb() -> c_int { O_WRONLY as c_int }
719 let mut fflags: c_int = wb();