1 // Copyright 2012 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! The main parser interface
15 use codemap::{Span, CodeMap, FileMap, FileSubstr};
17 use diagnostic::{span_handler, mk_span_handler, mk_handler, Emitter};
18 use parse::attr::parser_attr;
19 use parse::lexer::reader;
20 use parse::parser::Parser;
32 /// Common routines shared by parser mods
35 /// Routines the parser uses to classify AST nodes
38 /// Reporting obsolete syntax
41 // info about a parsing session.
42 pub struct ParseSess {
43 cm: @codemap::CodeMap, // better be the same as the one in the reader!
44 span_diagnostic: @mut span_handler, // better be the same as the one in the reader!
45 /// Used to determine and report recursive mod inclusions
46 included_mod_stack: ~[Path],
49 pub fn new_parse_sess(demitter: Option<@Emitter>) -> @mut ParseSess {
50 let cm = @CodeMap::new();
53 span_diagnostic: mk_span_handler(mk_handler(demitter), cm),
54 included_mod_stack: ~[],
58 pub fn new_parse_sess_special_handler(sh: @mut span_handler,
59 cm: @codemap::CodeMap)
64 included_mod_stack: ~[],
68 // a bunch of utility functions of the form parse_<thing>_from_<source>
69 // where <thing> includes crate, expr, item, stmt, tts, and one that
70 // uses a HOF to parse anything, and <source> includes file and
73 pub fn parse_crate_from_file(
75 cfg: ast::CrateConfig,
78 new_parser_from_file(sess, /*bad*/ cfg.clone(), input).parse_crate_mod()
79 // why is there no p.abort_if_errors here?
82 pub fn parse_crate_from_source_str(
85 cfg: ast::CrateConfig,
88 let p = new_parser_from_source_str(sess,
92 maybe_aborted(p.parse_crate_mod(),p)
95 pub fn parse_expr_from_source_str(
98 cfg: ast::CrateConfig,
101 let p = new_parser_from_source_str(
107 maybe_aborted(p.parse_expr(), p)
110 pub fn parse_item_from_source_str(
113 cfg: ast::CrateConfig,
114 attrs: ~[ast::Attribute],
116 ) -> Option<@ast::item> {
117 let p = new_parser_from_source_str(
123 maybe_aborted(p.parse_item(attrs),p)
126 pub fn parse_meta_from_source_str(
129 cfg: ast::CrateConfig,
131 ) -> @ast::MetaItem {
132 let p = new_parser_from_source_str(
138 maybe_aborted(p.parse_meta_item(),p)
141 pub fn parse_stmt_from_source_str(
144 cfg: ast::CrateConfig,
145 attrs: ~[ast::Attribute],
148 let p = new_parser_from_source_str(
154 maybe_aborted(p.parse_stmt(attrs),p)
157 pub fn parse_tts_from_source_str(
160 cfg: ast::CrateConfig,
162 ) -> ~[ast::token_tree] {
163 let p = new_parser_from_source_str(
169 *p.quote_depth += 1u;
170 // right now this is re-creating the token trees from ... token trees.
171 maybe_aborted(p.parse_all_token_trees(),p)
174 // given a function and parsing information (source str,
175 // filename, crate cfg, and sess), create a parser,
176 // apply the function, and check that the parser
177 // consumed all of the input before returning the function's
179 pub fn parse_from_source_str<T>(
182 ss: codemap::FileSubstr,
184 cfg: ast::CrateConfig,
185 sess: @mut ParseSess)
187 let p = new_parser_from_source_substr(sess, cfg, name, ss, source);
189 if !p.reader.is_eof() {
190 p.reader.fatal(~"expected end-of-string");
195 // Create a new parser from a source string
196 pub fn new_parser_from_source_str(sess: @mut ParseSess,
197 cfg: ast::CrateConfig,
201 filemap_to_parser(sess,string_to_filemap(sess,source,name),cfg)
204 // Create a new parser from a source string where the origin
205 // is specified as a substring of another file.
206 pub fn new_parser_from_source_substr(sess: @mut ParseSess,
207 cfg: ast::CrateConfig,
209 ss: codemap::FileSubstr,
212 filemap_to_parser(sess,substring_to_filemap(sess,source,name,ss),cfg)
215 /// Create a new parser, handling errors as appropriate
216 /// if the file doesn't exist
217 pub fn new_parser_from_file(
218 sess: @mut ParseSess,
219 cfg: ast::CrateConfig,
222 filemap_to_parser(sess,file_to_filemap(sess,path,None),cfg)
225 /// Given a session, a crate config, a path, and a span, add
226 /// the file at the given path to the codemap, and return a parser.
227 /// On an error, use the given span as the source of the problem.
228 pub fn new_sub_parser_from_file(
229 sess: @mut ParseSess,
230 cfg: ast::CrateConfig,
234 filemap_to_parser(sess,file_to_filemap(sess,path,Some(sp)),cfg)
237 /// Given a filemap and config, return a parser
238 pub fn filemap_to_parser(sess: @mut ParseSess,
240 cfg: ast::CrateConfig) -> Parser {
241 tts_to_parser(sess,filemap_to_tts(sess,filemap),cfg)
244 // must preserve old name for now, because quote! from the *existing*
245 // compiler expands into it
246 pub fn new_parser_from_tts(sess: @mut ParseSess,
247 cfg: ast::CrateConfig,
248 tts: ~[ast::token_tree]) -> Parser {
249 tts_to_parser(sess,tts,cfg)
255 /// Given a session and a path and an optional span (for error reporting),
256 /// add the path to the session's codemap and return the new filemap.
257 pub fn file_to_filemap(sess: @mut ParseSess, path: &Path, spanopt: Option<Span>)
259 let err = |msg: &str| {
261 Some(sp) => sess.span_diagnostic.span_fatal(sp, msg),
262 None => sess.span_diagnostic.handler().fatal(msg),
265 let bytes = match io::result(|| File::open(path).read_to_end()) {
268 err(format!("couldn't read {}: {}", path.display(), e.desc));
272 match str::from_utf8_owned_opt(bytes) {
274 return string_to_filemap(sess, s.to_managed(),
275 path.as_str().unwrap().to_managed());
278 err(format!("{} is not UTF-8 encoded", path.display()))
284 // given a session and a string, add the string to
285 // the session's codemap and return the new filemap
286 pub fn string_to_filemap(sess: @mut ParseSess, source: @str, path: @str)
288 sess.cm.new_filemap(path, source)
291 // given a session and a string and a path and a FileSubStr, add
292 // the string to the CodeMap and return the new FileMap
293 pub fn substring_to_filemap(sess: @mut ParseSess, source: @str, path: @str,
294 filesubstr: FileSubstr) -> @FileMap {
295 sess.cm.new_filemap_w_substr(path,filesubstr,source)
298 // given a filemap, produce a sequence of token-trees
299 pub fn filemap_to_tts(sess: @mut ParseSess, filemap: @FileMap)
300 -> ~[ast::token_tree] {
301 // it appears to me that the cfg doesn't matter here... indeed,
302 // parsing tt's probably shouldn't require a parser at all.
304 let srdr = lexer::new_string_reader(sess.span_diagnostic, filemap);
305 let p1 = Parser(sess, cfg, srdr as @mut reader);
306 p1.parse_all_token_trees()
309 // given tts and cfg, produce a parser
310 pub fn tts_to_parser(sess: @mut ParseSess,
311 tts: ~[ast::token_tree],
312 cfg: ast::CrateConfig) -> Parser {
313 let trdr = lexer::new_tt_reader(sess.span_diagnostic, None, tts);
314 Parser(sess, cfg, trdr as @mut reader)
317 // abort if necessary
318 pub fn maybe_aborted<T>(result : T, p: Parser) -> T {
328 use extra::serialize::Encodable;
331 use std::io::Decorator;
332 use std::io::mem::MemWriter;
334 use codemap::{Span, BytePos, Spanned};
338 use parse::parser::Parser;
339 use parse::token::{str_to_ident};
340 use util::parser_testing::{string_to_tts, string_to_parser};
341 use util::parser_testing::{string_to_expr, string_to_item};
342 use util::parser_testing::string_to_stmt;
345 fn to_json_str<'a, E: Encodable<extra::json::Encoder<'a>>>(val: &E) -> ~str {
346 let mut writer = MemWriter::new();
347 let mut encoder = extra::json::Encoder::init(&mut writer as &mut io::Writer);
348 val.encode(&mut encoder);
349 str::from_utf8_owned(writer.inner())
352 // produce a codemap::span
353 fn sp(a: u32, b: u32) -> Span {
354 Span{lo:BytePos(a),hi:BytePos(b),expn_info:None}
357 #[test] fn path_exprs_1() {
358 assert_eq!(string_to_expr(@"a"),
360 id: ast::DUMMY_NODE_ID,
361 node: ast::ExprPath(ast::Path {
366 identifier: str_to_ident("a"),
367 lifetimes: opt_vec::Empty,
368 types: opt_vec::Empty,
376 #[test] fn path_exprs_2 () {
377 assert_eq!(string_to_expr(@"::a::b"),
379 id: ast::DUMMY_NODE_ID,
380 node: ast::ExprPath(ast::Path {
385 identifier: str_to_ident("a"),
386 lifetimes: opt_vec::Empty,
387 types: opt_vec::Empty,
390 identifier: str_to_ident("b"),
391 lifetimes: opt_vec::Empty,
392 types: opt_vec::Empty,
401 #[test] fn bad_path_expr_1() {
402 string_to_expr(@"::abc::def::return");
405 // check the token-tree-ization of macros
406 #[test] fn string_to_tts_macro () {
407 let tts = string_to_tts(@"macro_rules! zip (($a)=>($a))");
410 ast::tt_tok(_,token::NOT),
412 ast::tt_delim(delim_elts)] =>
414 [ast::tt_tok(_,token::LPAREN),
415 ast::tt_delim(first_set),
416 ast::tt_tok(_,token::FAT_ARROW),
417 ast::tt_delim(second_set),
418 ast::tt_tok(_,token::RPAREN)] =>
420 [ast::tt_tok(_,token::LPAREN),
421 ast::tt_tok(_,token::DOLLAR),
423 ast::tt_tok(_,token::RPAREN)] =>
425 [ast::tt_tok(_,token::LPAREN),
426 ast::tt_tok(_,token::DOLLAR),
428 ast::tt_tok(_,token::RPAREN)] =>
429 assert_eq!("correct","correct"),
430 _ => assert_eq!("wrong 4","correct")
433 error!("failing value 3: {:?}",first_set);
434 assert_eq!("wrong 3","correct")
438 error!("failing value 2: {:?}",delim_elts);
439 assert_eq!("wrong","correct");
444 error!("failing value: {:?}",tts);
445 assert_eq!("wrong 1","correct");
450 #[test] fn string_to_tts_1 () {
451 let tts = string_to_tts(@"fn a (b : int) { b; }");
452 assert_eq!(to_json_str(@tts),
455 \"variant\":\"tt_tok\",\
459 \"variant\":\"IDENT\",\
468 \"variant\":\"tt_tok\",\
472 \"variant\":\"IDENT\",\
481 \"variant\":\"tt_delim\",\
485 \"variant\":\"tt_tok\",\
492 \"variant\":\"tt_tok\",\
496 \"variant\":\"IDENT\",\
505 \"variant\":\"tt_tok\",\
512 \"variant\":\"tt_tok\",\
516 \"variant\":\"IDENT\",\
525 \"variant\":\"tt_tok\",\
535 \"variant\":\"tt_delim\",\
539 \"variant\":\"tt_tok\",\
546 \"variant\":\"tt_tok\",\
550 \"variant\":\"IDENT\",\
559 \"variant\":\"tt_tok\",\
566 \"variant\":\"tt_tok\",\
579 #[test] fn ret_expr() {
580 assert_eq!(string_to_expr(@"return d"),
582 id: ast::DUMMY_NODE_ID,
583 node:ast::ExprRet(Some(@ast::Expr{
584 id: ast::DUMMY_NODE_ID,
585 node:ast::ExprPath(ast::Path{
590 identifier: str_to_ident("d"),
591 lifetimes: opt_vec::Empty,
592 types: opt_vec::Empty,
602 #[test] fn parse_stmt_1 () {
603 assert_eq!(string_to_stmt(@"b;"),
605 node: ast::StmtExpr(@ast::Expr {
606 id: ast::DUMMY_NODE_ID,
607 node: ast::ExprPath(ast::Path {
612 identifier: str_to_ident("b"),
613 lifetimes: opt_vec::Empty,
614 types: opt_vec::Empty,
624 fn parser_done(p: Parser){
625 assert_eq!((*p.token).clone(), token::EOF);
628 #[test] fn parse_ident_pat () {
629 let parser = string_to_parser(@"b");
630 assert_eq!(parser.parse_pat(),
631 @ast::Pat{id: ast::DUMMY_NODE_ID,
633 ast::BindByValue(ast::MutImmutable),
639 identifier: str_to_ident("b"),
640 lifetimes: opt_vec::Empty,
641 types: opt_vec::Empty,
650 // check the contents of the tt manually:
651 #[test] fn parse_fundecl () {
652 // this test depends on the intern order of "fn" and "int"
653 assert_eq!(string_to_item(@"fn a (b : int) { b; }"),
655 @ast::item{ident:str_to_ident("a"),
657 id: ast::DUMMY_NODE_ID,
658 node: ast::item_fn(ast::P(ast::fn_decl{
660 ty: ast::P(ast::Ty{id: ast::DUMMY_NODE_ID,
661 node: ast::ty_path(ast::Path{
668 lifetimes: opt_vec::Empty,
669 types: opt_vec::Empty,
672 }, None, ast::DUMMY_NODE_ID),
676 id: ast::DUMMY_NODE_ID,
678 ast::BindByValue(ast::MutImmutable),
686 lifetimes: opt_vec::Empty,
687 types: opt_vec::Empty,
695 id: ast::DUMMY_NODE_ID
697 output: ast::P(ast::Ty{id: ast::DUMMY_NODE_ID,
699 span:sp(15,15)}), // not sure
705 ast::Generics{ // no idea on either of these:
706 lifetimes: opt_vec::Empty,
707 ty_params: opt_vec::Empty,
712 node: ast::StmtSemi(@ast::Expr{
713 id: ast::DUMMY_NODE_ID,
734 id: ast::DUMMY_NODE_ID,
735 rules: ast::DefaultBlock, // no idea
743 #[test] fn parse_exprs () {
744 // just make sure that they parse....
745 string_to_expr(@"3 + 4");
746 string_to_expr(@"a::z.froob(b,@(987+3))");
749 #[test] fn attrs_fix_bug () {
750 string_to_item(@"pub fn mk_file_writer(path: &Path, flags: &[FileFlag])
751 -> Result<@Writer, ~str> {
754 (O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int
758 fn wb() -> c_int { O_WRONLY as c_int }
760 let mut fflags: c_int = wb();