1 // Copyright 2012 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! The main parser interface
15 use codemap::{Span, CodeMap, FileMap, FileSubstr};
17 use diagnostic::{span_handler, mk_span_handler, mk_handler, Emitter};
18 use parse::attr::parser_attr;
19 use parse::lexer::reader;
20 use parse::parser::Parser;
23 use std::rt::io::File;
32 /// Common routines shared by parser mods
35 /// Routines the parser uses to classify AST nodes
38 /// Reporting obsolete syntax
41 // info about a parsing session.
42 pub struct ParseSess {
43 cm: @codemap::CodeMap, // better be the same as the one in the reader!
44 span_diagnostic: @mut span_handler, // better be the same as the one in the reader!
45 /// Used to determine and report recursive mod inclusions
46 included_mod_stack: ~[Path],
49 pub fn new_parse_sess(demitter: Option<@Emitter>) -> @mut ParseSess {
50 let cm = @CodeMap::new();
53 span_diagnostic: mk_span_handler(mk_handler(demitter), cm),
54 included_mod_stack: ~[],
58 pub fn new_parse_sess_special_handler(sh: @mut span_handler,
59 cm: @codemap::CodeMap)
64 included_mod_stack: ~[],
68 // a bunch of utility functions of the form parse_<thing>_from_<source>
69 // where <thing> includes crate, expr, item, stmt, tts, and one that
70 // uses a HOF to parse anything, and <source> includes file and
73 pub fn parse_crate_from_file(
75 cfg: ast::CrateConfig,
78 new_parser_from_file(sess, /*bad*/ cfg.clone(), input).parse_crate_mod()
79 // why is there no p.abort_if_errors here?
82 pub fn parse_crate_from_source_str(
85 cfg: ast::CrateConfig,
88 let p = new_parser_from_source_str(sess,
92 maybe_aborted(p.parse_crate_mod(),p)
95 pub fn parse_expr_from_source_str(
98 cfg: ast::CrateConfig,
101 let p = new_parser_from_source_str(
107 maybe_aborted(p.parse_expr(), p)
110 pub fn parse_item_from_source_str(
113 cfg: ast::CrateConfig,
114 attrs: ~[ast::Attribute],
116 ) -> Option<@ast::item> {
117 let p = new_parser_from_source_str(
123 maybe_aborted(p.parse_item(attrs),p)
126 pub fn parse_meta_from_source_str(
129 cfg: ast::CrateConfig,
131 ) -> @ast::MetaItem {
132 let p = new_parser_from_source_str(
138 maybe_aborted(p.parse_meta_item(),p)
141 pub fn parse_stmt_from_source_str(
144 cfg: ast::CrateConfig,
145 attrs: ~[ast::Attribute],
148 let p = new_parser_from_source_str(
154 maybe_aborted(p.parse_stmt(attrs),p)
157 pub fn parse_tts_from_source_str(
160 cfg: ast::CrateConfig,
162 ) -> ~[ast::token_tree] {
163 let p = new_parser_from_source_str(
169 *p.quote_depth += 1u;
170 // right now this is re-creating the token trees from ... token trees.
171 maybe_aborted(p.parse_all_token_trees(),p)
174 // given a function and parsing information (source str,
175 // filename, crate cfg, and sess), create a parser,
176 // apply the function, and check that the parser
177 // consumed all of the input before returning the function's
179 pub fn parse_from_source_str<T>(
180 f: &fn(&Parser) -> T,
181 name: @str, ss: codemap::FileSubstr,
183 cfg: ast::CrateConfig,
186 let p = new_parser_from_source_substr(
194 if !p.reader.is_eof() {
195 p.reader.fatal(~"expected end-of-string");
200 // Create a new parser from a source string
201 pub fn new_parser_from_source_str(sess: @mut ParseSess,
202 cfg: ast::CrateConfig,
206 filemap_to_parser(sess,string_to_filemap(sess,source,name),cfg)
209 // Create a new parser from a source string where the origin
210 // is specified as a substring of another file.
211 pub fn new_parser_from_source_substr(sess: @mut ParseSess,
212 cfg: ast::CrateConfig,
214 ss: codemap::FileSubstr,
217 filemap_to_parser(sess,substring_to_filemap(sess,source,name,ss),cfg)
220 /// Create a new parser, handling errors as appropriate
221 /// if the file doesn't exist
222 pub fn new_parser_from_file(
223 sess: @mut ParseSess,
224 cfg: ast::CrateConfig,
227 filemap_to_parser(sess,file_to_filemap(sess,path,None),cfg)
230 /// Given a session, a crate config, a path, and a span, add
231 /// the file at the given path to the codemap, and return a parser.
232 /// On an error, use the given span as the source of the problem.
233 pub fn new_sub_parser_from_file(
234 sess: @mut ParseSess,
235 cfg: ast::CrateConfig,
239 filemap_to_parser(sess,file_to_filemap(sess,path,Some(sp)),cfg)
242 /// Given a filemap and config, return a parser
243 pub fn filemap_to_parser(sess: @mut ParseSess,
245 cfg: ast::CrateConfig) -> Parser {
246 tts_to_parser(sess,filemap_to_tts(sess,filemap),cfg)
249 // must preserve old name for now, because quote! from the *existing*
250 // compiler expands into it
251 pub fn new_parser_from_tts(sess: @mut ParseSess,
252 cfg: ast::CrateConfig,
253 tts: ~[ast::token_tree]) -> Parser {
254 tts_to_parser(sess,tts,cfg)
260 /// Given a session and a path and an optional span (for error reporting),
261 /// add the path to the session's codemap and return the new filemap.
262 pub fn file_to_filemap(sess: @mut ParseSess, path: &Path, spanopt: Option<Span>)
264 let err = |msg: &str| {
266 Some(sp) => sess.span_diagnostic.span_fatal(sp, msg),
267 None => sess.span_diagnostic.handler().fatal(msg),
270 let bytes = match io::result(|| File::open(path).read_to_end()) {
273 err(format!("couldn't read {}: {}", path.display(), e.desc));
277 match str::from_utf8_owned_opt(bytes) {
279 return string_to_filemap(sess, s.to_managed(),
280 path.as_str().unwrap().to_managed());
283 err(format!("{} is not UTF-8 encoded", path.display()))
289 // given a session and a string, add the string to
290 // the session's codemap and return the new filemap
291 pub fn string_to_filemap(sess: @mut ParseSess, source: @str, path: @str)
293 sess.cm.new_filemap(path, source)
296 // given a session and a string and a path and a FileSubStr, add
297 // the string to the CodeMap and return the new FileMap
298 pub fn substring_to_filemap(sess: @mut ParseSess, source: @str, path: @str,
299 filesubstr: FileSubstr) -> @FileMap {
300 sess.cm.new_filemap_w_substr(path,filesubstr,source)
303 // given a filemap, produce a sequence of token-trees
304 pub fn filemap_to_tts(sess: @mut ParseSess, filemap: @FileMap)
305 -> ~[ast::token_tree] {
306 // it appears to me that the cfg doesn't matter here... indeed,
307 // parsing tt's probably shouldn't require a parser at all.
309 let srdr = lexer::new_string_reader(sess.span_diagnostic, filemap);
310 let p1 = Parser(sess, cfg, srdr as @mut reader);
311 p1.parse_all_token_trees()
314 // given tts and cfg, produce a parser
315 pub fn tts_to_parser(sess: @mut ParseSess,
316 tts: ~[ast::token_tree],
317 cfg: ast::CrateConfig) -> Parser {
318 let trdr = lexer::new_tt_reader(sess.span_diagnostic, None, tts);
319 Parser(sess, cfg, trdr as @mut reader)
322 // abort if necessary
323 pub fn maybe_aborted<T>(result : T, p: Parser) -> T {
333 use extra::serialize::Encodable;
336 use std::rt::io::Decorator;
337 use std::rt::io::mem::MemWriter;
339 use codemap::{Span, BytePos, Spanned};
343 use parse::parser::Parser;
344 use parse::token::{str_to_ident};
345 use util::parser_testing::{string_to_tts, string_to_parser};
346 use util::parser_testing::{string_to_expr, string_to_item};
347 use util::parser_testing::string_to_stmt;
349 #[cfg(test)] fn to_json_str<E : Encodable<extra::json::Encoder>>(val: @E) -> ~str {
350 let writer = @mut MemWriter::new();
351 let mut encoder = extra::json::Encoder(writer as @mut io::Writer);
352 val.encode(&mut encoder);
353 str::from_utf8(*writer.inner_ref())
356 // produce a codemap::span
357 fn sp (a: uint, b: uint) -> Span {
358 Span{lo:BytePos(a),hi:BytePos(b),expn_info:None}
361 #[test] fn path_exprs_1() {
362 assert_eq!(string_to_expr(@"a"),
364 id: ast::DUMMY_NODE_ID,
365 node: ast::ExprPath(ast::Path {
370 identifier: str_to_ident("a"),
372 types: opt_vec::Empty,
380 #[test] fn path_exprs_2 () {
381 assert_eq!(string_to_expr(@"::a::b"),
383 id: ast::DUMMY_NODE_ID,
384 node: ast::ExprPath(ast::Path {
389 identifier: str_to_ident("a"),
391 types: opt_vec::Empty,
394 identifier: str_to_ident("b"),
396 types: opt_vec::Empty,
405 #[test] fn bad_path_expr_1() {
406 string_to_expr(@"::abc::def::return");
409 // check the token-tree-ization of macros
410 #[test] fn string_to_tts_macro () {
411 let tts = string_to_tts(@"macro_rules! zip (($a)=>($a))");
414 ast::tt_tok(_,token::NOT),
416 ast::tt_delim(delim_elts)] =>
418 [ast::tt_tok(_,token::LPAREN),
419 ast::tt_delim(first_set),
420 ast::tt_tok(_,token::FAT_ARROW),
421 ast::tt_delim(second_set),
422 ast::tt_tok(_,token::RPAREN)] =>
424 [ast::tt_tok(_,token::LPAREN),
425 ast::tt_tok(_,token::DOLLAR),
427 ast::tt_tok(_,token::RPAREN)] =>
429 [ast::tt_tok(_,token::LPAREN),
430 ast::tt_tok(_,token::DOLLAR),
432 ast::tt_tok(_,token::RPAREN)] =>
433 assert_eq!("correct","correct"),
434 _ => assert_eq!("wrong 4","correct")
437 error!("failing value 3: {:?}",first_set);
438 assert_eq!("wrong 3","correct")
442 error!("failing value 2: {:?}",delim_elts);
443 assert_eq!("wrong","correct");
448 error!("failing value: {:?}",tts);
449 assert_eq!("wrong 1","correct");
454 #[test] fn string_to_tts_1 () {
455 let tts = string_to_tts(@"fn a (b : int) { b; }");
456 assert_eq!(to_json_str(@tts),
459 \"variant\":\"tt_tok\",\
463 \"variant\":\"IDENT\",\
472 \"variant\":\"tt_tok\",\
476 \"variant\":\"IDENT\",\
485 \"variant\":\"tt_delim\",\
489 \"variant\":\"tt_tok\",\
496 \"variant\":\"tt_tok\",\
500 \"variant\":\"IDENT\",\
509 \"variant\":\"tt_tok\",\
516 \"variant\":\"tt_tok\",\
520 \"variant\":\"IDENT\",\
529 \"variant\":\"tt_tok\",\
539 \"variant\":\"tt_delim\",\
543 \"variant\":\"tt_tok\",\
550 \"variant\":\"tt_tok\",\
554 \"variant\":\"IDENT\",\
563 \"variant\":\"tt_tok\",\
570 \"variant\":\"tt_tok\",\
583 #[test] fn ret_expr() {
584 assert_eq!(string_to_expr(@"return d"),
586 id: ast::DUMMY_NODE_ID,
587 node:ast::ExprRet(Some(@ast::Expr{
588 id: ast::DUMMY_NODE_ID,
589 node:ast::ExprPath(ast::Path{
594 identifier: str_to_ident("d"),
596 types: opt_vec::Empty,
606 #[test] fn parse_stmt_1 () {
607 assert_eq!(string_to_stmt(@"b;"),
609 node: ast::StmtExpr(@ast::Expr {
610 id: ast::DUMMY_NODE_ID,
611 node: ast::ExprPath(ast::Path {
616 identifier: str_to_ident("b"),
618 types: opt_vec::Empty,
628 fn parser_done(p: Parser){
629 assert_eq!((*p.token).clone(), token::EOF);
632 #[test] fn parse_ident_pat () {
633 let parser = string_to_parser(@"b");
634 assert_eq!(parser.parse_pat(),
635 @ast::Pat{id: ast::DUMMY_NODE_ID,
637 ast::BindByValue(ast::MutImmutable),
643 identifier: str_to_ident("b"),
645 types: opt_vec::Empty,
654 // check the contents of the tt manually:
655 #[test] fn parse_fundecl () {
656 // this test depends on the intern order of "fn" and "int"
657 assert_eq!(string_to_item(@"fn a (b : int) { b; }"),
659 @ast::item{ident:str_to_ident("a"),
661 id: ast::DUMMY_NODE_ID,
662 node: ast::item_fn(ast::fn_decl{
664 ty: ast::Ty{id: ast::DUMMY_NODE_ID,
665 node: ast::ty_path(ast::Path{
673 types: opt_vec::Empty,
676 }, None, ast::DUMMY_NODE_ID),
680 id: ast::DUMMY_NODE_ID,
682 ast::BindByValue(ast::MutImmutable),
691 types: opt_vec::Empty,
699 id: ast::DUMMY_NODE_ID
701 output: ast::Ty{id: ast::DUMMY_NODE_ID,
703 span:sp(15,15)}, // not sure
709 ast::Generics{ // no idea on either of these:
710 lifetimes: opt_vec::Empty,
711 ty_params: opt_vec::Empty,
716 node: ast::StmtSemi(@ast::Expr{
717 id: ast::DUMMY_NODE_ID,
738 id: ast::DUMMY_NODE_ID,
739 rules: ast::DefaultBlock, // no idea
747 #[test] fn parse_exprs () {
748 // just make sure that they parse....
749 string_to_expr(@"3 + 4");
750 string_to_expr(@"a::z.froob(b,@(987+3))");
753 #[test] fn attrs_fix_bug () {
754 string_to_item(@"pub fn mk_file_writer(path: &Path, flags: &[FileFlag])
755 -> Result<@Writer, ~str> {
758 (O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int
762 fn wb() -> c_int { O_WRONLY as c_int }
764 let mut fflags: c_int = wb();