]> git.lizzy.rs Git - rust.git/blob - src/libsyntax/parse/mod.rs
auto merge of #13600 : brandonw/rust/master, r=brson
[rust.git] / src / libsyntax / parse / mod.rs
1 // Copyright 2012 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
4 //
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
10
11 //! The main parser interface
12
13
14 use ast;
15 use codemap::{Span, CodeMap, FileMap};
16 use diagnostic::{SpanHandler, mk_span_handler, default_handler};
17 use parse::attr::ParserAttr;
18 use parse::parser::Parser;
19
20 use std::cell::RefCell;
21 use std::io::File;
22 use std::rc::Rc;
23 use std::str;
24
25 pub mod lexer;
26 pub mod parser;
27 pub mod token;
28 pub mod comments;
29 pub mod attr;
30
31 /// Common routines shared by parser mods
32 pub mod common;
33
34 /// Routines the parser uses to classify AST nodes
35 pub mod classify;
36
37 /// Reporting obsolete syntax
38 pub mod obsolete;
39
40 // info about a parsing session.
41 pub struct ParseSess {
42     pub span_diagnostic: SpanHandler, // better be the same as the one in the reader!
43     /// Used to determine and report recursive mod inclusions
44     included_mod_stack: RefCell<Vec<Path>>,
45 }
46
47 pub fn new_parse_sess() -> ParseSess {
48     ParseSess {
49         span_diagnostic: mk_span_handler(default_handler(), CodeMap::new()),
50         included_mod_stack: RefCell::new(Vec::new()),
51     }
52 }
53
54 pub fn new_parse_sess_special_handler(sh: SpanHandler) -> ParseSess {
55     ParseSess {
56         span_diagnostic: sh,
57         included_mod_stack: RefCell::new(Vec::new()),
58     }
59 }
60
61 // a bunch of utility functions of the form parse_<thing>_from_<source>
62 // where <thing> includes crate, expr, item, stmt, tts, and one that
63 // uses a HOF to parse anything, and <source> includes file and
64 // source_str.
65
66 pub fn parse_crate_from_file(
67     input: &Path,
68     cfg: ast::CrateConfig,
69     sess: &ParseSess
70 ) -> ast::Crate {
71     new_parser_from_file(sess, cfg, input).parse_crate_mod()
72     // why is there no p.abort_if_errors here?
73 }
74
75 pub fn parse_crate_attrs_from_file(
76     input: &Path,
77     cfg: ast::CrateConfig,
78     sess: &ParseSess
79 ) -> Vec<ast::Attribute> {
80     let mut parser = new_parser_from_file(sess, cfg, input);
81     let (inner, _) = parser.parse_inner_attrs_and_next();
82     inner
83 }
84
85 pub fn parse_crate_from_source_str(name: ~str,
86                                    source: ~str,
87                                    cfg: ast::CrateConfig,
88                                    sess: &ParseSess)
89                                    -> ast::Crate {
90     let mut p = new_parser_from_source_str(sess,
91                                            cfg,
92                                            name,
93                                            source);
94     maybe_aborted(p.parse_crate_mod(),p)
95 }
96
97 pub fn parse_crate_attrs_from_source_str(name: ~str,
98                                          source: ~str,
99                                          cfg: ast::CrateConfig,
100                                          sess: &ParseSess)
101                                          -> Vec<ast::Attribute> {
102     let mut p = new_parser_from_source_str(sess,
103                                            cfg,
104                                            name,
105                                            source);
106     let (inner, _) = maybe_aborted(p.parse_inner_attrs_and_next(),p);
107     inner
108 }
109
110 pub fn parse_expr_from_source_str(name: ~str,
111                                   source: ~str,
112                                   cfg: ast::CrateConfig,
113                                   sess: &ParseSess)
114                                   -> @ast::Expr {
115     let mut p = new_parser_from_source_str(sess, cfg, name, source);
116     maybe_aborted(p.parse_expr(), p)
117 }
118
119 pub fn parse_item_from_source_str(name: ~str,
120                                   source: ~str,
121                                   cfg: ast::CrateConfig,
122                                   sess: &ParseSess)
123                                   -> Option<@ast::Item> {
124     let mut p = new_parser_from_source_str(sess, cfg, name, source);
125     let attrs = p.parse_outer_attributes();
126     maybe_aborted(p.parse_item(attrs),p)
127 }
128
129 pub fn parse_meta_from_source_str(name: ~str,
130                                   source: ~str,
131                                   cfg: ast::CrateConfig,
132                                   sess: &ParseSess)
133                                   -> @ast::MetaItem {
134     let mut p = new_parser_from_source_str(sess, cfg, name, source);
135     maybe_aborted(p.parse_meta_item(),p)
136 }
137
138 pub fn parse_stmt_from_source_str(name: ~str,
139                                   source: ~str,
140                                   cfg: ast::CrateConfig,
141                                   attrs: Vec<ast::Attribute> ,
142                                   sess: &ParseSess)
143                                   -> @ast::Stmt {
144     let mut p = new_parser_from_source_str(
145         sess,
146         cfg,
147         name,
148         source
149     );
150     maybe_aborted(p.parse_stmt(attrs),p)
151 }
152
153 pub fn parse_tts_from_source_str(name: ~str,
154                                  source: ~str,
155                                  cfg: ast::CrateConfig,
156                                  sess: &ParseSess)
157                                  -> Vec<ast::TokenTree> {
158     let mut p = new_parser_from_source_str(
159         sess,
160         cfg,
161         name,
162         source
163     );
164     p.quote_depth += 1u;
165     // right now this is re-creating the token trees from ... token trees.
166     maybe_aborted(p.parse_all_token_trees(),p)
167 }
168
169 // Create a new parser from a source string
170 pub fn new_parser_from_source_str<'a>(sess: &'a ParseSess,
171                                       cfg: ast::CrateConfig,
172                                       name: ~str,
173                                       source: ~str)
174                                       -> Parser<'a> {
175     filemap_to_parser(sess, string_to_filemap(sess, source, name), cfg)
176 }
177
178 /// Create a new parser, handling errors as appropriate
179 /// if the file doesn't exist
180 pub fn new_parser_from_file<'a>(sess: &'a ParseSess,
181                                 cfg: ast::CrateConfig,
182                                 path: &Path) -> Parser<'a> {
183     filemap_to_parser(sess, file_to_filemap(sess, path, None), cfg)
184 }
185
186 /// Given a session, a crate config, a path, and a span, add
187 /// the file at the given path to the codemap, and return a parser.
188 /// On an error, use the given span as the source of the problem.
189 pub fn new_sub_parser_from_file<'a>(sess: &'a ParseSess,
190                                     cfg: ast::CrateConfig,
191                                     path: &Path,
192                                     sp: Span) -> Parser<'a> {
193     filemap_to_parser(sess, file_to_filemap(sess, path, Some(sp)), cfg)
194 }
195
196 /// Given a filemap and config, return a parser
197 pub fn filemap_to_parser<'a>(sess: &'a ParseSess,
198                              filemap: Rc<FileMap>,
199                              cfg: ast::CrateConfig) -> Parser<'a> {
200     tts_to_parser(sess, filemap_to_tts(sess, filemap), cfg)
201 }
202
203 // must preserve old name for now, because quote! from the *existing*
204 // compiler expands into it
205 pub fn new_parser_from_tts<'a>(sess: &'a ParseSess,
206                                cfg: ast::CrateConfig,
207                                tts: Vec<ast::TokenTree>) -> Parser<'a> {
208     tts_to_parser(sess, tts, cfg)
209 }
210
211
212 // base abstractions
213
214 /// Given a session and a path and an optional span (for error reporting),
215 /// add the path to the session's codemap and return the new filemap.
216 pub fn file_to_filemap(sess: &ParseSess, path: &Path, spanopt: Option<Span>)
217     -> Rc<FileMap> {
218     let err = |msg: &str| {
219         match spanopt {
220             Some(sp) => sess.span_diagnostic.span_fatal(sp, msg),
221             None => sess.span_diagnostic.handler().fatal(msg),
222         }
223     };
224     let bytes = match File::open(path).read_to_end() {
225         Ok(bytes) => bytes,
226         Err(e) => {
227             err(format!("couldn't read {}: {}", path.display(), e));
228             unreachable!()
229         }
230     };
231     match str::from_utf8(bytes.as_slice()) {
232         Some(s) => {
233             return string_to_filemap(sess, s.to_owned(),
234                                      path.as_str().unwrap().to_str())
235         }
236         None => err(format!("{} is not UTF-8 encoded", path.display())),
237     }
238     unreachable!()
239 }
240
241 // given a session and a string, add the string to
242 // the session's codemap and return the new filemap
243 pub fn string_to_filemap(sess: &ParseSess, source: ~str, path: ~str)
244                          -> Rc<FileMap> {
245     sess.span_diagnostic.cm.new_filemap(path, source)
246 }
247
248 // given a filemap, produce a sequence of token-trees
249 pub fn filemap_to_tts(sess: &ParseSess, filemap: Rc<FileMap>)
250     -> Vec<ast::TokenTree> {
251     // it appears to me that the cfg doesn't matter here... indeed,
252     // parsing tt's probably shouldn't require a parser at all.
253     let cfg = Vec::new();
254     let srdr = lexer::new_string_reader(&sess.span_diagnostic, filemap);
255     let mut p1 = Parser(sess, cfg, ~srdr);
256     p1.parse_all_token_trees()
257 }
258
259 // given tts and cfg, produce a parser
260 pub fn tts_to_parser<'a>(sess: &'a ParseSess,
261                          tts: Vec<ast::TokenTree>,
262                          cfg: ast::CrateConfig) -> Parser<'a> {
263     let trdr = lexer::new_tt_reader(&sess.span_diagnostic, None, tts);
264     Parser(sess, cfg, ~trdr)
265 }
266
267 // abort if necessary
268 pub fn maybe_aborted<T>(result: T, mut p: Parser) -> T {
269     p.abort_if_errors();
270     result
271 }
272
273
274
275 #[cfg(test)]
276 mod test {
277     use super::*;
278     use serialize::{json, Encodable};
279     use std::io;
280     use std::io::MemWriter;
281     use std::str;
282     use codemap::{Span, BytePos, Spanned};
283     use owned_slice::OwnedSlice;
284     use ast;
285     use abi;
286     use parse::parser::Parser;
287     use parse::token::{str_to_ident};
288     use util::parser_testing::{string_to_tts, string_to_parser};
289     use util::parser_testing::{string_to_expr, string_to_item};
290     use util::parser_testing::string_to_stmt;
291
292     fn to_json_str<'a, E: Encodable<json::Encoder<'a>, io::IoError>>(val: &E) -> ~str {
293         let mut writer = MemWriter::new();
294         let mut encoder = json::Encoder::new(&mut writer as &mut io::Writer);
295         let _ = val.encode(&mut encoder);
296         str::from_utf8(writer.unwrap().as_slice()).unwrap().to_owned()
297     }
298
299     // produce a codemap::span
300     fn sp(a: u32, b: u32) -> Span {
301         Span{lo:BytePos(a),hi:BytePos(b),expn_info:None}
302     }
303
304     #[test] fn path_exprs_1() {
305         assert!(string_to_expr(~"a") ==
306                    @ast::Expr{
307                     id: ast::DUMMY_NODE_ID,
308                     node: ast::ExprPath(ast::Path {
309                         span: sp(0, 1),
310                         global: false,
311                         segments: vec!(
312                             ast::PathSegment {
313                                 identifier: str_to_ident("a"),
314                                 lifetimes: Vec::new(),
315                                 types: OwnedSlice::empty(),
316                             }
317                         ),
318                     }),
319                     span: sp(0, 1)
320                    })
321     }
322
323     #[test] fn path_exprs_2 () {
324         assert!(string_to_expr(~"::a::b") ==
325                    @ast::Expr {
326                     id: ast::DUMMY_NODE_ID,
327                     node: ast::ExprPath(ast::Path {
328                             span: sp(0, 6),
329                             global: true,
330                             segments: vec!(
331                                 ast::PathSegment {
332                                     identifier: str_to_ident("a"),
333                                     lifetimes: Vec::new(),
334                                     types: OwnedSlice::empty(),
335                                 },
336                                 ast::PathSegment {
337                                     identifier: str_to_ident("b"),
338                                     lifetimes: Vec::new(),
339                                     types: OwnedSlice::empty(),
340                                 }
341                             )
342                         }),
343                     span: sp(0, 6)
344                    })
345     }
346
347     #[should_fail]
348     #[test] fn bad_path_expr_1() {
349         string_to_expr(~"::abc::def::return");
350     }
351
352     // check the token-tree-ization of macros
353     #[test] fn string_to_tts_macro () {
354         let tts = string_to_tts(~"macro_rules! zip (($a)=>($a))");
355         let tts: &[ast::TokenTree] = tts.as_slice();
356         match tts {
357             [ast::TTTok(_,_),
358              ast::TTTok(_,token::NOT),
359              ast::TTTok(_,_),
360              ast::TTDelim(ref delim_elts)] => {
361                 let delim_elts: &[ast::TokenTree] = delim_elts.as_slice();
362                 match delim_elts {
363                     [ast::TTTok(_,token::LPAREN),
364                      ast::TTDelim(ref first_set),
365                      ast::TTTok(_,token::FAT_ARROW),
366                      ast::TTDelim(ref second_set),
367                      ast::TTTok(_,token::RPAREN)] => {
368                         let first_set: &[ast::TokenTree] =
369                             first_set.as_slice();
370                         match first_set {
371                             [ast::TTTok(_,token::LPAREN),
372                              ast::TTTok(_,token::DOLLAR),
373                              ast::TTTok(_,_),
374                              ast::TTTok(_,token::RPAREN)] => {
375                                 let second_set: &[ast::TokenTree] =
376                                     second_set.as_slice();
377                                 match second_set {
378                                     [ast::TTTok(_,token::LPAREN),
379                                      ast::TTTok(_,token::DOLLAR),
380                                      ast::TTTok(_,_),
381                                      ast::TTTok(_,token::RPAREN)] => {
382                                         assert_eq!("correct","correct")
383                                     }
384                                     _ => assert_eq!("wrong 4","correct")
385                                 }
386                             },
387                             _ => {
388                                 error!("failing value 3: {:?}",first_set);
389                                 assert_eq!("wrong 3","correct")
390                             }
391                         }
392                     },
393                     _ => {
394                         error!("failing value 2: {:?}",delim_elts);
395                         assert_eq!("wrong","correct");
396                     }
397                 }
398             },
399             _ => {
400                 error!("failing value: {:?}",tts);
401                 assert_eq!("wrong 1","correct");
402             }
403         }
404     }
405
406     #[test] fn string_to_tts_1 () {
407         let tts = string_to_tts(~"fn a (b : int) { b; }");
408         assert_eq!(to_json_str(&tts),
409         ~"[\
410     {\
411         \"variant\":\"TTTok\",\
412         \"fields\":[\
413             null,\
414             {\
415                 \"variant\":\"IDENT\",\
416                 \"fields\":[\
417                     \"fn\",\
418                     false\
419                 ]\
420             }\
421         ]\
422     },\
423     {\
424         \"variant\":\"TTTok\",\
425         \"fields\":[\
426             null,\
427             {\
428                 \"variant\":\"IDENT\",\
429                 \"fields\":[\
430                     \"a\",\
431                     false\
432                 ]\
433             }\
434         ]\
435     },\
436     {\
437         \"variant\":\"TTDelim\",\
438         \"fields\":[\
439             [\
440                 {\
441                     \"variant\":\"TTTok\",\
442                     \"fields\":[\
443                         null,\
444                         \"LPAREN\"\
445                     ]\
446                 },\
447                 {\
448                     \"variant\":\"TTTok\",\
449                     \"fields\":[\
450                         null,\
451                         {\
452                             \"variant\":\"IDENT\",\
453                             \"fields\":[\
454                                 \"b\",\
455                                 false\
456                             ]\
457                         }\
458                     ]\
459                 },\
460                 {\
461                     \"variant\":\"TTTok\",\
462                     \"fields\":[\
463                         null,\
464                         \"COLON\"\
465                     ]\
466                 },\
467                 {\
468                     \"variant\":\"TTTok\",\
469                     \"fields\":[\
470                         null,\
471                         {\
472                             \"variant\":\"IDENT\",\
473                             \"fields\":[\
474                                 \"int\",\
475                                 false\
476                             ]\
477                         }\
478                     ]\
479                 },\
480                 {\
481                     \"variant\":\"TTTok\",\
482                     \"fields\":[\
483                         null,\
484                         \"RPAREN\"\
485                     ]\
486                 }\
487             ]\
488         ]\
489     },\
490     {\
491         \"variant\":\"TTDelim\",\
492         \"fields\":[\
493             [\
494                 {\
495                     \"variant\":\"TTTok\",\
496                     \"fields\":[\
497                         null,\
498                         \"LBRACE\"\
499                     ]\
500                 },\
501                 {\
502                     \"variant\":\"TTTok\",\
503                     \"fields\":[\
504                         null,\
505                         {\
506                             \"variant\":\"IDENT\",\
507                             \"fields\":[\
508                                 \"b\",\
509                                 false\
510                             ]\
511                         }\
512                     ]\
513                 },\
514                 {\
515                     \"variant\":\"TTTok\",\
516                     \"fields\":[\
517                         null,\
518                         \"SEMI\"\
519                     ]\
520                 },\
521                 {\
522                     \"variant\":\"TTTok\",\
523                     \"fields\":[\
524                         null,\
525                         \"RBRACE\"\
526                     ]\
527                 }\
528             ]\
529         ]\
530     }\
531 ]"
532         );
533     }
534
535     #[test] fn ret_expr() {
536         assert!(string_to_expr(~"return d") ==
537                    @ast::Expr{
538                     id: ast::DUMMY_NODE_ID,
539                     node:ast::ExprRet(Some(@ast::Expr{
540                         id: ast::DUMMY_NODE_ID,
541                         node:ast::ExprPath(ast::Path{
542                             span: sp(7, 8),
543                             global: false,
544                             segments: vec!(
545                                 ast::PathSegment {
546                                     identifier: str_to_ident("d"),
547                                     lifetimes: Vec::new(),
548                                     types: OwnedSlice::empty(),
549                                 }
550                             ),
551                         }),
552                         span:sp(7,8)
553                     })),
554                     span:sp(0,8)
555                    })
556     }
557
558     #[test] fn parse_stmt_1 () {
559         assert!(string_to_stmt(~"b;") ==
560                    @Spanned{
561                        node: ast::StmtExpr(@ast::Expr {
562                            id: ast::DUMMY_NODE_ID,
563                            node: ast::ExprPath(ast::Path {
564                                span:sp(0,1),
565                                global:false,
566                                segments: vec!(
567                                 ast::PathSegment {
568                                     identifier: str_to_ident("b"),
569                                     lifetimes: Vec::new(),
570                                     types: OwnedSlice::empty(),
571                                 }
572                                ),
573                             }),
574                            span: sp(0,1)},
575                                            ast::DUMMY_NODE_ID),
576                        span: sp(0,1)})
577
578     }
579
580     fn parser_done(p: Parser){
581         assert_eq!(p.token.clone(), token::EOF);
582     }
583
584     #[test] fn parse_ident_pat () {
585         let sess = new_parse_sess();
586         let mut parser = string_to_parser(&sess, ~"b");
587         assert!(parser.parse_pat() ==
588                    @ast::Pat{id: ast::DUMMY_NODE_ID,
589                              node: ast::PatIdent(
590                                 ast::BindByValue(ast::MutImmutable),
591                                 ast::Path {
592                                     span:sp(0,1),
593                                     global:false,
594                                     segments: vec!(
595                                         ast::PathSegment {
596                                             identifier: str_to_ident("b"),
597                                             lifetimes: Vec::new(),
598                                             types: OwnedSlice::empty(),
599                                         }
600                                     ),
601                                 },
602                                 None /* no idea */),
603                              span: sp(0,1)});
604         parser_done(parser);
605     }
606
607     // check the contents of the tt manually:
608     #[test] fn parse_fundecl () {
609         // this test depends on the intern order of "fn" and "int"
610         assert!(string_to_item(~"fn a (b : int) { b; }") ==
611                   Some(
612                       @ast::Item{ident:str_to_ident("a"),
613                             attrs:Vec::new(),
614                             id: ast::DUMMY_NODE_ID,
615                             node: ast::ItemFn(ast::P(ast::FnDecl {
616                                 inputs: vec!(ast::Arg{
617                                     ty: ast::P(ast::Ty{id: ast::DUMMY_NODE_ID,
618                                                        node: ast::TyPath(ast::Path{
619                                         span:sp(10,13),
620                                         global:false,
621                                         segments: vec!(
622                                             ast::PathSegment {
623                                                 identifier:
624                                                     str_to_ident("int"),
625                                                 lifetimes: Vec::new(),
626                                                 types: OwnedSlice::empty(),
627                                             }
628                                         ),
629                                         }, None, ast::DUMMY_NODE_ID),
630                                         span:sp(10,13)
631                                     }),
632                                     pat: @ast::Pat {
633                                         id: ast::DUMMY_NODE_ID,
634                                         node: ast::PatIdent(
635                                             ast::BindByValue(ast::MutImmutable),
636                                             ast::Path {
637                                                 span:sp(6,7),
638                                                 global:false,
639                                                 segments: vec!(
640                                                     ast::PathSegment {
641                                                         identifier:
642                                                             str_to_ident("b"),
643                                                         lifetimes: Vec::new(),
644                                                         types: OwnedSlice::empty(),
645                                                     }
646                                                 ),
647                                             },
648                                             None // no idea
649                                         ),
650                                         span: sp(6,7)
651                                     },
652                                     id: ast::DUMMY_NODE_ID
653                                 }),
654                                 output: ast::P(ast::Ty{id: ast::DUMMY_NODE_ID,
655                                                        node: ast::TyNil,
656                                                        span:sp(15,15)}), // not sure
657                                 cf: ast::Return,
658                                 variadic: false
659                             }),
660                                     ast::NormalFn,
661                                     abi::Rust,
662                                     ast::Generics{ // no idea on either of these:
663                                         lifetimes: Vec::new(),
664                                         ty_params: OwnedSlice::empty(),
665                                     },
666                                     ast::P(ast::Block {
667                                         view_items: Vec::new(),
668                                         stmts: vec!(@Spanned{
669                                             node: ast::StmtSemi(@ast::Expr{
670                                                 id: ast::DUMMY_NODE_ID,
671                                                 node: ast::ExprPath(
672                                                       ast::Path{
673                                                         span:sp(17,18),
674                                                         global:false,
675                                                         segments: vec!(
676                                                             ast::PathSegment {
677                                                                 identifier:
678                                                                 str_to_ident(
679                                                                     "b"),
680                                                                 lifetimes:
681                                                                 Vec::new(),
682                                                                 types:
683                                                                 OwnedSlice::empty()
684                                                             }
685                                                         ),
686                                                       }),
687                                                 span: sp(17,18)},
688                                                 ast::DUMMY_NODE_ID),
689                                             span: sp(17,18)}),
690                                         expr: None,
691                                         id: ast::DUMMY_NODE_ID,
692                                         rules: ast::DefaultBlock, // no idea
693                                         span: sp(15,21),
694                                     })),
695                             vis: ast::Inherited,
696                             span: sp(0,21)}));
697     }
698
699
700     #[test] fn parse_exprs () {
701         // just make sure that they parse....
702         string_to_expr(~"3 + 4");
703         string_to_expr(~"a::z.froob(b,@(987+3))");
704     }
705
706     #[test] fn attrs_fix_bug () {
707         string_to_item(~"pub fn mk_file_writer(path: &Path, flags: &[FileFlag])
708                    -> Result<@Writer, ~str> {
709     #[cfg(windows)]
710     fn wb() -> c_int {
711       (O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int
712     }
713
714     #[cfg(unix)]
715     fn wb() -> c_int { O_WRONLY as c_int }
716
717     let mut fflags: c_int = wb();
718 }");
719     }
720
721 }