]> git.lizzy.rs Git - rust.git/blob - src/libsyntax/parse/mod.rs
auto merge of #11598 : alexcrichton/rust/io-export, r=brson
[rust.git] / src / libsyntax / parse / mod.rs
1 // Copyright 2012 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
4 //
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
10
11 //! The main parser interface
12
13
14 use ast;
15 use codemap::{Span, CodeMap, FileMap};
16 use codemap;
17 use diagnostic::{SpanHandler, mk_span_handler, mk_handler, Emitter};
18 use parse::attr::ParserAttr;
19 use parse::parser::Parser;
20
21 use std::cell::RefCell;
22 use std::io;
23 use std::io::File;
24 use std::str;
25
26 pub mod lexer;
27 pub mod parser;
28 pub mod token;
29 pub mod comments;
30 pub mod attr;
31
32 /// Common routines shared by parser mods
33 pub mod common;
34
35 /// Routines the parser uses to classify AST nodes
36 pub mod classify;
37
38 /// Reporting obsolete syntax
39 pub mod obsolete;
40
41 // info about a parsing session.
42 pub struct ParseSess {
43     cm: @codemap::CodeMap, // better be the same as the one in the reader!
44     span_diagnostic: @SpanHandler, // better be the same as the one in the reader!
45     /// Used to determine and report recursive mod inclusions
46     included_mod_stack: RefCell<~[Path]>,
47 }
48
49 pub fn new_parse_sess(demitter: Option<@Emitter>) -> @ParseSess {
50     let cm = @CodeMap::new();
51     @ParseSess {
52         cm: cm,
53         span_diagnostic: mk_span_handler(mk_handler(demitter), cm),
54         included_mod_stack: RefCell::new(~[]),
55     }
56 }
57
58 pub fn new_parse_sess_special_handler(sh: @SpanHandler,
59                                       cm: @codemap::CodeMap)
60                                       -> @ParseSess {
61     @ParseSess {
62         cm: cm,
63         span_diagnostic: sh,
64         included_mod_stack: RefCell::new(~[]),
65     }
66 }
67
68 // a bunch of utility functions of the form parse_<thing>_from_<source>
69 // where <thing> includes crate, expr, item, stmt, tts, and one that
70 // uses a HOF to parse anything, and <source> includes file and
71 // source_str.
72
73 pub fn parse_crate_from_file(
74     input: &Path,
75     cfg: ast::CrateConfig,
76     sess: @ParseSess
77 ) -> ast::Crate {
78     new_parser_from_file(sess, /*bad*/ cfg.clone(), input).parse_crate_mod()
79     // why is there no p.abort_if_errors here?
80 }
81
82 pub fn parse_crate_attrs_from_file(
83     input: &Path,
84     cfg: ast::CrateConfig,
85     sess: @ParseSess
86 ) -> ~[ast::Attribute] {
87     let mut parser = new_parser_from_file(sess, cfg, input);
88     let (inner, _) = parser.parse_inner_attrs_and_next();
89     return inner;
90 }
91
92 pub fn parse_crate_from_source_str(
93     name: @str,
94     source: @str,
95     cfg: ast::CrateConfig,
96     sess: @ParseSess
97 ) -> ast::Crate {
98     let mut p = new_parser_from_source_str(sess,
99                                            /*bad*/ cfg.clone(),
100                                            name,
101                                            source);
102     maybe_aborted(p.parse_crate_mod(),p)
103 }
104
105 pub fn parse_crate_attrs_from_source_str(
106     name: @str,
107     source: @str,
108     cfg: ast::CrateConfig,
109     sess: @ParseSess
110 ) -> ~[ast::Attribute] {
111     let mut p = new_parser_from_source_str(sess,
112                                            /*bad*/ cfg.clone(),
113                                            name,
114                                            source);
115     let (inner, _) = maybe_aborted(p.parse_inner_attrs_and_next(),p);
116     return inner;
117 }
118
119 pub fn parse_expr_from_source_str(
120     name: @str,
121     source: @str,
122     cfg: ast::CrateConfig,
123     sess: @ParseSess
124 ) -> @ast::Expr {
125     let mut p = new_parser_from_source_str(sess, cfg, name, source);
126     maybe_aborted(p.parse_expr(), p)
127 }
128
129 pub fn parse_item_from_source_str(
130     name: @str,
131     source: @str,
132     cfg: ast::CrateConfig,
133     attrs: ~[ast::Attribute],
134     sess: @ParseSess
135 ) -> Option<@ast::Item> {
136     let mut p = new_parser_from_source_str(sess, cfg, name, source);
137     maybe_aborted(p.parse_item(attrs),p)
138 }
139
140 pub fn parse_meta_from_source_str(
141     name: @str,
142     source: @str,
143     cfg: ast::CrateConfig,
144     sess: @ParseSess
145 ) -> @ast::MetaItem {
146     let mut p = new_parser_from_source_str(sess, cfg, name, source);
147     maybe_aborted(p.parse_meta_item(),p)
148 }
149
150 pub fn parse_stmt_from_source_str(
151     name: @str,
152     source: @str,
153     cfg: ast::CrateConfig,
154     attrs: ~[ast::Attribute],
155     sess: @ParseSess
156 ) -> @ast::Stmt {
157     let mut p = new_parser_from_source_str(
158         sess,
159         cfg,
160         name,
161         source
162     );
163     maybe_aborted(p.parse_stmt(attrs),p)
164 }
165
166 pub fn parse_tts_from_source_str(
167     name: @str,
168     source: @str,
169     cfg: ast::CrateConfig,
170     sess: @ParseSess
171 ) -> ~[ast::TokenTree] {
172     let mut p = new_parser_from_source_str(
173         sess,
174         cfg,
175         name,
176         source
177     );
178     p.quote_depth += 1u;
179     // right now this is re-creating the token trees from ... token trees.
180     maybe_aborted(p.parse_all_token_trees(),p)
181 }
182
183 // Create a new parser from a source string
184 pub fn new_parser_from_source_str(sess: @ParseSess,
185                                   cfg: ast::CrateConfig,
186                                   name: @str,
187                                   source: @str)
188                                -> Parser {
189     filemap_to_parser(sess,string_to_filemap(sess,source,name),cfg)
190 }
191
192 /// Create a new parser, handling errors as appropriate
193 /// if the file doesn't exist
194 pub fn new_parser_from_file(
195     sess: @ParseSess,
196     cfg: ast::CrateConfig,
197     path: &Path
198 ) -> Parser {
199     filemap_to_parser(sess,file_to_filemap(sess,path,None),cfg)
200 }
201
202 /// Given a session, a crate config, a path, and a span, add
203 /// the file at the given path to the codemap, and return a parser.
204 /// On an error, use the given span as the source of the problem.
205 pub fn new_sub_parser_from_file(
206     sess: @ParseSess,
207     cfg: ast::CrateConfig,
208     path: &Path,
209     sp: Span
210 ) -> Parser {
211     filemap_to_parser(sess,file_to_filemap(sess,path,Some(sp)),cfg)
212 }
213
214 /// Given a filemap and config, return a parser
215 pub fn filemap_to_parser(sess: @ParseSess,
216                          filemap: @FileMap,
217                          cfg: ast::CrateConfig) -> Parser {
218     tts_to_parser(sess,filemap_to_tts(sess,filemap),cfg)
219 }
220
221 // must preserve old name for now, because quote! from the *existing*
222 // compiler expands into it
223 pub fn new_parser_from_tts(sess: @ParseSess,
224                      cfg: ast::CrateConfig,
225                      tts: ~[ast::TokenTree]) -> Parser {
226     tts_to_parser(sess,tts,cfg)
227 }
228
229
230 // base abstractions
231
232 /// Given a session and a path and an optional span (for error reporting),
233 /// add the path to the session's codemap and return the new filemap.
234 pub fn file_to_filemap(sess: @ParseSess, path: &Path, spanopt: Option<Span>)
235     -> @FileMap {
236     let err = |msg: &str| {
237         match spanopt {
238             Some(sp) => sess.span_diagnostic.span_fatal(sp, msg),
239             None => sess.span_diagnostic.handler().fatal(msg),
240         }
241     };
242     let bytes = match io::result(|| File::open(path).read_to_end()) {
243         Ok(bytes) => bytes,
244         Err(e) => {
245             err(format!("couldn't read {}: {}", path.display(), e.desc));
246             unreachable!()
247         }
248     };
249     match str::from_utf8_owned_opt(bytes) {
250         Some(s) => {
251             return string_to_filemap(sess, s.to_managed(),
252                                      path.as_str().unwrap().to_managed());
253         }
254         None => {
255             err(format!("{} is not UTF-8 encoded", path.display()))
256         }
257     }
258     unreachable!()
259 }
260
261 // given a session and a string, add the string to
262 // the session's codemap and return the new filemap
263 pub fn string_to_filemap(sess: @ParseSess, source: @str, path: @str)
264     -> @FileMap {
265     sess.cm.new_filemap(path, source)
266 }
267
268 // given a filemap, produce a sequence of token-trees
269 pub fn filemap_to_tts(sess: @ParseSess, filemap: @FileMap)
270     -> ~[ast::TokenTree] {
271     // it appears to me that the cfg doesn't matter here... indeed,
272     // parsing tt's probably shouldn't require a parser at all.
273     let cfg = ~[];
274     let srdr = lexer::new_string_reader(sess.span_diagnostic, filemap);
275     let mut p1 = Parser(sess, cfg, srdr as @lexer::Reader);
276     p1.parse_all_token_trees()
277 }
278
279 // given tts and cfg, produce a parser
280 pub fn tts_to_parser(sess: @ParseSess,
281                      tts: ~[ast::TokenTree],
282                      cfg: ast::CrateConfig) -> Parser {
283     let trdr = lexer::new_tt_reader(sess.span_diagnostic, None, tts);
284     Parser(sess, cfg, trdr as @lexer::Reader)
285 }
286
287 // abort if necessary
288 pub fn maybe_aborted<T>(result: T, mut p: Parser) -> T {
289     p.abort_if_errors();
290     result
291 }
292
293
294
295 #[cfg(test)]
296 mod test {
297     use super::*;
298     use extra::serialize::Encodable;
299     use extra;
300     use std::io;
301     use std::io::MemWriter;
302     use std::str;
303     use codemap::{Span, BytePos, Spanned};
304     use opt_vec;
305     use ast;
306     use abi;
307     use parse::parser::Parser;
308     use parse::token::{str_to_ident};
309     use util::parser_testing::{string_to_tts, string_to_parser};
310     use util::parser_testing::{string_to_expr, string_to_item};
311     use util::parser_testing::string_to_stmt;
312
313     #[cfg(test)]
314     fn to_json_str<'a, E: Encodable<extra::json::Encoder<'a>>>(val: &E) -> ~str {
315         let mut writer = MemWriter::new();
316         let mut encoder = extra::json::Encoder::new(&mut writer as &mut io::Writer);
317         val.encode(&mut encoder);
318         str::from_utf8_owned(writer.unwrap())
319     }
320
321     // produce a codemap::span
322     fn sp(a: u32, b: u32) -> Span {
323         Span{lo:BytePos(a),hi:BytePos(b),expn_info:None}
324     }
325
326     #[test] fn path_exprs_1() {
327         assert_eq!(string_to_expr(@"a"),
328                    @ast::Expr{
329                     id: ast::DUMMY_NODE_ID,
330                     node: ast::ExprPath(ast::Path {
331                         span: sp(0, 1),
332                         global: false,
333                         segments: ~[
334                             ast::PathSegment {
335                                 identifier: str_to_ident("a"),
336                                 lifetimes: opt_vec::Empty,
337                                 types: opt_vec::Empty,
338                             }
339                         ],
340                     }),
341                     span: sp(0, 1)
342                    })
343     }
344
345     #[test] fn path_exprs_2 () {
346         assert_eq!(string_to_expr(@"::a::b"),
347                    @ast::Expr {
348                     id: ast::DUMMY_NODE_ID,
349                     node: ast::ExprPath(ast::Path {
350                             span: sp(0, 6),
351                             global: true,
352                             segments: ~[
353                                 ast::PathSegment {
354                                     identifier: str_to_ident("a"),
355                                     lifetimes: opt_vec::Empty,
356                                     types: opt_vec::Empty,
357                                 },
358                                 ast::PathSegment {
359                                     identifier: str_to_ident("b"),
360                                     lifetimes: opt_vec::Empty,
361                                     types: opt_vec::Empty,
362                                 }
363                             ]
364                         }),
365                     span: sp(0, 6)
366                    })
367     }
368
369     #[should_fail]
370     #[test] fn bad_path_expr_1() {
371         string_to_expr(@"::abc::def::return");
372     }
373
374     // check the token-tree-ization of macros
375     #[test] fn string_to_tts_macro () {
376         let tts = string_to_tts(@"macro_rules! zip (($a)=>($a))");
377         match tts {
378             [ast::TTTok(_,_),
379              ast::TTTok(_,token::NOT),
380              ast::TTTok(_,_),
381              ast::TTDelim(delim_elts)] =>
382                 match *delim_elts {
383                 [ast::TTTok(_,token::LPAREN),
384                  ast::TTDelim(first_set),
385                  ast::TTTok(_,token::FAT_ARROW),
386                  ast::TTDelim(second_set),
387                  ast::TTTok(_,token::RPAREN)] =>
388                     match *first_set {
389                     [ast::TTTok(_,token::LPAREN),
390                      ast::TTTok(_,token::DOLLAR),
391                      ast::TTTok(_,_),
392                      ast::TTTok(_,token::RPAREN)] =>
393                         match *second_set {
394                         [ast::TTTok(_,token::LPAREN),
395                          ast::TTTok(_,token::DOLLAR),
396                          ast::TTTok(_,_),
397                          ast::TTTok(_,token::RPAREN)] =>
398                             assert_eq!("correct","correct"),
399                         _ => assert_eq!("wrong 4","correct")
400                     },
401                     _ => {
402                         error!("failing value 3: {:?}",first_set);
403                         assert_eq!("wrong 3","correct")
404                     }
405                 },
406                 _ => {
407                     error!("failing value 2: {:?}",delim_elts);
408                     assert_eq!("wrong","correct");
409                 }
410
411             },
412             _ => {
413                 error!("failing value: {:?}",tts);
414                 assert_eq!("wrong 1","correct");
415             }
416         }
417     }
418
419     #[test] fn string_to_tts_1 () {
420         let tts = string_to_tts(@"fn a (b : int) { b; }");
421         assert_eq!(to_json_str(&tts),
422         ~"[\
423     {\
424         \"variant\":\"TTTok\",\
425         \"fields\":[\
426             null,\
427             {\
428                 \"variant\":\"IDENT\",\
429                 \"fields\":[\
430                     \"fn\",\
431                     false\
432                 ]\
433             }\
434         ]\
435     },\
436     {\
437         \"variant\":\"TTTok\",\
438         \"fields\":[\
439             null,\
440             {\
441                 \"variant\":\"IDENT\",\
442                 \"fields\":[\
443                     \"a\",\
444                     false\
445                 ]\
446             }\
447         ]\
448     },\
449     {\
450         \"variant\":\"TTDelim\",\
451         \"fields\":[\
452             [\
453                 {\
454                     \"variant\":\"TTTok\",\
455                     \"fields\":[\
456                         null,\
457                         \"LPAREN\"\
458                     ]\
459                 },\
460                 {\
461                     \"variant\":\"TTTok\",\
462                     \"fields\":[\
463                         null,\
464                         {\
465                             \"variant\":\"IDENT\",\
466                             \"fields\":[\
467                                 \"b\",\
468                                 false\
469                             ]\
470                         }\
471                     ]\
472                 },\
473                 {\
474                     \"variant\":\"TTTok\",\
475                     \"fields\":[\
476                         null,\
477                         \"COLON\"\
478                     ]\
479                 },\
480                 {\
481                     \"variant\":\"TTTok\",\
482                     \"fields\":[\
483                         null,\
484                         {\
485                             \"variant\":\"IDENT\",\
486                             \"fields\":[\
487                                 \"int\",\
488                                 false\
489                             ]\
490                         }\
491                     ]\
492                 },\
493                 {\
494                     \"variant\":\"TTTok\",\
495                     \"fields\":[\
496                         null,\
497                         \"RPAREN\"\
498                     ]\
499                 }\
500             ]\
501         ]\
502     },\
503     {\
504         \"variant\":\"TTDelim\",\
505         \"fields\":[\
506             [\
507                 {\
508                     \"variant\":\"TTTok\",\
509                     \"fields\":[\
510                         null,\
511                         \"LBRACE\"\
512                     ]\
513                 },\
514                 {\
515                     \"variant\":\"TTTok\",\
516                     \"fields\":[\
517                         null,\
518                         {\
519                             \"variant\":\"IDENT\",\
520                             \"fields\":[\
521                                 \"b\",\
522                                 false\
523                             ]\
524                         }\
525                     ]\
526                 },\
527                 {\
528                     \"variant\":\"TTTok\",\
529                     \"fields\":[\
530                         null,\
531                         \"SEMI\"\
532                     ]\
533                 },\
534                 {\
535                     \"variant\":\"TTTok\",\
536                     \"fields\":[\
537                         null,\
538                         \"RBRACE\"\
539                     ]\
540                 }\
541             ]\
542         ]\
543     }\
544 ]"
545         );
546     }
547
548     #[test] fn ret_expr() {
549         assert_eq!(string_to_expr(@"return d"),
550                    @ast::Expr{
551                     id: ast::DUMMY_NODE_ID,
552                     node:ast::ExprRet(Some(@ast::Expr{
553                         id: ast::DUMMY_NODE_ID,
554                         node:ast::ExprPath(ast::Path{
555                             span: sp(7, 8),
556                             global: false,
557                             segments: ~[
558                                 ast::PathSegment {
559                                     identifier: str_to_ident("d"),
560                                     lifetimes: opt_vec::Empty,
561                                     types: opt_vec::Empty,
562                                 }
563                             ],
564                         }),
565                         span:sp(7,8)
566                     })),
567                     span:sp(0,8)
568                    })
569     }
570
571     #[test] fn parse_stmt_1 () {
572         assert_eq!(string_to_stmt(@"b;"),
573                    @Spanned{
574                        node: ast::StmtExpr(@ast::Expr {
575                            id: ast::DUMMY_NODE_ID,
576                            node: ast::ExprPath(ast::Path {
577                                span:sp(0,1),
578                                global:false,
579                                segments: ~[
580                                 ast::PathSegment {
581                                     identifier: str_to_ident("b"),
582                                     lifetimes: opt_vec::Empty,
583                                     types: opt_vec::Empty,
584                                 }
585                                ],
586                             }),
587                            span: sp(0,1)},
588                                            ast::DUMMY_NODE_ID),
589                        span: sp(0,1)})
590
591     }
592
593     fn parser_done(p: Parser){
594         assert_eq!(p.token.clone(), token::EOF);
595     }
596
597     #[test] fn parse_ident_pat () {
598         let mut parser = string_to_parser(@"b");
599         assert_eq!(parser.parse_pat(),
600                    @ast::Pat{id: ast::DUMMY_NODE_ID,
601                              node: ast::PatIdent(
602                                 ast::BindByValue(ast::MutImmutable),
603                                 ast::Path {
604                                     span:sp(0,1),
605                                     global:false,
606                                     segments: ~[
607                                         ast::PathSegment {
608                                             identifier: str_to_ident("b"),
609                                             lifetimes: opt_vec::Empty,
610                                             types: opt_vec::Empty,
611                                         }
612                                     ],
613                                 },
614                                 None /* no idea */),
615                              span: sp(0,1)});
616         parser_done(parser);
617     }
618
619     // check the contents of the tt manually:
620     #[test] fn parse_fundecl () {
621         // this test depends on the intern order of "fn" and "int"
622         assert_eq!(string_to_item(@"fn a (b : int) { b; }"),
623                   Some(
624                       @ast::Item{ident:str_to_ident("a"),
625                             attrs:~[],
626                             id: ast::DUMMY_NODE_ID,
627                             node: ast::ItemFn(ast::P(ast::FnDecl {
628                                 inputs: ~[ast::Arg{
629                                     ty: ast::P(ast::Ty{id: ast::DUMMY_NODE_ID,
630                                                        node: ast::TyPath(ast::Path{
631                                         span:sp(10,13),
632                                         global:false,
633                                         segments: ~[
634                                             ast::PathSegment {
635                                                 identifier:
636                                                     str_to_ident("int"),
637                                                 lifetimes: opt_vec::Empty,
638                                                 types: opt_vec::Empty,
639                                             }
640                                         ],
641                                         }, None, ast::DUMMY_NODE_ID),
642                                         span:sp(10,13)
643                                     }),
644                                     pat: @ast::Pat {
645                                         id: ast::DUMMY_NODE_ID,
646                                         node: ast::PatIdent(
647                                             ast::BindByValue(ast::MutImmutable),
648                                             ast::Path {
649                                                 span:sp(6,7),
650                                                 global:false,
651                                                 segments: ~[
652                                                     ast::PathSegment {
653                                                         identifier:
654                                                             str_to_ident("b"),
655                                                         lifetimes: opt_vec::Empty,
656                                                         types: opt_vec::Empty,
657                                                     }
658                                                 ],
659                                             },
660                                             None // no idea
661                                         ),
662                                         span: sp(6,7)
663                                     },
664                                     id: ast::DUMMY_NODE_ID
665                                 }],
666                                 output: ast::P(ast::Ty{id: ast::DUMMY_NODE_ID,
667                                                        node: ast::TyNil,
668                                                        span:sp(15,15)}), // not sure
669                                 cf: ast::Return,
670                                 variadic: false
671                             }),
672                                     ast::ImpureFn,
673                                     abi::AbiSet::Rust(),
674                                     ast::Generics{ // no idea on either of these:
675                                         lifetimes: opt_vec::Empty,
676                                         ty_params: opt_vec::Empty,
677                                     },
678                                     ast::P(ast::Block {
679                                         view_items: ~[],
680                                         stmts: ~[@Spanned{
681                                             node: ast::StmtSemi(@ast::Expr{
682                                                 id: ast::DUMMY_NODE_ID,
683                                                 node: ast::ExprPath(
684                                                       ast::Path{
685                                                         span:sp(17,18),
686                                                         global:false,
687                                                         segments: ~[
688                                                             ast::PathSegment {
689                                                                 identifier:
690                                                                 str_to_ident(
691                                                                     "b"),
692                                                                 lifetimes:
693                                                                 opt_vec::Empty,
694                                                                 types:
695                                                                 opt_vec::Empty
696                                                             }
697                                                         ],
698                                                       }),
699                                                 span: sp(17,18)},
700                                                 ast::DUMMY_NODE_ID),
701                                             span: sp(17,18)}],
702                                         expr: None,
703                                         id: ast::DUMMY_NODE_ID,
704                                         rules: ast::DefaultBlock, // no idea
705                                         span: sp(15,21),
706                                     })),
707                             vis: ast::Inherited,
708                             span: sp(0,21)}));
709     }
710
711
712     #[test] fn parse_exprs () {
713         // just make sure that they parse....
714         string_to_expr(@"3 + 4");
715         string_to_expr(@"a::z.froob(b,@(987+3))");
716     }
717
718     #[test] fn attrs_fix_bug () {
719         string_to_item(@"pub fn mk_file_writer(path: &Path, flags: &[FileFlag])
720                    -> Result<@Writer, ~str> {
721     #[cfg(windows)]
722     fn wb() -> c_int {
723       (O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int
724     }
725
726     #[cfg(unix)]
727     fn wb() -> c_int { O_WRONLY as c_int }
728
729     let mut fflags: c_int = wb();
730 }");
731     }
732
733 }