]> git.lizzy.rs Git - rust.git/blob - src/libsyntax/parse/mod.rs
76ccc53b846cb218830d4a7f61007c8ddb48c60c
[rust.git] / src / libsyntax / parse / mod.rs
1 // Copyright 2012 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
4 //
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
10
11 //! The main parser interface
12
13
14 use ast;
15 use codemap::{Span, CodeMap, FileMap};
16 use codemap;
17 use diagnostic::{SpanHandler, mk_span_handler, mk_handler, Emitter};
18 use parse::attr::ParserAttr;
19 use parse::parser::Parser;
20
21 use std::cell::RefCell;
22 use std::io;
23 use std::io::File;
24 use std::str;
25
26 pub mod lexer;
27 pub mod parser;
28 pub mod token;
29 pub mod comments;
30 pub mod attr;
31
32 /// Common routines shared by parser mods
33 pub mod common;
34
35 /// Routines the parser uses to classify AST nodes
36 pub mod classify;
37
38 /// Reporting obsolete syntax
39 pub mod obsolete;
40
41 // info about a parsing session.
42 pub struct ParseSess {
43     cm: @codemap::CodeMap, // better be the same as the one in the reader!
44     span_diagnostic: @SpanHandler, // better be the same as the one in the reader!
45     /// Used to determine and report recursive mod inclusions
46     included_mod_stack: RefCell<~[Path]>,
47 }
48
49 pub fn new_parse_sess(demitter: Option<@Emitter>) -> @ParseSess {
50     let cm = @CodeMap::new();
51     @ParseSess {
52         cm: cm,
53         span_diagnostic: mk_span_handler(mk_handler(demitter), cm),
54         included_mod_stack: RefCell::new(~[]),
55     }
56 }
57
58 pub fn new_parse_sess_special_handler(sh: @SpanHandler,
59                                       cm: @codemap::CodeMap)
60                                       -> @ParseSess {
61     @ParseSess {
62         cm: cm,
63         span_diagnostic: sh,
64         included_mod_stack: RefCell::new(~[]),
65     }
66 }
67
68 // a bunch of utility functions of the form parse_<thing>_from_<source>
69 // where <thing> includes crate, expr, item, stmt, tts, and one that
70 // uses a HOF to parse anything, and <source> includes file and
71 // source_str.
72
73 pub fn parse_crate_from_file(
74     input: &Path,
75     cfg: ast::CrateConfig,
76     sess: @ParseSess
77 ) -> ast::Crate {
78     new_parser_from_file(sess, /*bad*/ cfg.clone(), input).parse_crate_mod()
79     // why is there no p.abort_if_errors here?
80 }
81
82 pub fn parse_crate_attrs_from_file(
83     input: &Path,
84     cfg: ast::CrateConfig,
85     sess: @ParseSess
86 ) -> ~[ast::Attribute] {
87     let mut parser = new_parser_from_file(sess, cfg, input);
88     let (inner, _) = parser.parse_inner_attrs_and_next();
89     return inner;
90 }
91
92 pub fn parse_crate_from_source_str(name: ~str,
93                                    source: ~str,
94                                    cfg: ast::CrateConfig,
95                                    sess: @ParseSess)
96                                    -> ast::Crate {
97     let mut p = new_parser_from_source_str(sess,
98                                            /*bad*/ cfg.clone(),
99                                            name,
100                                            source);
101     maybe_aborted(p.parse_crate_mod(),p)
102 }
103
104 pub fn parse_crate_attrs_from_source_str(name: ~str,
105                                          source: ~str,
106                                          cfg: ast::CrateConfig,
107                                          sess: @ParseSess)
108                                          -> ~[ast::Attribute] {
109     let mut p = new_parser_from_source_str(sess,
110                                            /*bad*/ cfg.clone(),
111                                            name,
112                                            source);
113     let (inner, _) = maybe_aborted(p.parse_inner_attrs_and_next(),p);
114     return inner;
115 }
116
117 pub fn parse_expr_from_source_str(name: ~str,
118                                   source: ~str,
119                                   cfg: ast::CrateConfig,
120                                   sess: @ParseSess)
121                                   -> @ast::Expr {
122     let mut p = new_parser_from_source_str(sess, cfg, name, source);
123     maybe_aborted(p.parse_expr(), p)
124 }
125
126 pub fn parse_item_from_source_str(name: ~str,
127                                   source: ~str,
128                                   cfg: ast::CrateConfig,
129                                   sess: @ParseSess)
130                                   -> Option<@ast::Item> {
131     let mut p = new_parser_from_source_str(sess, cfg, name, source);
132     let attrs = p.parse_outer_attributes();
133     maybe_aborted(p.parse_item(attrs),p)
134 }
135
136 pub fn parse_meta_from_source_str(name: ~str,
137                                   source: ~str,
138                                   cfg: ast::CrateConfig,
139                                   sess: @ParseSess)
140                                   -> @ast::MetaItem {
141     let mut p = new_parser_from_source_str(sess, cfg, name, source);
142     maybe_aborted(p.parse_meta_item(),p)
143 }
144
145 pub fn parse_stmt_from_source_str(name: ~str,
146                                   source: ~str,
147                                   cfg: ast::CrateConfig,
148                                   attrs: ~[ast::Attribute],
149                                   sess: @ParseSess)
150                                   -> @ast::Stmt {
151     let mut p = new_parser_from_source_str(
152         sess,
153         cfg,
154         name,
155         source
156     );
157     maybe_aborted(p.parse_stmt(attrs),p)
158 }
159
160 pub fn parse_tts_from_source_str(name: ~str,
161                                  source: ~str,
162                                  cfg: ast::CrateConfig,
163                                  sess: @ParseSess)
164                                  -> ~[ast::TokenTree] {
165     let mut p = new_parser_from_source_str(
166         sess,
167         cfg,
168         name,
169         source
170     );
171     p.quote_depth += 1u;
172     // right now this is re-creating the token trees from ... token trees.
173     maybe_aborted(p.parse_all_token_trees(),p)
174 }
175
176 // Create a new parser from a source string
177 pub fn new_parser_from_source_str(sess: @ParseSess,
178                                   cfg: ast::CrateConfig,
179                                   name: ~str,
180                                   source: ~str)
181                                   -> Parser {
182     filemap_to_parser(sess,string_to_filemap(sess,source,name),cfg)
183 }
184
185 /// Create a new parser, handling errors as appropriate
186 /// if the file doesn't exist
187 pub fn new_parser_from_file(
188     sess: @ParseSess,
189     cfg: ast::CrateConfig,
190     path: &Path
191 ) -> Parser {
192     filemap_to_parser(sess,file_to_filemap(sess,path,None),cfg)
193 }
194
195 /// Given a session, a crate config, a path, and a span, add
196 /// the file at the given path to the codemap, and return a parser.
197 /// On an error, use the given span as the source of the problem.
198 pub fn new_sub_parser_from_file(
199     sess: @ParseSess,
200     cfg: ast::CrateConfig,
201     path: &Path,
202     sp: Span
203 ) -> Parser {
204     filemap_to_parser(sess,file_to_filemap(sess,path,Some(sp)),cfg)
205 }
206
207 /// Given a filemap and config, return a parser
208 pub fn filemap_to_parser(sess: @ParseSess,
209                          filemap: @FileMap,
210                          cfg: ast::CrateConfig) -> Parser {
211     tts_to_parser(sess,filemap_to_tts(sess,filemap),cfg)
212 }
213
214 // must preserve old name for now, because quote! from the *existing*
215 // compiler expands into it
216 pub fn new_parser_from_tts(sess: @ParseSess,
217                      cfg: ast::CrateConfig,
218                      tts: ~[ast::TokenTree]) -> Parser {
219     tts_to_parser(sess,tts,cfg)
220 }
221
222
223 // base abstractions
224
225 /// Given a session and a path and an optional span (for error reporting),
226 /// add the path to the session's codemap and return the new filemap.
227 pub fn file_to_filemap(sess: @ParseSess, path: &Path, spanopt: Option<Span>)
228     -> @FileMap {
229     let err = |msg: &str| {
230         match spanopt {
231             Some(sp) => sess.span_diagnostic.span_fatal(sp, msg),
232             None => sess.span_diagnostic.handler().fatal(msg),
233         }
234     };
235     let bytes = match io::result(|| File::open(path).read_to_end()) {
236         Ok(bytes) => bytes,
237         Err(e) => {
238             err(format!("couldn't read {}: {}", path.display(), e.desc));
239             unreachable!()
240         }
241     };
242     match str::from_utf8_owned(bytes) {
243         Some(s) => {
244             return string_to_filemap(sess, s, path.as_str().unwrap().to_str())
245         }
246         None => err(format!("{} is not UTF-8 encoded", path.display())),
247     }
248     unreachable!()
249 }
250
251 // given a session and a string, add the string to
252 // the session's codemap and return the new filemap
253 pub fn string_to_filemap(sess: @ParseSess, source: ~str, path: ~str)
254                          -> @FileMap {
255     sess.cm.new_filemap(path, source)
256 }
257
258 // given a filemap, produce a sequence of token-trees
259 pub fn filemap_to_tts(sess: @ParseSess, filemap: @FileMap)
260     -> ~[ast::TokenTree] {
261     // it appears to me that the cfg doesn't matter here... indeed,
262     // parsing tt's probably shouldn't require a parser at all.
263     let cfg = ~[];
264     let srdr = lexer::new_string_reader(sess.span_diagnostic, filemap);
265     let mut p1 = Parser(sess, cfg, srdr as @lexer::Reader);
266     p1.parse_all_token_trees()
267 }
268
269 // given tts and cfg, produce a parser
270 pub fn tts_to_parser(sess: @ParseSess,
271                      tts: ~[ast::TokenTree],
272                      cfg: ast::CrateConfig) -> Parser {
273     let trdr = lexer::new_tt_reader(sess.span_diagnostic, None, tts);
274     Parser(sess, cfg, trdr as @lexer::Reader)
275 }
276
277 // abort if necessary
278 pub fn maybe_aborted<T>(result: T, mut p: Parser) -> T {
279     p.abort_if_errors();
280     result
281 }
282
283
284
285 #[cfg(test)]
286 mod test {
287     use super::*;
288     use extra::serialize::Encodable;
289     use extra;
290     use std::io;
291     use std::io::MemWriter;
292     use std::str;
293     use codemap::{Span, BytePos, Spanned};
294     use opt_vec;
295     use ast;
296     use abi;
297     use parse::parser::Parser;
298     use parse::token::{str_to_ident};
299     use util::parser_testing::{string_to_tts, string_to_parser};
300     use util::parser_testing::{string_to_expr, string_to_item};
301     use util::parser_testing::string_to_stmt;
302
303     #[cfg(test)]
304     fn to_json_str<'a, E: Encodable<extra::json::Encoder<'a>>>(val: &E) -> ~str {
305         let mut writer = MemWriter::new();
306         let mut encoder = extra::json::Encoder::new(&mut writer as &mut io::Writer);
307         val.encode(&mut encoder);
308         str::from_utf8_owned(writer.unwrap()).unwrap()
309     }
310
311     // produce a codemap::span
312     fn sp(a: u32, b: u32) -> Span {
313         Span{lo:BytePos(a),hi:BytePos(b),expn_info:None}
314     }
315
316     #[test] fn path_exprs_1() {
317         assert_eq!(string_to_expr(@"a"),
318                    @ast::Expr{
319                     id: ast::DUMMY_NODE_ID,
320                     node: ast::ExprPath(ast::Path {
321                         span: sp(0, 1),
322                         global: false,
323                         segments: ~[
324                             ast::PathSegment {
325                                 identifier: str_to_ident("a"),
326                                 lifetimes: opt_vec::Empty,
327                                 types: opt_vec::Empty,
328                             }
329                         ],
330                     }),
331                     span: sp(0, 1)
332                    })
333     }
334
335     #[test] fn path_exprs_2 () {
336         assert_eq!(string_to_expr(@"::a::b"),
337                    @ast::Expr {
338                     id: ast::DUMMY_NODE_ID,
339                     node: ast::ExprPath(ast::Path {
340                             span: sp(0, 6),
341                             global: true,
342                             segments: ~[
343                                 ast::PathSegment {
344                                     identifier: str_to_ident("a"),
345                                     lifetimes: opt_vec::Empty,
346                                     types: opt_vec::Empty,
347                                 },
348                                 ast::PathSegment {
349                                     identifier: str_to_ident("b"),
350                                     lifetimes: opt_vec::Empty,
351                                     types: opt_vec::Empty,
352                                 }
353                             ]
354                         }),
355                     span: sp(0, 6)
356                    })
357     }
358
359     #[should_fail]
360     #[test] fn bad_path_expr_1() {
361         string_to_expr(@"::abc::def::return");
362     }
363
364     // check the token-tree-ization of macros
365     #[test] fn string_to_tts_macro () {
366         let tts = string_to_tts(@"macro_rules! zip (($a)=>($a))");
367         match tts {
368             [ast::TTTok(_,_),
369              ast::TTTok(_,token::NOT),
370              ast::TTTok(_,_),
371              ast::TTDelim(delim_elts)] =>
372                 match *delim_elts {
373                 [ast::TTTok(_,token::LPAREN),
374                  ast::TTDelim(first_set),
375                  ast::TTTok(_,token::FAT_ARROW),
376                  ast::TTDelim(second_set),
377                  ast::TTTok(_,token::RPAREN)] =>
378                     match *first_set {
379                     [ast::TTTok(_,token::LPAREN),
380                      ast::TTTok(_,token::DOLLAR),
381                      ast::TTTok(_,_),
382                      ast::TTTok(_,token::RPAREN)] =>
383                         match *second_set {
384                         [ast::TTTok(_,token::LPAREN),
385                          ast::TTTok(_,token::DOLLAR),
386                          ast::TTTok(_,_),
387                          ast::TTTok(_,token::RPAREN)] =>
388                             assert_eq!("correct","correct"),
389                         _ => assert_eq!("wrong 4","correct")
390                     },
391                     _ => {
392                         error!("failing value 3: {:?}",first_set);
393                         assert_eq!("wrong 3","correct")
394                     }
395                 },
396                 _ => {
397                     error!("failing value 2: {:?}",delim_elts);
398                     assert_eq!("wrong","correct");
399                 }
400
401             },
402             _ => {
403                 error!("failing value: {:?}",tts);
404                 assert_eq!("wrong 1","correct");
405             }
406         }
407     }
408
409     #[test] fn string_to_tts_1 () {
410         let tts = string_to_tts(@"fn a (b : int) { b; }");
411         assert_eq!(to_json_str(&tts),
412         ~"[\
413     {\
414         \"variant\":\"TTTok\",\
415         \"fields\":[\
416             null,\
417             {\
418                 \"variant\":\"IDENT\",\
419                 \"fields\":[\
420                     \"fn\",\
421                     false\
422                 ]\
423             }\
424         ]\
425     },\
426     {\
427         \"variant\":\"TTTok\",\
428         \"fields\":[\
429             null,\
430             {\
431                 \"variant\":\"IDENT\",\
432                 \"fields\":[\
433                     \"a\",\
434                     false\
435                 ]\
436             }\
437         ]\
438     },\
439     {\
440         \"variant\":\"TTDelim\",\
441         \"fields\":[\
442             [\
443                 {\
444                     \"variant\":\"TTTok\",\
445                     \"fields\":[\
446                         null,\
447                         \"LPAREN\"\
448                     ]\
449                 },\
450                 {\
451                     \"variant\":\"TTTok\",\
452                     \"fields\":[\
453                         null,\
454                         {\
455                             \"variant\":\"IDENT\",\
456                             \"fields\":[\
457                                 \"b\",\
458                                 false\
459                             ]\
460                         }\
461                     ]\
462                 },\
463                 {\
464                     \"variant\":\"TTTok\",\
465                     \"fields\":[\
466                         null,\
467                         \"COLON\"\
468                     ]\
469                 },\
470                 {\
471                     \"variant\":\"TTTok\",\
472                     \"fields\":[\
473                         null,\
474                         {\
475                             \"variant\":\"IDENT\",\
476                             \"fields\":[\
477                                 \"int\",\
478                                 false\
479                             ]\
480                         }\
481                     ]\
482                 },\
483                 {\
484                     \"variant\":\"TTTok\",\
485                     \"fields\":[\
486                         null,\
487                         \"RPAREN\"\
488                     ]\
489                 }\
490             ]\
491         ]\
492     },\
493     {\
494         \"variant\":\"TTDelim\",\
495         \"fields\":[\
496             [\
497                 {\
498                     \"variant\":\"TTTok\",\
499                     \"fields\":[\
500                         null,\
501                         \"LBRACE\"\
502                     ]\
503                 },\
504                 {\
505                     \"variant\":\"TTTok\",\
506                     \"fields\":[\
507                         null,\
508                         {\
509                             \"variant\":\"IDENT\",\
510                             \"fields\":[\
511                                 \"b\",\
512                                 false\
513                             ]\
514                         }\
515                     ]\
516                 },\
517                 {\
518                     \"variant\":\"TTTok\",\
519                     \"fields\":[\
520                         null,\
521                         \"SEMI\"\
522                     ]\
523                 },\
524                 {\
525                     \"variant\":\"TTTok\",\
526                     \"fields\":[\
527                         null,\
528                         \"RBRACE\"\
529                     ]\
530                 }\
531             ]\
532         ]\
533     }\
534 ]"
535         );
536     }
537
538     #[test] fn ret_expr() {
539         assert_eq!(string_to_expr(@"return d"),
540                    @ast::Expr{
541                     id: ast::DUMMY_NODE_ID,
542                     node:ast::ExprRet(Some(@ast::Expr{
543                         id: ast::DUMMY_NODE_ID,
544                         node:ast::ExprPath(ast::Path{
545                             span: sp(7, 8),
546                             global: false,
547                             segments: ~[
548                                 ast::PathSegment {
549                                     identifier: str_to_ident("d"),
550                                     lifetimes: opt_vec::Empty,
551                                     types: opt_vec::Empty,
552                                 }
553                             ],
554                         }),
555                         span:sp(7,8)
556                     })),
557                     span:sp(0,8)
558                    })
559     }
560
561     #[test] fn parse_stmt_1 () {
562         assert_eq!(string_to_stmt(@"b;"),
563                    @Spanned{
564                        node: ast::StmtExpr(@ast::Expr {
565                            id: ast::DUMMY_NODE_ID,
566                            node: ast::ExprPath(ast::Path {
567                                span:sp(0,1),
568                                global:false,
569                                segments: ~[
570                                 ast::PathSegment {
571                                     identifier: str_to_ident("b"),
572                                     lifetimes: opt_vec::Empty,
573                                     types: opt_vec::Empty,
574                                 }
575                                ],
576                             }),
577                            span: sp(0,1)},
578                                            ast::DUMMY_NODE_ID),
579                        span: sp(0,1)})
580
581     }
582
583     fn parser_done(p: Parser){
584         assert_eq!(p.token.clone(), token::EOF);
585     }
586
587     #[test] fn parse_ident_pat () {
588         let mut parser = string_to_parser(@"b");
589         assert_eq!(parser.parse_pat(),
590                    @ast::Pat{id: ast::DUMMY_NODE_ID,
591                              node: ast::PatIdent(
592                                 ast::BindByValue(ast::MutImmutable),
593                                 ast::Path {
594                                     span:sp(0,1),
595                                     global:false,
596                                     segments: ~[
597                                         ast::PathSegment {
598                                             identifier: str_to_ident("b"),
599                                             lifetimes: opt_vec::Empty,
600                                             types: opt_vec::Empty,
601                                         }
602                                     ],
603                                 },
604                                 None /* no idea */),
605                              span: sp(0,1)});
606         parser_done(parser);
607     }
608
609     // check the contents of the tt manually:
610     #[test] fn parse_fundecl () {
611         // this test depends on the intern order of "fn" and "int"
612         assert_eq!(string_to_item(@"fn a (b : int) { b; }"),
613                   Some(
614                       @ast::Item{ident:str_to_ident("a"),
615                             attrs:~[],
616                             id: ast::DUMMY_NODE_ID,
617                             node: ast::ItemFn(ast::P(ast::FnDecl {
618                                 inputs: ~[ast::Arg{
619                                     ty: ast::P(ast::Ty{id: ast::DUMMY_NODE_ID,
620                                                        node: ast::TyPath(ast::Path{
621                                         span:sp(10,13),
622                                         global:false,
623                                         segments: ~[
624                                             ast::PathSegment {
625                                                 identifier:
626                                                     str_to_ident("int"),
627                                                 lifetimes: opt_vec::Empty,
628                                                 types: opt_vec::Empty,
629                                             }
630                                         ],
631                                         }, None, ast::DUMMY_NODE_ID),
632                                         span:sp(10,13)
633                                     }),
634                                     pat: @ast::Pat {
635                                         id: ast::DUMMY_NODE_ID,
636                                         node: ast::PatIdent(
637                                             ast::BindByValue(ast::MutImmutable),
638                                             ast::Path {
639                                                 span:sp(6,7),
640                                                 global:false,
641                                                 segments: ~[
642                                                     ast::PathSegment {
643                                                         identifier:
644                                                             str_to_ident("b"),
645                                                         lifetimes: opt_vec::Empty,
646                                                         types: opt_vec::Empty,
647                                                     }
648                                                 ],
649                                             },
650                                             None // no idea
651                                         ),
652                                         span: sp(6,7)
653                                     },
654                                     id: ast::DUMMY_NODE_ID
655                                 }],
656                                 output: ast::P(ast::Ty{id: ast::DUMMY_NODE_ID,
657                                                        node: ast::TyNil,
658                                                        span:sp(15,15)}), // not sure
659                                 cf: ast::Return,
660                                 variadic: false
661                             }),
662                                     ast::ImpureFn,
663                                     abi::AbiSet::Rust(),
664                                     ast::Generics{ // no idea on either of these:
665                                         lifetimes: opt_vec::Empty,
666                                         ty_params: opt_vec::Empty,
667                                     },
668                                     ast::P(ast::Block {
669                                         view_items: ~[],
670                                         stmts: ~[@Spanned{
671                                             node: ast::StmtSemi(@ast::Expr{
672                                                 id: ast::DUMMY_NODE_ID,
673                                                 node: ast::ExprPath(
674                                                       ast::Path{
675                                                         span:sp(17,18),
676                                                         global:false,
677                                                         segments: ~[
678                                                             ast::PathSegment {
679                                                                 identifier:
680                                                                 str_to_ident(
681                                                                     "b"),
682                                                                 lifetimes:
683                                                                 opt_vec::Empty,
684                                                                 types:
685                                                                 opt_vec::Empty
686                                                             }
687                                                         ],
688                                                       }),
689                                                 span: sp(17,18)},
690                                                 ast::DUMMY_NODE_ID),
691                                             span: sp(17,18)}],
692                                         expr: None,
693                                         id: ast::DUMMY_NODE_ID,
694                                         rules: ast::DefaultBlock, // no idea
695                                         span: sp(15,21),
696                                     })),
697                             vis: ast::Inherited,
698                             span: sp(0,21)}));
699     }
700
701
702     #[test] fn parse_exprs () {
703         // just make sure that they parse....
704         string_to_expr(@"3 + 4");
705         string_to_expr(@"a::z.froob(b,@(987+3))");
706     }
707
708     #[test] fn attrs_fix_bug () {
709         string_to_item(@"pub fn mk_file_writer(path: &Path, flags: &[FileFlag])
710                    -> Result<@Writer, ~str> {
711     #[cfg(windows)]
712     fn wb() -> c_int {
713       (O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int
714     }
715
716     #[cfg(unix)]
717     fn wb() -> c_int { O_WRONLY as c_int }
718
719     let mut fflags: c_int = wb();
720 }");
721     }
722
723 }