]> git.lizzy.rs Git - rust.git/blob - src/libsyntax/parse/mod.rs
libsyntax/librustc: Allow calling variadic foreign functions.
[rust.git] / src / libsyntax / parse / mod.rs
1 // Copyright 2012 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
4 //
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
10
11 //! The main parser interface
12
13
14 use ast;
15 use codemap::{Span, CodeMap, FileMap, FileSubstr};
16 use codemap;
17 use diagnostic::{span_handler, mk_span_handler, mk_handler, Emitter};
18 use parse::attr::parser_attr;
19 use parse::lexer::reader;
20 use parse::parser::Parser;
21
22 use std::rt::io;
23 use std::rt::io::File;
24 use std::str;
25
26 pub mod lexer;
27 pub mod parser;
28 pub mod token;
29 pub mod comments;
30 pub mod attr;
31
32 /// Common routines shared by parser mods
33 pub mod common;
34
35 /// Routines the parser uses to classify AST nodes
36 pub mod classify;
37
38 /// Reporting obsolete syntax
39 pub mod obsolete;
40
41 // info about a parsing session.
42 pub struct ParseSess {
43     cm: @codemap::CodeMap, // better be the same as the one in the reader!
44     span_diagnostic: @mut span_handler, // better be the same as the one in the reader!
45     /// Used to determine and report recursive mod inclusions
46     included_mod_stack: ~[Path],
47 }
48
49 pub fn new_parse_sess(demitter: Option<@Emitter>) -> @mut ParseSess {
50     let cm = @CodeMap::new();
51     @mut ParseSess {
52         cm: cm,
53         span_diagnostic: mk_span_handler(mk_handler(demitter), cm),
54         included_mod_stack: ~[],
55     }
56 }
57
58 pub fn new_parse_sess_special_handler(sh: @mut span_handler,
59                                       cm: @codemap::CodeMap)
60                                    -> @mut ParseSess {
61     @mut ParseSess {
62         cm: cm,
63         span_diagnostic: sh,
64         included_mod_stack: ~[],
65     }
66 }
67
68 // a bunch of utility functions of the form parse_<thing>_from_<source>
69 // where <thing> includes crate, expr, item, stmt, tts, and one that
70 // uses a HOF to parse anything, and <source> includes file and
71 // source_str.
72
73 pub fn parse_crate_from_file(
74     input: &Path,
75     cfg: ast::CrateConfig,
76     sess: @mut ParseSess
77 ) -> ast::Crate {
78     new_parser_from_file(sess, /*bad*/ cfg.clone(), input).parse_crate_mod()
79     // why is there no p.abort_if_errors here?
80 }
81
82 pub fn parse_crate_from_source_str(
83     name: @str,
84     source: @str,
85     cfg: ast::CrateConfig,
86     sess: @mut ParseSess
87 ) -> ast::Crate {
88     let p = new_parser_from_source_str(sess,
89                                        /*bad*/ cfg.clone(),
90                                        name,
91                                        source);
92     maybe_aborted(p.parse_crate_mod(),p)
93 }
94
95 pub fn parse_expr_from_source_str(
96     name: @str,
97     source: @str,
98     cfg: ast::CrateConfig,
99     sess: @mut ParseSess
100 ) -> @ast::Expr {
101     let p = new_parser_from_source_str(
102         sess,
103         cfg,
104         name,
105         source
106     );
107     maybe_aborted(p.parse_expr(), p)
108 }
109
110 pub fn parse_item_from_source_str(
111     name: @str,
112     source: @str,
113     cfg: ast::CrateConfig,
114     attrs: ~[ast::Attribute],
115     sess: @mut ParseSess
116 ) -> Option<@ast::item> {
117     let p = new_parser_from_source_str(
118         sess,
119         cfg,
120         name,
121         source
122     );
123     maybe_aborted(p.parse_item(attrs),p)
124 }
125
126 pub fn parse_meta_from_source_str(
127     name: @str,
128     source: @str,
129     cfg: ast::CrateConfig,
130     sess: @mut ParseSess
131 ) -> @ast::MetaItem {
132     let p = new_parser_from_source_str(
133         sess,
134         cfg,
135         name,
136         source
137     );
138     maybe_aborted(p.parse_meta_item(),p)
139 }
140
141 pub fn parse_stmt_from_source_str(
142     name: @str,
143     source: @str,
144     cfg: ast::CrateConfig,
145     attrs: ~[ast::Attribute],
146     sess: @mut ParseSess
147 ) -> @ast::Stmt {
148     let p = new_parser_from_source_str(
149         sess,
150         cfg,
151         name,
152         source
153     );
154     maybe_aborted(p.parse_stmt(attrs),p)
155 }
156
157 pub fn parse_tts_from_source_str(
158     name: @str,
159     source: @str,
160     cfg: ast::CrateConfig,
161     sess: @mut ParseSess
162 ) -> ~[ast::token_tree] {
163     let p = new_parser_from_source_str(
164         sess,
165         cfg,
166         name,
167         source
168     );
169     *p.quote_depth += 1u;
170     // right now this is re-creating the token trees from ... token trees.
171     maybe_aborted(p.parse_all_token_trees(),p)
172 }
173
174 // given a function and parsing information (source str,
175 // filename, crate cfg, and sess), create a parser,
176 // apply the function, and check that the parser
177 // consumed all of the input before returning the function's
178 // result.
179 pub fn parse_from_source_str<T>(
180     f: &fn(&Parser) -> T,
181     name: @str, ss: codemap::FileSubstr,
182     source: @str,
183     cfg: ast::CrateConfig,
184     sess: @mut ParseSess
185 ) -> T {
186     let p = new_parser_from_source_substr(
187         sess,
188         cfg,
189         name,
190         ss,
191         source
192     );
193     let r = f(&p);
194     if !p.reader.is_eof() {
195         p.reader.fatal(~"expected end-of-string");
196     }
197     maybe_aborted(r,p)
198 }
199
200 // Create a new parser from a source string
201 pub fn new_parser_from_source_str(sess: @mut ParseSess,
202                                   cfg: ast::CrateConfig,
203                                   name: @str,
204                                   source: @str)
205                                -> Parser {
206     filemap_to_parser(sess,string_to_filemap(sess,source,name),cfg)
207 }
208
209 // Create a new parser from a source string where the origin
210 // is specified as a substring of another file.
211 pub fn new_parser_from_source_substr(sess: @mut ParseSess,
212                                   cfg: ast::CrateConfig,
213                                   name: @str,
214                                   ss: codemap::FileSubstr,
215                                   source: @str)
216                                -> Parser {
217     filemap_to_parser(sess,substring_to_filemap(sess,source,name,ss),cfg)
218 }
219
220 /// Create a new parser, handling errors as appropriate
221 /// if the file doesn't exist
222 pub fn new_parser_from_file(
223     sess: @mut ParseSess,
224     cfg: ast::CrateConfig,
225     path: &Path
226 ) -> Parser {
227     filemap_to_parser(sess,file_to_filemap(sess,path,None),cfg)
228 }
229
230 /// Given a session, a crate config, a path, and a span, add
231 /// the file at the given path to the codemap, and return a parser.
232 /// On an error, use the given span as the source of the problem.
233 pub fn new_sub_parser_from_file(
234     sess: @mut ParseSess,
235     cfg: ast::CrateConfig,
236     path: &Path,
237     sp: Span
238 ) -> Parser {
239     filemap_to_parser(sess,file_to_filemap(sess,path,Some(sp)),cfg)
240 }
241
242 /// Given a filemap and config, return a parser
243 pub fn filemap_to_parser(sess: @mut ParseSess,
244                          filemap: @FileMap,
245                          cfg: ast::CrateConfig) -> Parser {
246     tts_to_parser(sess,filemap_to_tts(sess,filemap),cfg)
247 }
248
249 // must preserve old name for now, because quote! from the *existing*
250 // compiler expands into it
251 pub fn new_parser_from_tts(sess: @mut ParseSess,
252                      cfg: ast::CrateConfig,
253                      tts: ~[ast::token_tree]) -> Parser {
254     tts_to_parser(sess,tts,cfg)
255 }
256
257
258 // base abstractions
259
260 /// Given a session and a path and an optional span (for error reporting),
261 /// add the path to the session's codemap and return the new filemap.
262 pub fn file_to_filemap(sess: @mut ParseSess, path: &Path, spanopt: Option<Span>)
263     -> @FileMap {
264     let err = |msg: &str| {
265         match spanopt {
266             Some(sp) => sess.span_diagnostic.span_fatal(sp, msg),
267             None => sess.span_diagnostic.handler().fatal(msg),
268         }
269     };
270     let bytes = match io::result(|| File::open(path).read_to_end()) {
271         Ok(bytes) => bytes,
272         Err(e) => {
273             err(format!("couldn't read {}: {}", path.display(), e.desc));
274             unreachable!()
275         }
276     };
277     match str::from_utf8_owned_opt(bytes) {
278         Some(s) => {
279             return string_to_filemap(sess, s.to_managed(),
280                                      path.as_str().unwrap().to_managed());
281         }
282         None => {
283             err(format!("{} is not UTF-8 encoded", path.display()))
284         }
285     }
286     unreachable!()
287 }
288
289 // given a session and a string, add the string to
290 // the session's codemap and return the new filemap
291 pub fn string_to_filemap(sess: @mut ParseSess, source: @str, path: @str)
292     -> @FileMap {
293     sess.cm.new_filemap(path, source)
294 }
295
296 // given a session and a string and a path and a FileSubStr, add
297 // the string to the CodeMap and return the new FileMap
298 pub fn substring_to_filemap(sess: @mut ParseSess, source: @str, path: @str,
299                            filesubstr: FileSubstr) -> @FileMap {
300     sess.cm.new_filemap_w_substr(path,filesubstr,source)
301 }
302
303 // given a filemap, produce a sequence of token-trees
304 pub fn filemap_to_tts(sess: @mut ParseSess, filemap: @FileMap)
305     -> ~[ast::token_tree] {
306     // it appears to me that the cfg doesn't matter here... indeed,
307     // parsing tt's probably shouldn't require a parser at all.
308     let cfg = ~[];
309     let srdr = lexer::new_string_reader(sess.span_diagnostic, filemap);
310     let p1 = Parser(sess, cfg, srdr as @mut reader);
311     p1.parse_all_token_trees()
312 }
313
314 // given tts and cfg, produce a parser
315 pub fn tts_to_parser(sess: @mut ParseSess,
316                      tts: ~[ast::token_tree],
317                      cfg: ast::CrateConfig) -> Parser {
318     let trdr = lexer::new_tt_reader(sess.span_diagnostic, None, tts);
319     Parser(sess, cfg, trdr as @mut reader)
320 }
321
322 // abort if necessary
323 pub fn maybe_aborted<T>(result : T, p: Parser) -> T {
324     p.abort_if_errors();
325     result
326 }
327
328
329
330 #[cfg(test)]
331 mod test {
332     use super::*;
333     use extra::serialize::Encodable;
334     use extra;
335     use std::rt::io;
336     use std::rt::io::Decorator;
337     use std::rt::io::mem::MemWriter;
338     use std::str;
339     use codemap::{Span, BytePos, Spanned};
340     use opt_vec;
341     use ast;
342     use abi;
343     use parse::parser::Parser;
344     use parse::token::{str_to_ident};
345     use util::parser_testing::{string_to_tts, string_to_parser};
346     use util::parser_testing::{string_to_expr, string_to_item};
347     use util::parser_testing::string_to_stmt;
348
349     #[cfg(test)] fn to_json_str<E : Encodable<extra::json::Encoder>>(val: @E) -> ~str {
350         let writer = @mut MemWriter::new();
351         let mut encoder = extra::json::Encoder(writer as @mut io::Writer);
352         val.encode(&mut encoder);
353         str::from_utf8(*writer.inner_ref())
354     }
355
356     // produce a codemap::span
357     fn sp (a: uint, b: uint) -> Span {
358         Span{lo:BytePos(a),hi:BytePos(b),expn_info:None}
359     }
360
361     #[test] fn path_exprs_1() {
362         assert_eq!(string_to_expr(@"a"),
363                    @ast::Expr{
364                     id: ast::DUMMY_NODE_ID,
365                     node: ast::ExprPath(ast::Path {
366                         span: sp(0, 1),
367                         global: false,
368                         segments: ~[
369                             ast::PathSegment {
370                                 identifier: str_to_ident("a"),
371                                 lifetime: None,
372                                 types: opt_vec::Empty,
373                             }
374                         ],
375                     }),
376                     span: sp(0, 1)
377                    })
378     }
379
380     #[test] fn path_exprs_2 () {
381         assert_eq!(string_to_expr(@"::a::b"),
382                    @ast::Expr {
383                     id: ast::DUMMY_NODE_ID,
384                     node: ast::ExprPath(ast::Path {
385                             span: sp(0, 6),
386                             global: true,
387                             segments: ~[
388                                 ast::PathSegment {
389                                     identifier: str_to_ident("a"),
390                                     lifetime: None,
391                                     types: opt_vec::Empty,
392                                 },
393                                 ast::PathSegment {
394                                     identifier: str_to_ident("b"),
395                                     lifetime: None,
396                                     types: opt_vec::Empty,
397                                 }
398                             ]
399                         }),
400                     span: sp(0, 6)
401                    })
402     }
403
404     #[should_fail]
405     #[test] fn bad_path_expr_1() {
406         string_to_expr(@"::abc::def::return");
407     }
408
409     // check the token-tree-ization of macros
410     #[test] fn string_to_tts_macro () {
411         let tts = string_to_tts(@"macro_rules! zip (($a)=>($a))");
412         match tts {
413             [ast::tt_tok(_,_),
414              ast::tt_tok(_,token::NOT),
415              ast::tt_tok(_,_),
416              ast::tt_delim(delim_elts)] =>
417                 match *delim_elts {
418                 [ast::tt_tok(_,token::LPAREN),
419                  ast::tt_delim(first_set),
420                  ast::tt_tok(_,token::FAT_ARROW),
421                  ast::tt_delim(second_set),
422                  ast::tt_tok(_,token::RPAREN)] =>
423                     match *first_set {
424                     [ast::tt_tok(_,token::LPAREN),
425                      ast::tt_tok(_,token::DOLLAR),
426                      ast::tt_tok(_,_),
427                      ast::tt_tok(_,token::RPAREN)] =>
428                         match *second_set {
429                         [ast::tt_tok(_,token::LPAREN),
430                          ast::tt_tok(_,token::DOLLAR),
431                          ast::tt_tok(_,_),
432                          ast::tt_tok(_,token::RPAREN)] =>
433                             assert_eq!("correct","correct"),
434                         _ => assert_eq!("wrong 4","correct")
435                     },
436                     _ => {
437                         error!("failing value 3: {:?}",first_set);
438                         assert_eq!("wrong 3","correct")
439                     }
440                 },
441                 _ => {
442                     error!("failing value 2: {:?}",delim_elts);
443                     assert_eq!("wrong","correct");
444                 }
445
446             },
447             _ => {
448                 error!("failing value: {:?}",tts);
449                 assert_eq!("wrong 1","correct");
450             }
451         }
452     }
453
454     #[test] fn string_to_tts_1 () {
455         let tts = string_to_tts(@"fn a (b : int) { b; }");
456         assert_eq!(to_json_str(@tts),
457         ~"[\
458     {\
459         \"variant\":\"tt_tok\",\
460         \"fields\":[\
461             null,\
462             {\
463                 \"variant\":\"IDENT\",\
464                 \"fields\":[\
465                     \"fn\",\
466                     false\
467                 ]\
468             }\
469         ]\
470     },\
471     {\
472         \"variant\":\"tt_tok\",\
473         \"fields\":[\
474             null,\
475             {\
476                 \"variant\":\"IDENT\",\
477                 \"fields\":[\
478                     \"a\",\
479                     false\
480                 ]\
481             }\
482         ]\
483     },\
484     {\
485         \"variant\":\"tt_delim\",\
486         \"fields\":[\
487             [\
488                 {\
489                     \"variant\":\"tt_tok\",\
490                     \"fields\":[\
491                         null,\
492                         \"LPAREN\"\
493                     ]\
494                 },\
495                 {\
496                     \"variant\":\"tt_tok\",\
497                     \"fields\":[\
498                         null,\
499                         {\
500                             \"variant\":\"IDENT\",\
501                             \"fields\":[\
502                                 \"b\",\
503                                 false\
504                             ]\
505                         }\
506                     ]\
507                 },\
508                 {\
509                     \"variant\":\"tt_tok\",\
510                     \"fields\":[\
511                         null,\
512                         \"COLON\"\
513                     ]\
514                 },\
515                 {\
516                     \"variant\":\"tt_tok\",\
517                     \"fields\":[\
518                         null,\
519                         {\
520                             \"variant\":\"IDENT\",\
521                             \"fields\":[\
522                                 \"int\",\
523                                 false\
524                             ]\
525                         }\
526                     ]\
527                 },\
528                 {\
529                     \"variant\":\"tt_tok\",\
530                     \"fields\":[\
531                         null,\
532                         \"RPAREN\"\
533                     ]\
534                 }\
535             ]\
536         ]\
537     },\
538     {\
539         \"variant\":\"tt_delim\",\
540         \"fields\":[\
541             [\
542                 {\
543                     \"variant\":\"tt_tok\",\
544                     \"fields\":[\
545                         null,\
546                         \"LBRACE\"\
547                     ]\
548                 },\
549                 {\
550                     \"variant\":\"tt_tok\",\
551                     \"fields\":[\
552                         null,\
553                         {\
554                             \"variant\":\"IDENT\",\
555                             \"fields\":[\
556                                 \"b\",\
557                                 false\
558                             ]\
559                         }\
560                     ]\
561                 },\
562                 {\
563                     \"variant\":\"tt_tok\",\
564                     \"fields\":[\
565                         null,\
566                         \"SEMI\"\
567                     ]\
568                 },\
569                 {\
570                     \"variant\":\"tt_tok\",\
571                     \"fields\":[\
572                         null,\
573                         \"RBRACE\"\
574                     ]\
575                 }\
576             ]\
577         ]\
578     }\
579 ]"
580         );
581     }
582
583     #[test] fn ret_expr() {
584         assert_eq!(string_to_expr(@"return d"),
585                    @ast::Expr{
586                     id: ast::DUMMY_NODE_ID,
587                     node:ast::ExprRet(Some(@ast::Expr{
588                         id: ast::DUMMY_NODE_ID,
589                         node:ast::ExprPath(ast::Path{
590                             span: sp(7, 8),
591                             global: false,
592                             segments: ~[
593                                 ast::PathSegment {
594                                     identifier: str_to_ident("d"),
595                                     lifetime: None,
596                                     types: opt_vec::Empty,
597                                 }
598                             ],
599                         }),
600                         span:sp(7,8)
601                     })),
602                     span:sp(0,8)
603                    })
604     }
605
606     #[test] fn parse_stmt_1 () {
607         assert_eq!(string_to_stmt(@"b;"),
608                    @Spanned{
609                        node: ast::StmtExpr(@ast::Expr {
610                            id: ast::DUMMY_NODE_ID,
611                            node: ast::ExprPath(ast::Path {
612                                span:sp(0,1),
613                                global:false,
614                                segments: ~[
615                                 ast::PathSegment {
616                                     identifier: str_to_ident("b"),
617                                     lifetime: None,
618                                     types: opt_vec::Empty,
619                                 }
620                                ],
621                             }),
622                            span: sp(0,1)},
623                                            ast::DUMMY_NODE_ID),
624                        span: sp(0,1)})
625
626     }
627
628     fn parser_done(p: Parser){
629         assert_eq!((*p.token).clone(), token::EOF);
630     }
631
632     #[test] fn parse_ident_pat () {
633         let parser = string_to_parser(@"b");
634         assert_eq!(parser.parse_pat(),
635                    @ast::Pat{id: ast::DUMMY_NODE_ID,
636                              node: ast::PatIdent(
637                                 ast::BindByValue(ast::MutImmutable),
638                                 ast::Path {
639                                     span:sp(0,1),
640                                     global:false,
641                                     segments: ~[
642                                         ast::PathSegment {
643                                             identifier: str_to_ident("b"),
644                                             lifetime: None,
645                                             types: opt_vec::Empty,
646                                         }
647                                     ],
648                                 },
649                                 None /* no idea */),
650                              span: sp(0,1)});
651         parser_done(parser);
652     }
653
654     // check the contents of the tt manually:
655     #[test] fn parse_fundecl () {
656         // this test depends on the intern order of "fn" and "int"
657         assert_eq!(string_to_item(@"fn a (b : int) { b; }"),
658                   Some(
659                       @ast::item{ident:str_to_ident("a"),
660                             attrs:~[],
661                             id: ast::DUMMY_NODE_ID,
662                             node: ast::item_fn(ast::fn_decl{
663                                 inputs: ~[ast::arg{
664                                     ty: ast::Ty{id: ast::DUMMY_NODE_ID,
665                                                 node: ast::ty_path(ast::Path{
666                                         span:sp(10,13),
667                                         global:false,
668                                         segments: ~[
669                                             ast::PathSegment {
670                                                 identifier:
671                                                     str_to_ident("int"),
672                                                 lifetime: None,
673                                                 types: opt_vec::Empty,
674                                             }
675                                         ],
676                                         }, None, ast::DUMMY_NODE_ID),
677                                         span:sp(10,13)
678                                     },
679                                     pat: @ast::Pat {
680                                         id: ast::DUMMY_NODE_ID,
681                                         node: ast::PatIdent(
682                                             ast::BindByValue(ast::MutImmutable),
683                                             ast::Path {
684                                                 span:sp(6,7),
685                                                 global:false,
686                                                 segments: ~[
687                                                     ast::PathSegment {
688                                                         identifier:
689                                                             str_to_ident("b"),
690                                                         lifetime: None,
691                                                         types: opt_vec::Empty,
692                                                     }
693                                                 ],
694                                             },
695                                             None // no idea
696                                         ),
697                                         span: sp(6,7)
698                                     },
699                                     id: ast::DUMMY_NODE_ID
700                                 }],
701                                 output: ast::Ty{id: ast::DUMMY_NODE_ID,
702                                                  node: ast::ty_nil,
703                                                  span:sp(15,15)}, // not sure
704                                 cf: ast::return_val,
705                                 variadic: false
706                             },
707                                     ast::impure_fn,
708                                     abi::AbiSet::Rust(),
709                                     ast::Generics{ // no idea on either of these:
710                                         lifetimes: opt_vec::Empty,
711                                         ty_params: opt_vec::Empty,
712                                     },
713                                     ast::Block {
714                                         view_items: ~[],
715                                         stmts: ~[@Spanned{
716                                             node: ast::StmtSemi(@ast::Expr{
717                                                 id: ast::DUMMY_NODE_ID,
718                                                 node: ast::ExprPath(
719                                                       ast::Path{
720                                                         span:sp(17,18),
721                                                         global:false,
722                                                         segments: ~[
723                                                             ast::PathSegment {
724                                                                 identifier:
725                                                                 str_to_ident(
726                                                                     "b"),
727                                                                 lifetime:
728                                                                     None,
729                                                                 types:
730                                                                 opt_vec::Empty
731                                                             }
732                                                         ],
733                                                       }),
734                                                 span: sp(17,18)},
735                                                 ast::DUMMY_NODE_ID),
736                                             span: sp(17,18)}],
737                                         expr: None,
738                                         id: ast::DUMMY_NODE_ID,
739                                         rules: ast::DefaultBlock, // no idea
740                                         span: sp(15,21),
741                                     }),
742                             vis: ast::inherited,
743                             span: sp(0,21)}));
744     }
745
746
747     #[test] fn parse_exprs () {
748         // just make sure that they parse....
749         string_to_expr(@"3 + 4");
750         string_to_expr(@"a::z.froob(b,@(987+3))");
751     }
752
753     #[test] fn attrs_fix_bug () {
754         string_to_item(@"pub fn mk_file_writer(path: &Path, flags: &[FileFlag])
755                    -> Result<@Writer, ~str> {
756     #[cfg(windows)]
757     fn wb() -> c_int {
758       (O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int
759     }
760
761     #[cfg(unix)]
762     fn wb() -> c_int { O_WRONLY as c_int }
763
764     let mut fflags: c_int = wb();
765 }");
766     }
767
768 }