1 // Copyright 2012 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! The main parser interface
16 use codemap::{span, CodeMap, FileMap, FileSubstr};
18 use diagnostic::{span_handler, mk_span_handler, mk_handler, Emitter};
19 use parse::attr::parser_attr;
20 use parse::lexer::reader;
21 use parse::parser::Parser;
22 use parse::token::{ident_interner, mk_ident_interner};
25 use core::option::{None, Option, Some};
27 use core::result::{Err, Ok};
36 /// Common routines shared by parser mods
39 /// Routines the parser uses to classify AST nodes
42 /// Reporting obsolete syntax
45 // info about a parsing session.
46 // This structure and the reader both have
47 // an interner associated with them. If they're
48 // not the same, bad things can happen.
49 pub struct ParseSess {
50 cm: @codemap::CodeMap, // better be the same as the one in the reader!
52 span_diagnostic: @span_handler, // better be the same as the one in the reader!
53 interner: @ident_interner,
56 pub fn new_parse_sess(demitter: Option<Emitter>) -> @mut ParseSess {
57 let cm = @CodeMap::new();
61 span_diagnostic: mk_span_handler(mk_handler(demitter), cm),
62 interner: mk_ident_interner(),
66 pub fn new_parse_sess_special_handler(sh: @span_handler,
67 cm: @codemap::CodeMap)
73 interner: mk_ident_interner(),
77 // a bunch of utility functions of the form parse_<thing>_from_<source>
78 // where <thing> includes crate, expr, item, stmt, tts, and one that
79 // uses a HOF to parse anything, and <source> includes file and
82 pub fn parse_crate_from_file(
87 new_parser_from_file(sess, /*bad*/ copy cfg, input).parse_crate_mod()
88 // why is there no p.abort_if_errors here?
91 pub fn parse_crate_from_source_str(
97 let p = new_parser_from_source_str(
103 maybe_aborted(p.parse_crate_mod(),p)
106 pub fn parse_expr_from_source_str(
112 let p = new_parser_from_source_str(
118 maybe_aborted(p.parse_expr(), p)
121 pub fn parse_item_from_source_str(
125 attrs: ~[ast::attribute],
127 ) -> Option<@ast::item> {
128 let p = new_parser_from_source_str(
134 maybe_aborted(p.parse_item(attrs),p)
137 pub fn parse_meta_from_source_str(
142 ) -> @ast::meta_item {
143 let p = new_parser_from_source_str(
149 maybe_aborted(p.parse_meta_item(),p)
152 pub fn parse_stmt_from_source_str(
156 attrs: ~[ast::attribute],
159 let p = new_parser_from_source_str(
165 maybe_aborted(p.parse_stmt(attrs),p)
168 pub fn parse_tts_from_source_str(
173 ) -> ~[ast::token_tree] {
174 let p = new_parser_from_source_str(
180 *p.quote_depth += 1u;
181 // right now this is re-creating the token trees from ... token trees.
182 maybe_aborted(p.parse_all_token_trees(),p)
185 // given a function and parsing information (source str,
186 // filename, crate cfg, and sess), create a parser,
187 // apply the function, and check that the parser
188 // consumed all of the input before returning the function's
190 pub fn parse_from_source_str<T>(
191 f: &fn(&Parser) -> T,
192 name: ~str, ss: codemap::FileSubstr,
197 let p = new_parser_from_source_substr(
205 if !p.reader.is_eof() {
206 p.reader.fatal(~"expected end-of-string");
211 // return the next unused node id.
212 pub fn next_node_id(sess: @mut ParseSess) -> node_id {
213 let rv = sess.next_id;
215 // ID 0 is reserved for the crate and doesn't actually exist in the AST
220 // Create a new parser from a source string
221 pub fn new_parser_from_source_str(sess: @mut ParseSess,
226 filemap_to_parser(sess,string_to_filemap(sess,source,name),cfg)
229 // Create a new parser from a source string where the origin
230 // is specified as a substring of another file.
231 pub fn new_parser_from_source_substr(sess: @mut ParseSess,
234 ss: codemap::FileSubstr,
237 filemap_to_parser(sess,substring_to_filemap(sess,source,name,ss),cfg)
240 /// Create a new parser, handling errors as appropriate
241 /// if the file doesn't exist
242 pub fn new_parser_from_file(
243 sess: @mut ParseSess,
247 filemap_to_parser(sess,file_to_filemap(sess,path,None),cfg)
250 /// Given a session, a crate config, a path, and a span, add
251 /// the file at the given path to the codemap, and return a parser.
252 /// On an error, use the given span as the source of the problem.
253 pub fn new_sub_parser_from_file(
254 sess: @mut ParseSess,
259 filemap_to_parser(sess,file_to_filemap(sess,path,Some(sp)),cfg)
262 /// Given a filemap and config, return a parser
263 pub fn filemap_to_parser(sess: @mut ParseSess,
265 cfg: ast::crate_cfg) -> Parser {
266 tts_to_parser(sess,filemap_to_tts(sess,filemap),cfg)
269 // must preserve old name for now, because quote! from the *existing*
270 // compiler expands into it
271 pub fn new_parser_from_tts(sess: @mut ParseSess,
273 tts: ~[ast::token_tree]) -> Parser {
274 tts_to_parser(sess,tts,cfg)
280 /// Given a session and a path and an optional span (for error reporting),
281 /// add the path to the session's codemap and return the new filemap.
282 pub fn file_to_filemap(sess: @mut ParseSess, path: &Path, spanopt: Option<span>)
284 match io::read_whole_file_str(path) {
285 Ok(src) => string_to_filemap(sess, @src, path.to_str()),
288 Some(span) => sess.span_diagnostic.span_fatal(span, e),
289 None => sess.span_diagnostic.handler().fatal(e)
295 // given a session and a string, add the string to
296 // the session's codemap and return the new filemap
297 pub fn string_to_filemap(sess: @mut ParseSess, source: @~str, path: ~str)
299 sess.cm.new_filemap(path, source)
302 // given a session and a string and a path and a FileSubStr, add
303 // the string to the CodeMap and return the new FileMap
304 pub fn substring_to_filemap(sess: @mut ParseSess, source: @~str, path: ~str,
305 filesubstr: FileSubstr) -> @FileMap {
306 sess.cm.new_filemap_w_substr(path,filesubstr,source)
309 // given a filemap, produce a sequence of token-trees
310 pub fn filemap_to_tts(sess: @mut ParseSess, filemap: @FileMap)
311 -> ~[ast::token_tree] {
312 // it appears to me that the cfg doesn't matter here... indeed,
313 // parsing tt's probably shouldn't require a parser at all.
315 let srdr = lexer::new_string_reader(copy sess.span_diagnostic,
318 let p1 = Parser(sess, cfg, srdr as @reader);
319 p1.parse_all_token_trees()
322 // given tts and cfg, produce a parser
323 pub fn tts_to_parser(sess: @mut ParseSess,
324 tts: ~[ast::token_tree],
325 cfg: ast::crate_cfg) -> Parser {
326 let trdr = lexer::new_tt_reader(
327 copy sess.span_diagnostic,
332 Parser(sess, cfg, trdr as @reader)
335 // abort if necessary
336 pub fn maybe_aborted<T>(result : T, p: Parser) -> T {
346 use std::serialize::Encodable;
349 use core::option::Option;
350 use core::option::Some;
351 use core::option::None;
353 use core::num::NumCast;
354 use core::path::Path;
355 use codemap::{dummy_sp, CodeMap, span, BytePos, spanned};
359 use ast_util::mk_ident;
360 use parse::parser::Parser;
361 use parse::token::{ident_interner, mk_ident_interner, mk_fresh_ident_interner};
362 use diagnostic::{span_handler, mk_span_handler, mk_handler, Emitter};
364 // add known names to interner for testing
365 fn mk_testing_interner() -> @ident_interner {
366 let i = mk_fresh_ident_interner();
367 // baby hack; in order to put the identifiers
368 // 'a' and 'b' at known locations, we're going
369 // to fill up the interner to length 100. If
370 // the # of preloaded items on the interner
371 // ever gets larger than 100, we'll have to
372 // adjust this number (say, to 200) and
373 // change the numbers in the identifier
376 assert!(i.len() < 100);
377 for int::range(0,100-((i.len()).to_int())) |_dc| {
378 i.gensym(@~"dontcare");
384 i.intern(@~"return");
385 assert!(i.get(ast::ident{repr:101,ctxt:0}) == @~"b");
389 // make a parse_sess that's closed over a
390 // testing interner (where a -> 100, b -> 101)
391 fn mk_testing_parse_sess() -> @mut ParseSess {
392 let interner = mk_testing_interner();
393 let cm = @CodeMap::new();
397 span_diagnostic: mk_span_handler(mk_handler(None), cm),
402 // map a string to tts, using a made-up filename: return both the token_trees
404 fn string_to_tts_t (source_str : @~str) -> (~[ast::token_tree],@mut ParseSess) {
405 let ps = mk_testing_parse_sess();
406 (filemap_to_tts(ps,string_to_filemap(ps,source_str,~"bogofile")),ps)
409 // map a string to tts, return the tt without its parsesess
410 fn string_to_tts_only(source_str : @~str) -> ~[ast::token_tree] {
411 let (tts,ps) = string_to_tts_t(source_str);
415 // map string to parser (via tts)
416 fn string_to_parser(source_str: @~str) -> Parser {
417 let ps = mk_testing_parse_sess();
418 new_parser_from_source_str(ps,~[],~"bogofile",source_str)
421 #[cfg(test)] fn to_json_str<E : Encodable<std::json::Encoder>>(val: @E) -> ~str {
422 do io::with_str_writer |writer| {
423 val.encode(~std::json::Encoder(writer));
427 fn string_to_crate (source_str : @~str) -> @ast::crate {
428 string_to_parser(source_str).parse_crate_mod()
431 fn string_to_expr (source_str : @~str) -> @ast::expr {
432 string_to_parser(source_str).parse_expr()
435 fn string_to_item (source_str : @~str) -> Option<@ast::item> {
436 string_to_parser(source_str).parse_item(~[])
439 fn string_to_stmt (source_str : @~str) -> @ast::stmt {
440 string_to_parser(source_str).parse_stmt(~[])
443 // produce a codemap::span
444 fn sp (a: uint, b: uint) -> span {
445 span{lo:BytePos(a),hi:BytePos(b),expn_info:None}
448 // convert a vector of uints to a vector of ast::idents
449 fn ints_to_idents(ids: ~[uint]) -> ~[ast::ident] {
450 ids.map(|u| mk_ident(*u))
453 #[test] fn path_exprs_1 () {
454 assert_eq!(string_to_expr(@~"a"),
457 node:ast::expr_path(@ast::Path {span:sp(0,1),
459 idents:~[mk_ident(100)],
465 #[test] fn path_exprs_2 () {
466 assert_eq!(string_to_expr(@~"::a::b"),
469 node:ast::expr_path(@ast::Path {span:sp(0,6),
471 idents:ints_to_idents(~[100,101]),
478 #[test] fn bad_path_expr_1() {
479 string_to_expr(@~"::abc::def::return");
482 #[test] fn string_to_tts_1 () {
483 let (tts,ps) = string_to_tts_t(@~"fn a (b : int) { b; }");
484 assert_eq!(to_json_str(@tts),
486 [\"tt_tok\",null,[\"IDENT\",\"fn\",false]],\
487 [\"tt_tok\",null,[\"IDENT\",\"a\",false]],\
491 [\"tt_tok\",null,\"LPAREN\"],\
492 [\"tt_tok\",null,[\"IDENT\",\"b\",false]],\
493 [\"tt_tok\",null,\"COLON\"],\
494 [\"tt_tok\",null,[\"IDENT\",\"int\",false]],\
495 [\"tt_tok\",null,\"RPAREN\"]\
501 [\"tt_tok\",null,\"LBRACE\"],\
502 [\"tt_tok\",null,[\"IDENT\",\"b\",false]],\
503 [\"tt_tok\",null,\"SEMI\"],\
504 [\"tt_tok\",null,\"RBRACE\"]\
511 #[test] fn ret_expr() {
512 assert_eq!(string_to_expr(@~"return d"),
516 Some(@ast::expr{id:1,callee_id:2,
518 @ast::Path{span:sp(7,8),
520 idents:~[mk_ident(103)],
528 #[test] fn parse_stmt_1 () {
529 assert_eq!(string_to_stmt(@~"b;"),
531 node: ast::stmt_expr(@ast::expr{
534 node: ast::expr_path(
538 idents:~[mk_ident(101)],
547 fn parser_done(p: Parser){
548 assert_eq!(*p.token,token::EOF);
551 #[test] fn parse_ident_pat () {
552 let parser = string_to_parser(@~"b");
553 assert_eq!(parser.parse_pat(false),
554 @ast::pat{id:1, // fixme
555 node: ast::pat_ident(ast::bind_by_copy,
559 idents:~[mk_ident(101)],
568 #[test] fn parse_arg () {
569 let parser = string_to_parser(@~"b : int");
570 assert_eq!(parser.parse_arg_general(true),
573 ty: @ast::Ty{id:3, // fixme
574 node: ast::ty_path(@ast::Path{
575 span:sp(4,4), // this is bizarre...
576 // check this in the original parser?
578 idents:~[mk_ident(105)],
584 node: ast::pat_ident(ast::bind_by_copy,
588 idents:~[mk_ident(101)],
593 span: sp(0,3)}, // really?
598 // check the contents of the tt manually:
599 #[test] fn parse_fundecl () {
600 // this test depends on the intern order of "fn" and "int", and on the
601 // assignment order of the node_ids.
602 assert_eq!(string_to_item(@~"fn a (b : int) { b; }"),
604 @ast::item{ident:mk_ident(100),
607 node: ast::item_fn(ast::fn_decl{
610 ty: @ast::Ty{id:3, // fixme
611 node: ast::ty_path(@ast::Path{
614 idents:~[mk_ident(106)],
619 pat: @ast::pat{id:1, // fixme
620 node: ast::pat_ident(
625 idents:~[mk_ident(101)],
630 span: sp(6,9)}, // bleah.
633 output: @ast::Ty{id:5, // fixme
635 span:sp(15,15)}, // not sure
640 ast::Generics{ // no idea on either of these:
641 lifetimes: opt_vec::Empty,
642 ty_params: opt_vec::Empty,
649 node: ast::stmt_semi(@ast::expr{
652 node: ast::expr_path(
656 idents:~[mk_ident(101)],
664 rules: ast::default_blk // no idea
671 #[test] fn parse_exprs () {
672 // just make sure that they parse....
673 string_to_expr(@~"3 + 4");
674 string_to_expr(@~"a::z.froob(b,@(987+3))");