4 # This script is for extracting the grammar from the rust docs.
8 collections = { "gram": [],
18 for line in fileinput.input(openhook=fileinput.hook_encoded("utf-8")):
20 if line.startswith("~~~~"):
23 if coll in ["keyword", "reserved", "binop", "unop"]:
24 for word in line.split():
25 if word not in collections[coll]:
26 collections[coll].append(word)
28 collections[coll].append(line)
31 if line.startswith("~~~~"):
32 for cname in collections:
33 if ("." + cname) in line:
38 # Define operator symbol-names here
40 tokens = ["non_star", "non_slash", "non_eol",
41 "non_single_quote", "non_double_quote", "ident" ]
95 "/*": "openblockcomment",
96 "*/": "closeblockcomment",
97 "macro_rules": "macro_rules",
105 for line in collections["gram"]:
107 for word in line.split():
108 # replace strings with keyword-names or symbol-names from table
109 if word.startswith("\""):
112 word = symnames[word]
116 raise Exception("non-alpha apparent keyword: "
118 if word not in tokens:
119 if (word in collections["keyword"] or
120 word in collections["reserved"]):
123 raise Exception("unknown keyword/reserved word: "
130 for word in collections["keyword"] + collections["reserved"]:
131 if word not in tokens:
134 for sym in collections["unop"] + collections["binop"] + symnames.keys():
136 if word not in tokens:
140 print("%start parser, token;")
141 print("%%token %s ;" % ("\n\t, ".join(tokens)))
142 for coll in ["keyword", "reserved"]:
143 print("%s: %s ; " % (coll, "\n\t| ".join(collections[coll])));
144 for coll in ["binop", "unop"]:
145 print("%s: %s ; " % (coll, "\n\t| ".join([symnames[x]
146 for x in collections[coll]])));
147 print("\n".join(lines));