1 // Copyright 2012-2013 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 pub use self::BinOpToken::*;
12 pub use self::Nonterminal::*;
13 pub use self::DelimToken::*;
15 pub use self::Token::*;
20 use tokenstream::TokenTree;
25 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
40 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
42 /// A round parenthesis: `(` or `)`
44 /// A square bracket: `[` or `]`
46 /// A curly brace: `{` or `}`
48 /// An empty delimiter
53 pub fn len(self) -> usize {
54 if self == NoDelim { 0 } else { 1 }
58 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
65 StrRaw(ast::Name, usize), /* raw str delimited by n hash symbols */
67 ByteStrRaw(ast::Name, usize), /* raw byte str delimited by n hash symbols */
71 pub fn short_name(&self) -> &'static str {
75 Integer(_) => "integer",
77 Str_(_) | StrRaw(..) => "string",
78 ByteStr(_) | ByteStrRaw(..) => "byte string"
83 fn ident_can_begin_expr(ident: ast::Ident) -> bool {
84 let ident_token: Token = Ident(ident);
86 !ident_token.is_any_keyword() ||
87 ident_token.is_path_segment_keyword() ||
91 keywords::Break.name(),
92 keywords::Continue.name(),
93 keywords::False.name(),
96 keywords::Loop.name(),
97 keywords::Match.name(),
98 keywords::Move.name(),
99 keywords::Return.name(),
100 keywords::True.name(),
101 keywords::Unsafe.name(),
102 keywords::While.name(),
103 ].contains(&ident.name)
106 fn ident_can_begin_type(ident: ast::Ident) -> bool {
107 let ident_token: Token = Ident(ident);
109 !ident_token.is_any_keyword() ||
110 ident_token.is_path_segment_keyword() ||
112 keywords::For.name(),
113 keywords::Impl.name(),
115 keywords::Unsafe.name(),
116 keywords::Extern.name(),
117 keywords::Typeof.name(),
118 ].contains(&ident.name)
121 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug)]
123 /* Expression-operator symbols. */
138 /* Structural symbols */
153 /// An opening delimiter, eg. `{`
154 OpenDelim(DelimToken),
155 /// A closing delimiter, eg. `}`
156 CloseDelim(DelimToken),
159 Literal(Lit, Option<ast::Name>),
161 /* Name components */
164 Lifetime(ast::Ident),
166 /* For interpolation */
167 Interpolated(Rc<Nonterminal>),
168 // Can be expanded into several tokens.
170 DocComment(ast::Name),
171 // In right-hand-sides of MBE macros:
172 /// A syntactic variable that will be filled in by macro expansion.
175 // Junk. These carry no data because we don't really care about the data
176 // they *would* carry, and don't really want to allocate a new ident for
177 // them. Instead, users could extract that from the associated span.
189 /// Returns `true` if the token starts with '>'.
190 pub fn is_like_gt(&self) -> bool {
192 BinOp(Shr) | BinOpEq(Shr) | Gt | Ge => true,
197 /// Returns `true` if the token can appear at the start of an expression.
198 pub fn can_begin_expr(&self) -> bool {
200 Ident(ident) => ident_can_begin_expr(ident), // value name or keyword
201 OpenDelim(..) => true, // tuple, array or block
202 Literal(..) => true, // literal
203 Not => true, // operator not
204 BinOp(Minus) => true, // unary minus
205 BinOp(Star) => true, // dereference
206 BinOp(Or) | OrOr => true, // closure
207 BinOp(And) => true, // reference
208 AndAnd => true, // double reference
209 DotDot | DotDotDot => true, // range notation
210 Lt | BinOp(Shl) => true, // associated path
211 ModSep => true, // global path
212 Pound => true, // expression attributes
213 Interpolated(ref nt) => match **nt {
214 NtIdent(..) | NtExpr(..) | NtBlock(..) | NtPath(..) => true,
221 /// Returns `true` if the token can appear at the start of a type.
222 pub fn can_begin_type(&self) -> bool {
224 Ident(ident) => ident_can_begin_type(ident), // type name or keyword
225 OpenDelim(Paren) => true, // tuple
226 OpenDelim(Bracket) => true, // array
227 Underscore => true, // placeholder
228 Not => true, // never
229 BinOp(Star) => true, // raw pointer
230 BinOp(And) => true, // reference
231 AndAnd => true, // double reference
232 Question => true, // maybe bound in trait object
233 Lifetime(..) => true, // lifetime bound in trait object
234 Lt | BinOp(Shl) => true, // associated path
235 ModSep => true, // global path
236 Interpolated(ref nt) => match **nt {
237 NtIdent(..) | NtTy(..) | NtPath(..) => true,
244 /// Returns `true` if the token is any literal
245 pub fn is_lit(&self) -> bool {
252 pub fn ident(&self) -> Option<ast::Ident> {
254 Ident(ident) => Some(ident),
255 Interpolated(ref nt) => match **nt {
256 NtIdent(ident) => Some(ident.node),
263 /// Returns `true` if the token is an identifier.
264 pub fn is_ident(&self) -> bool {
265 self.ident().is_some()
268 /// Returns `true` if the token is a documentation comment.
269 pub fn is_doc_comment(&self) -> bool {
271 DocComment(..) => true,
276 /// Returns `true` if the token is interpolated.
277 pub fn is_interpolated(&self) -> bool {
279 Interpolated(..) => true,
284 /// Returns `true` if the token is an interpolated path.
285 pub fn is_path(&self) -> bool {
286 if let Interpolated(ref nt) = *self {
287 if let NtPath(..) = **nt {
294 /// Returns `true` if the token is a lifetime.
295 pub fn is_lifetime(&self) -> bool {
297 Lifetime(..) => true,
302 /// Returns `true` if the token is either the `mut` or `const` keyword.
303 pub fn is_mutability(&self) -> bool {
304 self.is_keyword(keywords::Mut) ||
305 self.is_keyword(keywords::Const)
308 pub fn is_qpath_start(&self) -> bool {
309 self == &Lt || self == &BinOp(Shl)
312 pub fn is_path_start(&self) -> bool {
313 self == &ModSep || self.is_qpath_start() || self.is_path() ||
314 self.is_path_segment_keyword() || self.is_ident() && !self.is_any_keyword()
317 /// Returns `true` if the token is a given keyword, `kw`.
318 pub fn is_keyword(&self, kw: keywords::Keyword) -> bool {
319 self.ident().map(|ident| ident.name == kw.name()).unwrap_or(false)
322 pub fn is_path_segment_keyword(&self) -> bool {
324 Some(id) => id.name == keywords::Super.name() ||
325 id.name == keywords::SelfValue.name() ||
326 id.name == keywords::SelfType.name(),
331 /// Returns `true` if the token is either a strict or reserved keyword.
332 pub fn is_any_keyword(&self) -> bool {
333 self.is_strict_keyword() || self.is_reserved_keyword()
336 /// Returns `true` if the token is a strict keyword.
337 pub fn is_strict_keyword(&self) -> bool {
339 Some(id) => id.name >= keywords::As.name() && id.name <= keywords::While.name(),
344 /// Returns `true` if the token is a keyword reserved for possible future use.
345 pub fn is_reserved_keyword(&self) -> bool {
347 Some(id) => id.name >= keywords::Abstract.name() && id.name <= keywords::Yield.name(),
353 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash)]
354 /// For interpolation during macro expansion.
355 pub enum Nonterminal {
356 NtItem(P<ast::Item>),
357 NtBlock(P<ast::Block>),
360 NtExpr(P<ast::Expr>),
362 NtIdent(ast::SpannedIdent),
363 /// Stuff inside brackets for attributes
364 NtMeta(ast::MetaItem),
366 NtVis(ast::Visibility),
368 // These are not exposed to macros, but are used by quasiquote.
370 NtImplItem(ast::ImplItem),
371 NtTraitItem(ast::TraitItem),
372 NtGenerics(ast::Generics),
373 NtWhereClause(ast::WhereClause),
377 impl fmt::Debug for Nonterminal {
378 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
380 NtItem(..) => f.pad("NtItem(..)"),
381 NtBlock(..) => f.pad("NtBlock(..)"),
382 NtStmt(..) => f.pad("NtStmt(..)"),
383 NtPat(..) => f.pad("NtPat(..)"),
384 NtExpr(..) => f.pad("NtExpr(..)"),
385 NtTy(..) => f.pad("NtTy(..)"),
386 NtIdent(..) => f.pad("NtIdent(..)"),
387 NtMeta(..) => f.pad("NtMeta(..)"),
388 NtPath(..) => f.pad("NtPath(..)"),
389 NtTT(..) => f.pad("NtTT(..)"),
390 NtArm(..) => f.pad("NtArm(..)"),
391 NtImplItem(..) => f.pad("NtImplItem(..)"),
392 NtTraitItem(..) => f.pad("NtTraitItem(..)"),
393 NtGenerics(..) => f.pad("NtGenerics(..)"),
394 NtWhereClause(..) => f.pad("NtWhereClause(..)"),
395 NtArg(..) => f.pad("NtArg(..)"),
396 NtVis(..) => f.pad("NtVis(..)"),