use syntax::std_inject;
use syntax::symbol::{kw, sym, Symbol};
use syntax::tokenstream::{TokenStream, TokenTree};
-use syntax::parse::token::{self, Token};
+use syntax::parse::token::{self, TokenKind};
use syntax::visit::{self, Visitor};
use syntax_pos::{DUMMY_SP, edition, Span};
}
}
- fn lower_token(&mut self, token: Token, span: Span) -> TokenStream {
+ fn lower_token(&mut self, token: TokenKind, span: Span) -> TokenStream {
match token {
token::Interpolated(nt) => {
let tts = nt.to_tokenstream(&self.sess.parse_sess, span);
use syntax::ext::hygiene::Mark;
use syntax::visit;
use syntax::symbol::{kw, sym};
-use syntax::parse::token::{self, Token};
+use syntax::parse::token::{self, TokenKind};
use syntax_pos::Span;
/// Creates `DefId`s for nodes in the AST.
}
}
- fn visit_token(&mut self, t: Token) {
+ fn visit_token(&mut self, t: TokenKind) {
if let token::Interpolated(nt) = t {
if let token::NtExpr(ref expr) = *nt {
if let ExprKind::Mac(..) = expr.node {
});
fn hash_token<'a, 'gcx, W: StableHasherResult>(
- token: &token::Token,
+ token: &token::TokenKind,
hcx: &mut StableHashingContext<'a>,
hasher: &mut StableHasher<W>,
) {
use syntax::ext::hygiene::Mark;
use syntax::ext::tt::macro_rules;
use syntax::feature_gate::is_builtin_attr;
-use syntax::parse::token::{self, Token};
+use syntax::parse::token::{self, TokenKind};
use syntax::span_err;
use syntax::std_inject::injected_crate_name;
use syntax::symbol::{kw, sym};
self.resolver.current_module = parent;
}
- fn visit_token(&mut self, t: Token) {
+ fn visit_token(&mut self, t: TokenKind) {
if let token::Interpolated(nt) = t {
if let token::NtExpr(ref expr) = *nt {
if let ast::ExprKind::Mac(..) = expr.node {
use std::cell::Cell;
use syntax::parse::lexer::{self, StringReader};
-use syntax::parse::token::{self, Token};
+use syntax::parse::token::{self, TokenKind};
use syntax_pos::*;
#[derive(Clone)]
lexer::StringReader::retokenize(&self.sess.parse_sess, span)
}
- pub fn sub_span_of_token(&self, span: Span, tok: Token) -> Option<Span> {
+ pub fn sub_span_of_token(&self, span: Span, tok: TokenKind) -> Option<Span> {
let mut toks = self.retokenise_span(span);
loop {
let next = toks.real_token();
use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration};
use crate::parse::parser::Parser;
use crate::parse::{self, ParseSess, PResult};
-use crate::parse::token::{self, Token};
+use crate::parse::token::{self, TokenKind};
use crate::ptr::P;
use crate::symbol::{sym, Symbol};
use crate::ThinVec;
idents.push(TokenTree::Token(mod_sep_span, token::ModSep).into());
}
idents.push(TokenTree::Token(segment.ident.span,
- Token::from_ast_ident(segment.ident)).into());
+ TokenKind::from_ast_ident(segment.ident)).into());
last_pos = segment.ident.span.hi();
}
self.node.tokens(self.span).append_to_tree_and_joint_vec(&mut idents);
use crate::ext::tt::quoted::{self, TokenTree};
use crate::parse::{Directory, ParseSess};
use crate::parse::parser::{Parser, PathStyle};
-use crate::parse::token::{self, DocComment, Nonterminal, Token};
+use crate::parse::token::{self, DocComment, Nonterminal, TokenKind};
use crate::print::pprust;
use crate::symbol::{kw, sym, Symbol};
use crate::tokenstream::{DelimSpan, TokenStream};
seq_op: Option<quoted::KleeneOp>,
/// The separator if we are in a repetition.
- sep: Option<Token>,
+ sep: Option<TokenKind>,
/// The "parent" matcher position if we are in a repetition. That is, the matcher position just
/// before we enter the sequence.
Success(T),
/// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected
/// end of macro invocation. Otherwise, it indicates that no rules expected the given token.
- Failure(syntax_pos::Span, Token, &'static str),
+ Failure(syntax_pos::Span, TokenKind, &'static str),
/// Fatal error (malformed macro?). Abort compilation.
Error(syntax_pos::Span, String),
}
/// Generates an appropriate parsing failure message. For EOF, this is "unexpected end...". For
/// other tokens, this is "unexpected token...".
-pub fn parse_failure_msg(tok: Token) -> String {
+pub fn parse_failure_msg(tok: TokenKind) -> String {
match tok {
token::Eof => "unexpected end of macro invocation".to_string(),
_ => format!(
}
/// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison)
-fn token_name_eq(t1: &Token, t2: &Token) -> bool {
+fn token_name_eq(t1: &TokenKind, t2: &TokenKind) -> bool {
if let (Some((id1, is_raw1)), Some((id2, is_raw2))) = (t1.ident(), t2.ident()) {
id1.name == id2.name && is_raw1 == is_raw2
} else if let (Some(id1), Some(id2)) = (t1.lifetime(), t2.lifetime()) {
next_items: &mut Vec<MatcherPosHandle<'root, 'tt>>,
eof_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>,
bb_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>,
- token: &Token,
+ token: &TokenKind,
span: syntax_pos::Span,
) -> ParseResult<()> {
// Pop items from `cur_items` until it is empty.
/// The token is an identifier, but not `_`.
/// We prohibit passing `_` to macros expecting `ident` for now.
-fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> {
+fn get_macro_ident(token: &TokenKind) -> Option<(Ident, bool)> {
match *token {
token::Ident(ident, is_raw) if ident.name != kw::Underscore =>
Some((ident, is_raw)),
///
/// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that
/// token. Be conservative (return true) if not sure.
-fn may_begin_with(name: Symbol, token: &Token) -> bool {
+fn may_begin_with(name: Symbol, token: &TokenKind) -> bool {
/// Checks whether the non-terminal may contain a single (non-keyword) identifier.
fn may_be_ident(nt: &token::Nonterminal) -> bool {
match *nt {
use crate::parse::{Directory, ParseSess};
use crate::parse::parser::Parser;
use crate::parse::token::{self, NtTT};
-use crate::parse::token::Token::*;
+use crate::parse::token::TokenKind::*;
use crate::symbol::{Symbol, kw, sym};
use crate::tokenstream::{DelimSpan, TokenStream, TokenTree};
impl Delimited {
/// Returns the opening delimiter (possibly `NoDelim`).
- pub fn open_token(&self) -> token::Token {
+ pub fn open_token(&self) -> token::TokenKind {
token::OpenDelim(self.delim)
}
/// Returns the closing delimiter (possibly `NoDelim`).
- pub fn close_token(&self) -> token::Token {
+ pub fn close_token(&self) -> token::TokenKind {
token::CloseDelim(self.delim)
}
/// The sequence of token trees
pub tts: Vec<TokenTree>,
/// The optional separator
- pub separator: Option<token::Token>,
+ pub separator: Option<token::TokenKind>,
/// Whether the sequence can be repeated zero (*), or one or more times (+)
pub op: KleeneOp,
/// The number of `Match`s that appear in the sequence (and subsequences)
/// are "first-class" token trees. Useful for parsing macros.
#[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)]
pub enum TokenTree {
- Token(Span, token::Token),
+ Token(Span, token::TokenKind),
Delimited(DelimSpan, Lrc<Delimited>),
/// A kleene-style repetition sequence
Sequence(DelimSpan, Lrc<SequenceRepetition>),
/// Takes a token and returns `Some(KleeneOp)` if the token is `+` `*` or `?`. Otherwise, return
/// `None`.
-fn kleene_op(token: &token::Token) -> Option<KleeneOp> {
+fn kleene_op(token: &token::TokenKind) -> Option<KleeneOp> {
match *token {
token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore),
token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore),
fn parse_kleene_op<I>(
input: &mut I,
span: Span,
-) -> Result<Result<(KleeneOp, Span), (token::Token, Span)>, Span>
+) -> Result<Result<(KleeneOp, Span), (token::TokenKind, Span)>, Span>
where
I: Iterator<Item = tokenstream::TokenTree>,
{
attrs: &[ast::Attribute],
edition: Edition,
macro_node_id: NodeId,
-) -> (Option<token::Token>, KleeneOp)
+) -> (Option<token::TokenKind>, KleeneOp)
where
I: Iterator<Item = tokenstream::TokenTree>,
{
_features: &Features,
_attrs: &[ast::Attribute],
macro_node_id: NodeId,
-) -> (Option<token::Token>, KleeneOp)
+) -> (Option<token::TokenKind>, KleeneOp)
where
I: Iterator<Item = tokenstream::TokenTree>,
{
sess: &ParseSess,
_features: &Features,
_attrs: &[ast::Attribute],
-) -> (Option<token::Token>, KleeneOp)
+) -> (Option<token::TokenKind>, KleeneOp)
where
I: Iterator<Item = tokenstream::TokenTree>,
{
use crate::ext::tt::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch};
use crate::ext::tt::quoted;
use crate::mut_visit::noop_visit_tt;
-use crate::parse::token::{self, NtTT, Token};
+use crate::parse::token::{self, NtTT, TokenKind};
use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
use smallvec::{smallvec, SmallVec};
/// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`).
enum Frame {
Delimited { forest: Lrc<quoted::Delimited>, idx: usize, span: DelimSpan },
- Sequence { forest: Lrc<quoted::SequenceRepetition>, idx: usize, sep: Option<Token> },
+ Sequence { forest: Lrc<quoted::SequenceRepetition>, idx: usize, sep: Option<TokenKind> },
}
impl Frame {
Ident::new(ident.name, ident.span.apply_mark(cx.current_expansion.mark));
sp = sp.apply_mark(cx.current_expansion.mark);
result.push(TokenTree::Token(sp, token::Dollar).into());
- result.push(TokenTree::Token(sp, token::Token::from_ast_ident(ident)).into());
+ result.push(TokenTree::Token(sp, token::TokenKind::from_ast_ident(ident)).into());
}
}
use crate::ast::*;
use crate::source_map::{Spanned, respan};
-use crate::parse::token::{self, Token};
+use crate::parse::token::{self, TokenKind};
use crate::ptr::P;
use crate::ThinVec;
use crate::tokenstream::*;
noop_visit_tts(tts, self);
}
- fn visit_token(&mut self, t: &mut Token) {
+ fn visit_token(&mut self, t: &mut TokenKind) {
noop_visit_token(t, self);
}
}
// apply ident visitor if it's an ident, apply other visits to interpolated nodes
-pub fn noop_visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
+pub fn noop_visit_token<T: MutVisitor>(t: &mut TokenKind, vis: &mut T) {
match t {
token::Ident(id, _is_raw) => vis.visit_ident(id),
token::Lifetime(id) => vis.visit_ident(id),
pub fn expected_one_of_not_found(
&mut self,
- edible: &[token::Token],
- inedible: &[token::Token],
+ edible: &[token::TokenKind],
+ inedible: &[token::TokenKind],
) -> PResult<'a, bool /* recovered */> {
fn tokens_to_string(tokens: &[TokenType]) -> String {
let mut i = tokens.iter();
/// Eats and discards tokens until one of `kets` is encountered. Respects token trees,
/// passes through any errors encountered. Used for error recovery.
- crate fn eat_to_tokens(&mut self, kets: &[&token::Token]) {
+ crate fn eat_to_tokens(&mut self, kets: &[&token::TokenKind]) {
let handler = self.diagnostic();
if let Err(ref mut err) = self.parse_seq_to_before_tokens(
/// let _ = vec![1, 2, 3].into_iter().collect::<Vec<usize>>>>();
/// ^^ help: remove extra angle brackets
/// ```
- crate fn check_trailing_angle_brackets(&mut self, segment: &PathSegment, end: token::Token) {
+ crate fn check_trailing_angle_brackets(&mut self, segment: &PathSegment, end: token::TokenKind) {
// This function is intended to be invoked after parsing a path segment where there are two
// cases:
//
/// closing delimiter.
pub fn unexpected_try_recover(
&mut self,
- t: &token::Token,
+ t: &token::TokenKind,
) -> PResult<'a, bool /* recovered */> {
let token_str = pprust::token_to_string(t);
let this_token_str = self.this_token_descr();
crate fn recover_closing_delimiter(
&mut self,
- tokens: &[token::Token],
+ tokens: &[token::TokenKind],
mut err: DiagnosticBuilder<'a>,
) -> PResult<'a, bool> {
let mut pos = None;
use crate::ast::{self, Ident};
use crate::parse::ParseSess;
-use crate::parse::token::{self, Token};
+use crate::parse::token::{self, TokenKind};
use crate::symbol::{sym, Symbol};
use crate::parse::unescape;
use crate::parse::unescape_error_reporting::{emit_unescape_error, push_escaped_char};
#[derive(Clone, Debug)]
pub struct TokenAndSpan {
- pub tok: Token,
+ pub tok: TokenKind,
pub sp: Span,
}
/// Stop reading src at this index.
crate end_src_index: usize,
// cached:
- peek_tok: Token,
+ peek_tok: TokenKind,
peek_span: Span,
peek_span_src_raw: Span,
fatal_errs: Vec<DiagnosticBuilder<'a>>,
}
}
- fn binop(&mut self, op: token::BinOpToken) -> Token {
+ fn binop(&mut self, op: token::BinOpToken) -> TokenKind {
self.bump();
if self.ch_is('=') {
self.bump();
/// Returns the next token from the string, advances the input past that
/// token, and updates the interner
- fn next_token_inner(&mut self) -> Result<Token, ()> {
+ fn next_token_inner(&mut self) -> Result<TokenKind, ()> {
let c = self.ch;
if ident_start(c) {
let (kind, symbol) = self.scan_number(c.unwrap());
let suffix = self.scan_optional_raw_name();
debug!("next_token_inner: scanned number {:?}, {:?}, {:?}", kind, symbol, suffix);
- return Ok(Token::lit(kind, symbol, suffix));
+ return Ok(TokenKind::lit(kind, symbol, suffix));
}
match c.expect("next_token_inner called at EOF") {
let symbol = self.name_from(start);
self.bump();
self.validate_char_escape(start_with_quote);
- return Ok(Token::lit(token::Char, symbol, None));
+ return Ok(TokenKind::lit(token::Char, symbol, None));
}
// Include the leading `'` in the real identifier, for macro
let symbol = self.scan_single_quoted_string(start_with_quote, msg);
self.validate_char_escape(start_with_quote);
let suffix = self.scan_optional_raw_name();
- Ok(Token::lit(token::Char, symbol, suffix))
+ Ok(TokenKind::lit(token::Char, symbol, suffix))
}
'b' => {
self.bump();
};
let suffix = self.scan_optional_raw_name();
- Ok(Token::lit(kind, symbol, suffix))
+ Ok(TokenKind::lit(kind, symbol, suffix))
}
'"' => {
let start_with_quote = self.pos;
let symbol = self.scan_double_quoted_string(msg);
self.validate_str_escape(start_with_quote);
let suffix = self.scan_optional_raw_name();
- Ok(Token::lit(token::Str, symbol, suffix))
+ Ok(TokenKind::lit(token::Str, symbol, suffix))
}
'r' => {
let start_bpos = self.pos;
};
let suffix = self.scan_optional_raw_name();
- Ok(Token::lit(token::StrRaw(hash_count), symbol, suffix))
+ Ok(TokenKind::lit(token::StrRaw(hash_count), symbol, suffix))
}
'-' => {
if self.nextch_is('>') {
// check that the given reader produces the desired stream
// of tokens (stop checking after exhausting the expected vec)
- fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<Token>) {
+ fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<TokenKind>) {
for expected_tok in &expected {
assert_eq!(&string_reader.next_token().tok, expected_tok);
}
}
// make the identifier by looking up the string in the interner
- fn mk_ident(id: &str) -> Token {
- Token::from_ast_ident(Ident::from_str(id))
+ fn mk_ident(id: &str) -> TokenKind {
+ TokenKind::from_ast_ident(Ident::from_str(id))
}
- fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> Token {
- Token::lit(kind, Symbol::intern(symbol), suffix.map(Symbol::intern))
+ fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind {
+ TokenKind::lit(kind, Symbol::intern(symbol), suffix.map(Symbol::intern))
}
#[test]
struct TokenTreesReader<'a> {
string_reader: StringReader<'a>,
- token: token::Token,
+ token: token::TokenKind,
span: Span,
/// Stack of open delimiters and their spans. Used for error message.
open_braces: Vec<(token::DelimToken, Span)>,
use crate::ast::{self, Ident, Lit, LitKind};
use crate::parse::parser::Parser;
use crate::parse::PResult;
-use crate::parse::token::{self, Token};
+use crate::parse::token::{self, TokenKind};
use crate::parse::unescape::{unescape_str, unescape_char, unescape_byte_str, unescape_byte};
use crate::print::pprust;
use crate::symbol::{kw, sym, Symbol};
}
/// Converts arbitrary token into an AST literal.
- crate fn from_token(token: &Token, span: Span) -> Result<Lit, LitError> {
+ crate fn from_token(token: &TokenKind, span: Span) -> Result<Lit, LitError> {
let lit = match *token {
token::Ident(ident, false) if ident.name == kw::True || ident.name == kw::False =>
token::Lit::new(token::Bool, ident.name, None),
let next_span = self.look_ahead_span(1);
if self.span.hi() == next_span.lo() {
let s = String::from("0.") + &symbol.as_str();
- let token = Token::lit(token::Float, Symbol::intern(&s), suffix);
+ let token = TokenKind::lit(token::Float, Symbol::intern(&s), suffix);
return Some((token, self.span.to(next_span)));
}
}
/// A sequence separator.
pub struct SeqSep {
/// The seperator token.
- pub sep: Option<token::Token>,
+ pub sep: Option<token::TokenKind>,
/// `true` if a trailing separator is allowed.
pub trailing_sep_allowed: bool,
}
impl SeqSep {
- pub fn trailing_allowed(t: token::Token) -> SeqSep {
+ pub fn trailing_allowed(t: token::TokenKind) -> SeqSep {
SeqSep {
sep: Some(t),
trailing_sep_allowed: true,
#[derive(Clone)]
pub struct Parser<'a> {
pub sess: &'a ParseSess,
- /// The current token.
- pub token: token::Token,
- /// The span of the current token.
+ /// the current token:
+ pub token: token::TokenKind,
+ /// the span of the current token:
pub span: Span,
meta_var_span: Option<Span>,
/// The span of the previous token.
[
TokenTree::Token(sp, token::Ident(ast::Ident::with_empty_ctxt(sym::doc), false)),
TokenTree::Token(sp, token::Eq),
- TokenTree::Token(sp, token::Token::lit(
+ TokenTree::Token(sp, token::TokenKind::lit(
token::StrRaw(num_of_hashes), Symbol::intern(&stripped), None
)),
]
#[derive(Clone, PartialEq)]
crate enum TokenType {
- Token(token::Token),
+ Token(token::TokenKind),
Keyword(Symbol),
Operator,
Lifetime,
///
/// Types can also be of the form `IDENT(u8, u8) -> u8`, however this assumes
/// that `IDENT` is not the ident of a fn trait.
-fn can_continue_type_after_non_fn_ident(t: &token::Token) -> bool {
+fn can_continue_type_after_non_fn_ident(t: &token::TokenKind) -> bool {
t == &token::ModSep || t == &token::Lt ||
t == &token::BinOp(token::Shl)
}
}
/// Expects and consumes the token `t`. Signals an error if the next token is not `t`.
- pub fn expect(&mut self, t: &token::Token) -> PResult<'a, bool /* recovered */> {
+ pub fn expect(&mut self, t: &token::TokenKind) -> PResult<'a, bool /* recovered */> {
if self.expected_tokens.is_empty() {
if self.token == *t {
self.bump();
/// anything. Signal a fatal error if next token is unexpected.
pub fn expect_one_of(
&mut self,
- edible: &[token::Token],
- inedible: &[token::Token],
+ edible: &[token::TokenKind],
+ inedible: &[token::TokenKind],
) -> PResult<'a, bool /* recovered */> {
if edible.contains(&self.token) {
self.bump();
///
/// This method will automatically add `tok` to `expected_tokens` if `tok` is not
/// encountered.
- crate fn check(&mut self, tok: &token::Token) -> bool {
+ crate fn check(&mut self, tok: &token::TokenKind) -> bool {
let is_present = self.token == *tok;
if !is_present { self.expected_tokens.push(TokenType::Token(tok.clone())); }
is_present
}
/// Consumes a token 'tok' if it exists. Returns whether the given token was present.
- pub fn eat(&mut self, tok: &token::Token) -> bool {
+ pub fn eat(&mut self, tok: &token::TokenKind) -> bool {
let is_present = self.check(tok);
if is_present { self.bump() }
is_present
/// `f` must consume tokens until reaching the next separator or
/// closing bracket.
pub fn parse_seq_to_end<T, F>(&mut self,
- ket: &token::Token,
+ ket: &token::TokenKind,
sep: SeqSep,
f: F)
-> PResult<'a, Vec<T>> where
/// closing bracket.
pub fn parse_seq_to_before_end<T, F>(
&mut self,
- ket: &token::Token,
+ ket: &token::TokenKind,
sep: SeqSep,
f: F,
) -> PResult<'a, (Vec<T>, bool)>
crate fn parse_seq_to_before_tokens<T, F>(
&mut self,
- kets: &[&token::Token],
+ kets: &[&token::TokenKind],
sep: SeqSep,
expect: TokenExpectType,
mut f: F,
/// closing bracket.
fn parse_unspanned_seq<T, F>(
&mut self,
- bra: &token::Token,
- ket: &token::Token,
+ bra: &token::TokenKind,
+ ket: &token::TokenKind,
sep: SeqSep,
f: F,
) -> PResult<'a, Vec<T>> where
/// Advance the parser using provided token as a next one. Use this when
/// consuming a part of a token. For example a single `<` from `<<`.
- fn bump_with(&mut self, next: token::Token, span: Span) {
+ fn bump_with(&mut self, next: token::TokenKind, span: Span) {
self.prev_span = self.span.with_hi(span.lo());
// It would be incorrect to record the kind of the current token, but
// fortunately for tokens currently using `bump_with`, the
}
pub fn look_ahead<R, F>(&self, dist: usize, f: F) -> R where
- F: FnOnce(&token::Token) -> R,
+ F: FnOnce(&token::TokenKind) -> R,
{
if dist == 0 {
return f(&self.token)
fn parse_path_segment(&mut self, style: PathStyle) -> PResult<'a, PathSegment> {
let ident = self.parse_path_segment_ident()?;
- let is_args_start = |token: &token::Token| match *token {
+ let is_args_start = |token: &token::TokenKind| match *token {
token::Lt | token::BinOp(token::Shl) | token::OpenDelim(token::Paren)
| token::LArrow => true,
_ => false,
let ex: ExprKind;
- // Note: when adding new syntax here, don't forget to adjust Token::can_begin_expr().
+ // Note: when adding new syntax here, don't forget to adjust TokenKind::can_begin_expr().
match self.token {
token::OpenDelim(token::Paren) => {
self.bump();
-> PResult<'a, P<Expr>> {
let attrs = self.parse_or_use_outer_attributes(already_parsed_attrs)?;
let lo = self.span;
- // Note: when adding new unary operators, don't forget to adjust Token::can_begin_expr()
+ // Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr()
let (hi, ex) = match self.token {
token::Not => {
self.bump();
// `not` is just an ordinary identifier in Rust-the-language,
// but as `rustc`-the-compiler, we can issue clever diagnostics
// for confused users who really want to say `!`
- let token_cannot_continue_expr = |t: &token::Token| match *t {
+ let token_cannot_continue_expr = |t: &token::TokenKind| match *t {
// These tokens can start an expression after `!`, but
// can't continue an expression after an ident
token::Ident(ident, is_raw) => token::ident_can_begin_expr(ident, is_raw),
let mut last_plus_span = None;
let mut was_negative = false;
loop {
- // This needs to be synchronized with `Token::can_begin_bound`.
+ // This needs to be synchronized with `TokenKind::can_begin_bound`.
let is_bound_start = self.check_path() || self.check_lifetime() ||
self.check(&token::Not) || // used for error reporting only
self.check(&token::Question) ||
}
/// Given a termination token, parses all of the items in a module.
- fn parse_mod_items(&mut self, term: &token::Token, inner_lo: Span) -> PResult<'a, Mod> {
+ fn parse_mod_items(&mut self, term: &token::TokenKind, inner_lo: Span) -> PResult<'a, Mod> {
let mut items = vec![];
while let Some(item) = self.parse_item()? {
items.push(item);
pub use Nonterminal::*;
pub use DelimToken::*;
pub use LitKind::*;
-pub use Token::*;
+pub use TokenKind::*;
use crate::ast::{self};
use crate::parse::ParseSess;
}
pub(crate) fn ident_can_begin_expr(ident: ast::Ident, is_raw: bool) -> bool {
- let ident_token: Token = Ident(ident, is_raw);
+ let ident_token: TokenKind = Ident(ident, is_raw);
!ident_token.is_reserved_ident() ||
ident_token.is_path_segment_keyword() ||
}
fn ident_can_begin_type(ident: ast::Ident, is_raw: bool) -> bool {
- let ident_token: Token = Ident(ident, is_raw);
+ let ident_token: TokenKind = Ident(ident, is_raw);
!ident_token.is_reserved_ident() ||
ident_token.is_path_segment_keyword() ||
}
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Debug)]
-pub enum Token {
+pub enum TokenKind {
/* Expression-operator symbols. */
Eq,
Lt,
Eof,
}
-// `Token` is used a lot. Make sure it doesn't unintentionally get bigger.
+// `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger.
#[cfg(target_arch = "x86_64")]
-static_assert_size!(Token, 16);
+static_assert_size!(TokenKind, 16);
-impl Token {
- /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary.
- pub fn from_ast_ident(ident: ast::Ident) -> Token {
+impl TokenKind {
+ /// Recovers a `TokenKind` from an `ast::Ident`. This creates a raw identifier if necessary.
+ pub fn from_ast_ident(ident: ast::Ident) -> TokenKind {
Ident(ident, ident.is_raw_guess())
}
self == &Question || self == &OpenDelim(Paren)
}
- pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Token {
+ pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> TokenKind {
Literal(Lit::new(kind, symbol, suffix))
}
}
}
- crate fn glue(self, joint: Token) -> Option<Token> {
+ crate fn glue(self, joint: TokenKind) -> Option<TokenKind> {
Some(match self {
Eq => match joint {
Eq => EqEq,
/// Returns tokens that are likely to be typed accidentally instead of the current token.
/// Enables better error recovery when the wrong token is found.
- crate fn similar_tokens(&self) -> Option<Vec<Token>> {
+ crate fn similar_tokens(&self) -> Option<Vec<TokenKind>> {
match *self {
Comma => Some(vec![Dot, Lt, Semi]),
Semi => Some(vec![Colon, Comma]),
// See comments in `Nonterminal::to_tokenstream` for why we care about
// *probably* equal here rather than actual equality
- crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool {
+ crate fn probably_equal_for_proc_macro(&self, other: &TokenKind) -> bool {
if mem::discriminant(self) != mem::discriminant(other) {
return false
}
}
}
-crate fn is_op(tok: &Token) -> bool {
+crate fn is_op(tok: &TokenKind) -> bool {
match *tok {
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) |
Ident(..) | Lifetime(..) | Interpolated(..) |
use crate::util::parser::{self, AssocOp, Fixity};
use crate::attr;
use crate::source_map::{self, SourceMap, Spanned};
-use crate::parse::token::{self, BinOpToken, Nonterminal, Token};
+use crate::parse::token::{self, BinOpToken, Nonterminal, TokenKind};
use crate::parse::lexer::comments;
use crate::parse::{self, ParseSess};
use crate::print::pp::{self, Breaks};
out
}
-pub fn token_to_string(tok: &Token) -> String {
+pub fn token_to_string(tok: &TokenKind) -> String {
match *tok {
token::Eq => "=".to_string(),
token::Lt => "<".to_string(),
use crate::ext::base;
use crate::ext::tt::{macro_parser, quoted};
use crate::parse::Directory;
-use crate::parse::token::{self, DelimToken, Token};
+use crate::parse::token::{self, DelimToken, TokenKind};
use crate::print::pprust;
use syntax_pos::{BytePos, Mark, Span, DUMMY_SP};
#[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)]
pub enum TokenTree {
/// A single token
- Token(Span, token::Token),
+ Token(Span, token::TokenKind),
/// A delimited sequence of token trees
Delimited(DelimSpan, DelimToken, TokenStream),
}
fn _dummy()
where
Span: Send + Sync,
- token::Token: Send + Sync,
+ token::TokenKind: Send + Sync,
DelimSpan: Send + Sync,
DelimToken: Send + Sync,
TokenStream: Send + Sync,
}
/// Indicates if the stream is a token that is equal to the provided token.
- pub fn eq_token(&self, t: Token) -> bool {
+ pub fn eq_token(&self, t: TokenKind) -> bool {
match *self {
TokenTree::Token(_, ref tk) => *tk == t,
_ => false,
}
}
-impl From<Token> for TokenStream {
- fn from(token: Token) -> TokenStream {
+impl From<TokenKind> for TokenStream {
+ fn from(token: TokenKind) -> TokenStream {
TokenTree::Token(DUMMY_SP, token).into()
}
}
use super::*;
use crate::syntax::ast::Ident;
use crate::with_default_globals;
- use crate::parse::token::Token;
+ use crate::parse::token::TokenKind;
use crate::util::parser_testing::string_to_stream;
use syntax_pos::{Span, BytePos, NO_EXPANSION};
-use crate::parse::token::{self, Token, BinOpToken};
+use crate::parse::token::{self, TokenKind, BinOpToken};
use crate::symbol::kw;
use crate::ast::{self, BinOpKind};
impl AssocOp {
/// Creates a new AssocOP from a token
- pub fn from_token(t: &Token) -> Option<AssocOp> {
+ pub fn from_token(t: &Token
Kind) -> Option<AssocOp> {
use AssocOp::*;
match *t {
token::BinOpEq(k) => Some(AssignOp(k)),
//! those that are created by the expansion of a macro.
use crate::ast::*;
-use crate::parse::token::Token;
+use crate::parse::token::TokenKind;
use crate::tokenstream::{TokenTree, TokenStream};
use syntax_pos::Span;
fn visit_tts(&mut self, tts: TokenStream) {
walk_tts(self, tts)
}
- fn visit_token(&mut self, _t: Token) {}
+ fn visit_token(&mut self, _t: TokenKind) {}
// FIXME: add `visit_interpolated` and `walk_interpolated`
fn visit_vis(&mut self, vis: &'ast Visibility) {
walk_vis(self, vis)
use syntax::source_map::Spanned;
use syntax::ext::base::*;
use syntax::ext::build::AstBuilder;
-use syntax::parse::token::{self, Token};
+use syntax::parse::token::{self, TokenKind};
use syntax::parse::parser::Parser;
use syntax::print::pprust;
use syntax::ptr::P;
tts: custom_message.unwrap_or_else(|| {
TokenStream::from(TokenTree::Token(
DUMMY_SP,
- Token::lit(token::Str, Symbol::intern(&format!(
+ TokenKind::lit(token::Str, Symbol::intern(&format!(
"assertion failed: {}",
pprust::expr_to_string(&cond_expr).escape_debug()
)), None),
let stream = vec![
Ident(ast::Ident::new(sym::doc, span), false),
Eq,
- Token::lit(token::Str, Symbol::intern(&escaped), None),
+ TokenKind::lit(token::Str, Symbol::intern(&escaped), None),
]
.into_iter()
.map(|token| tokenstream::TokenTree::Token(span, token))
}) if symbol.as_str().starts_with("-") => {
let minus = BinOp(BinOpToken::Minus);
let symbol = Symbol::intern(&symbol.as_str()[1..]);
- let integer = Token::lit(token::Integer, symbol, suffix);
+ let integer = TokenKind::lit(token::Integer, symbol, suffix);
let a = tokenstream::TokenTree::Token(span, minus);
let b = tokenstream::TokenTree::Token(span, integer);
return vec![a, b].into_iter().collect();
}) if symbol.as_str().starts_with("-") => {
let minus = BinOp(BinOpToken::Minus);
let symbol = Symbol::intern(&symbol.as_str()[1..]);
- let float = Token::lit(token::Float, symbol, suffix);
+ let float = TokenKind::lit(token::Float, symbol, suffix);
let a = tokenstream::TokenTree::Token(span, minus);
let b = tokenstream::TokenTree::Token(span, float);
return vec![a, b].into_iter().collect();
projects / rust.git / commitdiff