src/libsyntax/parse/lexer/mod.rs

   1 use crate::ast::{self, Ident};
   2 use crate::source_map::{SourceMap, FilePathMapping};
   3 use crate::parse::{token, ParseSess};
   4 use crate::symbol::Symbol;
   5
   6 use errors::{Applicability, FatalError, Diagnostic, DiagnosticBuilder};
   7 use syntax_pos::{BytePos, CharPos, Pos, Span, NO_EXPANSION};
   8 use core::unicode::property::Pattern_White_Space;
   9
  10 use std::borrow::Cow;
  11 use std::char;
  12 use std::iter;
  13 use std::mem::replace;
  14 use rustc_data_structures::sync::Lrc;
  15 use log::debug;
  16
  17 pub mod comments;
  18 mod tokentrees;
  19 mod unicode_chars;
  20
  21 #[derive(Clone, Debug)]
  22 pub struct TokenAndSpan {
  23     pub tok: token::Token,
  24     pub sp: Span,
  25 }
  26
  27 impl Default for TokenAndSpan {
  28     fn default() -> Self {
  29         TokenAndSpan {
  30             tok: token::Whitespace,
  31             sp: syntax_pos::DUMMY_SP,
  32         }
  33     }
  34 }
  35
  36 #[derive(Clone, Debug)]
  37 pub struct UnmatchedBrace {
  38     pub expected_delim: token::DelimToken,
  39     pub found_delim: token::DelimToken,
  40     pub found_span: Span,
  41     pub unclosed_span: Option<Span>,
  42     pub candidate_span: Option<Span>,
  43 }
  44
  45 pub struct StringReader<'a> {
  46     pub sess: &'a ParseSess,
  47     /// The absolute offset within the source_map of the next character to read
  48     pub next_pos: BytePos,
  49     /// The absolute offset within the source_map of the current character
  50     pub pos: BytePos,
  51     /// The current character (which has been read from self.pos)
  52     pub ch: Option<char>,
  53     pub source_file: Lrc<syntax_pos::SourceFile>,
  54     /// Stop reading src at this index.
  55     pub end_src_index: usize,
  56     // cached:
  57     peek_tok: token::Token,
  58     peek_span: Span,
  59     peek_span_src_raw: Span,
  60     fatal_errs: Vec<DiagnosticBuilder<'a>>,
  61     // cache a direct reference to the source text, so that we don't have to
  62     // retrieve it via `self.source_file.src.as_ref().unwrap()` all the time.
  63     src: Lrc<String>,
  64     token: token::Token,
  65     span: Span,
  66     /// The raw source span which *does not* take `override_span` into account
  67     span_src_raw: Span,
  68     /// Stack of open delimiters and their spans. Used for error message.
  69     open_braces: Vec<(token::DelimToken, Span)>,
  70     crate unmatched_braces: Vec<UnmatchedBrace>,
  71     /// The type and spans for all braces
  72     ///
  73     /// Used only for error recovery when arriving to EOF with mismatched braces.
  74     matching_delim_spans: Vec<(token::DelimToken, Span, Span)>,
  75     crate override_span: Option<Span>,
  76     last_unclosed_found_span: Option<Span>,
  77 }
  78
  79 impl<'a> StringReader<'a> {
  80     fn mk_sp(&self, lo: BytePos, hi: BytePos) -> Span {
  81         self.mk_sp_and_raw(lo, hi).0
  82     }
  83
  84     fn mk_sp_and_raw(&self, lo: BytePos, hi: BytePos) -> (Span, Span) {
  85         let raw = Span::new(lo, hi, NO_EXPANSION);
  86         let real = self.override_span.unwrap_or(raw);
  87
  88         (real, raw)
  89     }
  90
  91     fn mk_ident(&self, string: &str) -> Ident {
  92         let mut ident = Ident::from_str(string);
  93         if let Some(span) = self.override_span {
  94             ident.span = span;
  95         }
  96
  97         ident
  98     }
  99
 100     fn unwrap_or_abort(&mut self, res: Result<TokenAndSpan, ()>) -> TokenAndSpan {
 101         match res {
 102             Ok(tok) => tok,
 103             Err(_) => {
 104                 self.emit_fatal_errors();
 105                 FatalError.raise();
 106             }
 107         }
 108     }
 109
 110     fn next_token(&mut self) -> TokenAndSpan where Self: Sized {
 111         let res = self.try_next_token();
 112         self.unwrap_or_abort(res)
 113     }
 114
 115     /// Returns the next token. EFFECT: advances the string_reader.
 116     pub fn try_next_token(&mut self) -> Result<TokenAndSpan, ()> {
 117         assert!(self.fatal_errs.is_empty());
 118         let ret_val = TokenAndSpan {
 119             tok: replace(&mut self.peek_tok, token::Whitespace),
 120             sp: self.peek_span,
 121         };
 122         self.advance_token()?;
 123         self.span_src_raw = self.peek_span_src_raw;
 124
 125         Ok(ret_val)
 126     }
 127
 128     /// Immutably extract string if found at current position with given delimiters
 129     pub fn peek_delimited(&self, from_ch: char, to_ch: char) -> Option<String> {
 130         let mut pos = self.pos;
 131         let mut idx = self.src_index(pos);
 132         let mut ch = char_at(&self.src, idx);
 133         if ch != from_ch {
 134             return None;
 135         }
 136         pos = pos + Pos::from_usize(ch.len_utf8());
 137         let start_pos = pos;
 138         idx = self.src_index(pos);
 139         while idx < self.end_src_index {
 140             ch = char_at(&self.src, idx);
 141             if ch == to_ch {
 142                 return Some(self.src[self.src_index(start_pos)..self.src_index(pos)].to_string());
 143             }
 144             pos = pos + Pos::from_usize(ch.len_utf8());
 145             idx = self.src_index(pos);
 146         }
 147         return None;
 148     }
 149
 150     fn try_real_token(&mut self) -> Result<TokenAndSpan, ()> {
 151         let mut t = self.try_next_token()?;
 152         loop {
 153             match t.tok {
 154                 token::Whitespace | token::Comment | token::Shebang(_) => {
 155                     t = self.try_next_token()?;
 156                 }
 157                 _ => break,
 158             }
 159         }
 160
 161         self.token = t.tok.clone();
 162         self.span = t.sp;
 163
 164         Ok(t)
 165     }
 166
 167     pub fn real_token(&mut self) -> TokenAndSpan {
 168         let res = self.try_real_token();
 169         self.unwrap_or_abort(res)
 170     }
 171
 172     #[inline]
 173     fn is_eof(&self) -> bool {
 174         self.ch.is_none()
 175     }
 176
 177     fn fail_unterminated_raw_string(&self, pos: BytePos, hash_count: u16) {
 178         let mut err = self.struct_span_fatal(pos, pos, "unterminated raw string");
 179         err.span_label(self.mk_sp(pos, pos), "unterminated raw string");
 180
 181         if hash_count > 0 {
 182             err.note(&format!("this raw string should be terminated with `\"{}`",
 183                               "#".repeat(hash_count as usize)));
 184         }
 185
 186         err.emit();
 187         FatalError.raise();
 188     }
 189
 190     fn fatal(&self, m: &str) -> FatalError {
 191         self.fatal_span(self.peek_span, m)
 192     }
 193
 194     pub fn emit_fatal_errors(&mut self) {
 195         for err in &mut self.fatal_errs {
 196             err.emit();
 197         }
 198
 199         self.fatal_errs.clear();
 200     }
 201
 202     pub fn buffer_fatal_errors(&mut self) -> Vec<Diagnostic> {
 203         let mut buffer = Vec::new();
 204
 205         for err in self.fatal_errs.drain(..) {
 206             err.buffer(&mut buffer);
 207         }
 208
 209         buffer
 210     }
 211
 212     pub fn peek(&self) -> TokenAndSpan {
 213         // FIXME(pcwalton): Bad copy!
 214         TokenAndSpan {
 215             tok: self.peek_tok.clone(),
 216             sp: self.peek_span,
 217         }
 218     }
 219
 220     /// For comments.rs, which hackily pokes into next_pos and ch
 221     fn new_raw(sess: &'a ParseSess,
 222                source_file: Lrc<syntax_pos::SourceFile>,
 223                override_span: Option<Span>) -> Self {
 224         let mut sr = StringReader::new_raw_internal(sess, source_file, override_span);
 225         sr.bump();
 226
 227         sr
 228     }
 229
 230     fn new_raw_internal(sess: &'a ParseSess, source_file: Lrc<syntax_pos::SourceFile>,
 231         override_span: Option<Span>) -> Self
 232     {
 233         if source_file.src.is_none() {
 234             sess.span_diagnostic.bug(&format!("Cannot lex source_file without source: {}",
 235                                               source_file.name));
 236         }
 237
 238         let src = (*source_file.src.as_ref().unwrap()).clone();
 239
 240         StringReader {
 241             sess,
 242             next_pos: source_file.start_pos,
 243             pos: source_file.start_pos,
 244             ch: Some('\n'),
 245             source_file,
 246             end_src_index: src.len(),
 247             // dummy values; not read
 248             peek_tok: token::Eof,
 249             peek_span: syntax_pos::DUMMY_SP,
 250             peek_span_src_raw: syntax_pos::DUMMY_SP,
 251             src,
 252             fatal_errs: Vec::new(),
 253             token: token::Eof,
 254             span: syntax_pos::DUMMY_SP,
 255             span_src_raw: syntax_pos::DUMMY_SP,
 256             open_braces: Vec::new(),
 257             unmatched_braces: Vec::new(),
 258             matching_delim_spans: Vec::new(),
 259             override_span,
 260             last_unclosed_found_span: None,
 261         }
 262     }
 263
 264     pub fn new(sess: &'a ParseSess,
 265                source_file: Lrc<syntax_pos::SourceFile>,
 266                override_span: Option<Span>) -> Self {
 267         let mut sr = StringReader::new_raw(sess, source_file, override_span);
 268         if sr.advance_token().is_err() {
 269             sr.emit_fatal_errors();
 270             FatalError.raise();
 271         }
 272
 273         sr
 274     }
 275
 276     pub fn new_or_buffered_errs(sess: &'a ParseSess,
 277                                 source_file: Lrc<syntax_pos::SourceFile>,
 278                                 override_span: Option<Span>) -> Result<Self, Vec<Diagnostic>> {
 279         let mut sr = StringReader::new_raw(sess, source_file, override_span);
 280         if sr.advance_token().is_err() {
 281             Err(sr.buffer_fatal_errors())
 282         } else {
 283             Ok(sr)
 284         }
 285     }
 286
 287     pub fn retokenize(sess: &'a ParseSess, mut span: Span) -> Self {
 288         let begin = sess.source_map().lookup_byte_offset(span.lo());
 289         let end = sess.source_map().lookup_byte_offset(span.hi());
 290
 291         // Make the range zero-length if the span is invalid.
 292         if span.lo() > span.hi() || begin.sf.start_pos != end.sf.start_pos {
 293             span = span.shrink_to_lo();
 294         }
 295
 296         let mut sr = StringReader::new_raw_internal(sess, begin.sf, None);
 297
 298         // Seek the lexer to the right byte range.
 299         sr.next_pos = span.lo();
 300         sr.end_src_index = sr.src_index(span.hi());
 301
 302         sr.bump();
 303
 304         if sr.advance_token().is_err() {
 305             sr.emit_fatal_errors();
 306             FatalError.raise();
 307         }
 308
 309         sr
 310     }
 311
 312     #[inline]
 313     fn ch_is(&self, c: char) -> bool {
 314         self.ch == Some(c)
 315     }
 316
 317     /// Report a fatal lexical error with a given span.
 318     fn fatal_span(&self, sp: Span, m: &str) -> FatalError {
 319         self.sess.span_diagnostic.span_fatal(sp, m)
 320     }
 321
 322     /// Report a lexical error with a given span.
 323     fn err_span(&self, sp: Span, m: &str) {
 324         self.sess.span_diagnostic.struct_span_err(sp, m).emit();
 325     }
 326
 327
 328     /// Report a fatal error spanning [`from_pos`, `to_pos`).
 329     fn fatal_span_(&self, from_pos: BytePos, to_pos: BytePos, m: &str) -> FatalError {
 330         self.fatal_span(self.mk_sp(from_pos, to_pos), m)
 331     }
 332
 333     /// Report a lexical error spanning [`from_pos`, `to_pos`).
 334     fn err_span_(&self, from_pos: BytePos, to_pos: BytePos, m: &str) {
 335         self.err_span(self.mk_sp(from_pos, to_pos), m)
 336     }
 337
 338     /// Pushes a character to a message string for error reporting
 339     fn push_escaped_char_for_msg(m: &mut String, c: char) {
 340         match c {
 341             '\u{20}'..='\u{7e}' => {
 342                 // Don't escape \, ' or " for user-facing messages
 343                 m.push(c);
 344             }
 345             _ => {
 346                 m.extend(c.escape_default());
 347             }
 348         }
 349     }
 350
 351     /// Report a lexical error spanning [`from_pos`, `to_pos`), appending an
 352     /// escaped character to the error message
 353     fn fatal_span_char(&self, from_pos: BytePos, to_pos: BytePos, m: &str, c: char) -> FatalError {
 354         let mut m = m.to_string();
 355         m.push_str(": ");
 356         Self::push_escaped_char_for_msg(&mut m, c);
 357
 358         self.fatal_span_(from_pos, to_pos, &m[..])
 359     }
 360
 361     fn struct_span_fatal(&self, from_pos: BytePos, to_pos: BytePos, m: &str)
 362         -> DiagnosticBuilder<'a>
 363     {
 364         self.sess.span_diagnostic.struct_span_fatal(self.mk_sp(from_pos, to_pos), m)
 365     }
 366
 367     fn struct_fatal_span_char(&self, from_pos: BytePos, to_pos: BytePos, m: &str, c: char)
 368         -> DiagnosticBuilder<'a>
 369     {
 370         let mut m = m.to_string();
 371         m.push_str(": ");
 372         Self::push_escaped_char_for_msg(&mut m, c);
 373
 374         self.sess.span_diagnostic.struct_span_fatal(self.mk_sp(from_pos, to_pos), &m[..])
 375     }
 376
 377     /// Report a lexical error spanning [`from_pos`, `to_pos`), appending an
 378     /// escaped character to the error message
 379     fn err_span_char(&self, from_pos: BytePos, to_pos: BytePos, m: &str, c: char) {
 380         let mut m = m.to_string();
 381         m.push_str(": ");
 382         Self::push_escaped_char_for_msg(&mut m, c);
 383         self.err_span_(from_pos, to_pos, &m[..]);
 384     }
 385
 386     fn struct_err_span_char(&self, from_pos: BytePos, to_pos: BytePos, m: &str, c: char)
 387         -> DiagnosticBuilder<'a>
 388     {
 389         let mut m = m.to_string();
 390         m.push_str(": ");
 391         Self::push_escaped_char_for_msg(&mut m, c);
 392
 393         self.sess.span_diagnostic.struct_span_err(self.mk_sp(from_pos, to_pos), &m[..])
 394     }
 395
 396     /// Report a lexical error spanning [`from_pos`, `to_pos`), appending the
 397     /// offending string to the error message
 398     fn fatal_span_verbose(&self, from_pos: BytePos, to_pos: BytePos, mut m: String) -> FatalError {
 399         m.push_str(": ");
 400         m.push_str(&self.src[self.src_index(from_pos)..self.src_index(to_pos)]);
 401
 402         self.fatal_span_(from_pos, to_pos, &m[..])
 403     }
 404
 405     /// Advance peek_tok and peek_span to refer to the next token, and
 406     /// possibly update the interner.
 407     fn advance_token(&mut self) -> Result<(), ()> {
 408         match self.scan_whitespace_or_comment() {
 409             Some(comment) => {
 410                 self.peek_span_src_raw = comment.sp;
 411                 self.peek_span = comment.sp;
 412                 self.peek_tok = comment.tok;
 413             }
 414             None => {
 415                 if self.is_eof() {
 416                     self.peek_tok = token::Eof;
 417                     let (real, raw) = self.mk_sp_and_raw(
 418                         self.source_file.end_pos,
 419                         self.source_file.end_pos,
 420                     );
 421                     self.peek_span = real;
 422                     self.peek_span_src_raw = raw;
 423                 } else {
 424                     let start_bytepos = self.pos;
 425                     self.peek_tok = self.next_token_inner()?;
 426                     let (real, raw) = self.mk_sp_and_raw(start_bytepos, self.pos);
 427                     self.peek_span = real;
 428                     self.peek_span_src_raw = raw;
 429                 };
 430             }
 431         }
 432
 433         Ok(())
 434     }
 435
 436     #[inline]
 437     fn src_index(&self, pos: BytePos) -> usize {
 438         (pos - self.source_file.start_pos).to_usize()
 439     }
 440
 441     /// Calls `f` with a string slice of the source text spanning from `start`
 442     /// up to but excluding `self.pos`, meaning the slice does not include
 443     /// the character `self.ch`.
 444     fn with_str_from<T, F>(&self, start: BytePos, f: F) -> T
 445         where F: FnOnce(&str) -> T
 446     {
 447         self.with_str_from_to(start, self.pos, f)
 448     }
 449
 450     /// Creates a Name from a given offset to the current offset, each
 451     /// adjusted 1 towards each other (assumes that on either side there is a
 452     /// single-byte delimiter).
 453     fn name_from(&self, start: BytePos) -> ast::Name {
 454         debug!("taking an ident from {:?} to {:?}", start, self.pos);
 455         self.with_str_from(start, Symbol::intern)
 456     }
 457
 458     /// As name_from, with an explicit endpoint.
 459     fn name_from_to(&self, start: BytePos, end: BytePos) -> ast::Name {
 460         debug!("taking an ident from {:?} to {:?}", start, end);
 461         self.with_str_from_to(start, end, Symbol::intern)
 462     }
 463
 464     /// Calls `f` with a string slice of the source text spanning from `start`
 465     /// up to but excluding `end`.
 466     fn with_str_from_to<T, F>(&self, start: BytePos, end: BytePos, f: F) -> T
 467         where F: FnOnce(&str) -> T
 468     {
 469         f(&self.src[self.src_index(start)..self.src_index(end)])
 470     }
 471
 472     /// Converts CRLF to LF in the given string, raising an error on bare CR.
 473     fn translate_crlf<'b>(&self, start: BytePos, s: &'b str, errmsg: &'b str) -> Cow<'b, str> {
 474         let mut chars = s.char_indices().peekable();
 475         while let Some((i, ch)) = chars.next() {
 476             if ch == '\r' {
 477                 if let Some((lf_idx, '\n')) = chars.peek() {
 478                     return translate_crlf_(self, start, s, *lf_idx, chars, errmsg).into();
 479                 }
 480                 let pos = start + BytePos(i as u32);
 481                 let end_pos = start + BytePos((i + ch.len_utf8()) as u32);
 482                 self.err_span_(pos, end_pos, errmsg);
 483             }
 484         }
 485         return s.into();
 486
 487         fn translate_crlf_(rdr: &StringReader<'_>,
 488                            start: BytePos,
 489                            s: &str,
 490                            mut j: usize,
 491                            mut chars: iter::Peekable<impl Iterator<Item = (usize, char)>>,
 492                            errmsg: &str)
 493                            -> String {
 494             let mut buf = String::with_capacity(s.len());
 495             // Skip first CR
 496             buf.push_str(&s[.. j - 1]);
 497             while let Some((i, ch)) = chars.next() {
 498                 if ch == '\r' {
 499                     if j < i {
 500                         buf.push_str(&s[j..i]);
 501                     }
 502                     let next = i + ch.len_utf8();
 503                     j = next;
 504                     if chars.peek().map(|(_, ch)| *ch) != Some('\n') {
 505                         let pos = start + BytePos(i as u32);
 506                         let end_pos = start + BytePos(next as u32);
 507                         rdr.err_span_(pos, end_pos, errmsg);
 508                     }
 509                 }
 510             }
 511             if j < s.len() {
 512                 buf.push_str(&s[j..]);
 513             }
 514             buf
 515         }
 516     }
 517
 518     /// Advance the StringReader by one character.
 519     crate fn bump(&mut self) {
 520         let next_src_index = self.src_index(self.next_pos);
 521         if next_src_index < self.end_src_index {
 522             let next_ch = char_at(&self.src, next_src_index);
 523             let next_ch_len = next_ch.len_utf8();
 524
 525             self.ch = Some(next_ch);
 526             self.pos = self.next_pos;
 527             self.next_pos = self.next_pos + Pos::from_usize(next_ch_len);
 528         } else {
 529             self.ch = None;
 530             self.pos = self.next_pos;
 531         }
 532     }
 533
 534     fn nextch(&self) -> Option<char> {
 535         let next_src_index = self.src_index(self.next_pos);
 536         if next_src_index < self.end_src_index {
 537             Some(char_at(&self.src, next_src_index))
 538         } else {
 539             None
 540         }
 541     }
 542
 543     #[inline]
 544     fn nextch_is(&self, c: char) -> bool {
 545         self.nextch() == Some(c)
 546     }
 547
 548     fn nextnextch(&self) -> Option<char> {
 549         let next_src_index = self.src_index(self.next_pos);
 550         if next_src_index < self.end_src_index {
 551             let next_next_src_index =
 552                 next_src_index + char_at(&self.src, next_src_index).len_utf8();
 553             if next_next_src_index < self.end_src_index {
 554                 return Some(char_at(&self.src, next_next_src_index));
 555             }
 556         }
 557         None
 558     }
 559
 560     #[inline]
 561     fn nextnextch_is(&self, c: char) -> bool {
 562         self.nextnextch() == Some(c)
 563     }
 564
 565     /// Eats <XID_start><XID_continue>*, if possible.
 566     fn scan_optional_raw_name(&mut self) -> Option<ast::Name> {
 567         if !ident_start(self.ch) {
 568             return None;
 569         }
 570
 571         let start = self.pos;
 572         self.bump();
 573
 574         while ident_continue(self.ch) {
 575             self.bump();
 576         }
 577
 578         self.with_str_from(start, |string| {
 579             if string == "_" {
 580                 self.sess.span_diagnostic
 581                     .struct_span_warn(self.mk_sp(start, self.pos),
 582                                       "underscore literal suffix is not allowed")
 583                     .warn("this was previously accepted by the compiler but is \
 584                           being phased out; it will become a hard error in \
 585                           a future release!")
 586                     .note("for more information, see issue #42326 \
 587                           <https://github.com/rust-lang/rust/issues/42326>")
 588                     .emit();
 589                 None
 590             } else {
 591                 Some(Symbol::intern(string))
 592             }
 593         })
 594     }
 595
 596     /// PRECONDITION: self.ch is not whitespace
 597     /// Eats any kind of comment.
 598     fn scan_comment(&mut self) -> Option<TokenAndSpan> {
 599         if let Some(c) = self.ch {
 600             if c.is_whitespace() {
 601                 let msg = "called consume_any_line_comment, but there was whitespace";
 602                 self.sess.span_diagnostic.span_err(self.mk_sp(self.pos, self.pos), msg);
 603             }
 604         }
 605
 606         if self.ch_is('/') {
 607             match self.nextch() {
 608                 Some('/') => {
 609                     self.bump();
 610                     self.bump();
 611
 612                     // line comments starting with "///" or "//!" are doc-comments
 613                     let doc_comment = (self.ch_is('/') && !self.nextch_is('/')) || self.ch_is('!');
 614                     let start_bpos = self.pos - BytePos(2);
 615
 616                     while !self.is_eof() {
 617                         match self.ch.unwrap() {
 618                             '\n' => break,
 619                             '\r' => {
 620                                 if self.nextch_is('\n') {
 621                                     // CRLF
 622                                     break;
 623                                 } else if doc_comment {
 624                                     self.err_span_(self.pos,
 625                                                    self.next_pos,
 626                                                    "bare CR not allowed in doc-comment");
 627                                 }
 628                             }
 629                             _ => (),
 630                         }
 631                         self.bump();
 632                     }
 633
 634                     if doc_comment {
 635                         self.with_str_from(start_bpos, |string| {
 636                             // comments with only more "/"s are not doc comments
 637                             let tok = if is_doc_comment(string) {
 638                                 token::DocComment(Symbol::intern(string))
 639                             } else {
 640                                 token::Comment
 641                             };
 642
 643                             Some(TokenAndSpan {
 644                                 tok,
 645                                 sp: self.mk_sp(start_bpos, self.pos),
 646                             })
 647                         })
 648                     } else {
 649                         Some(TokenAndSpan {
 650                             tok: token::Comment,
 651                             sp: self.mk_sp(start_bpos, self.pos),
 652                         })
 653                     }
 654                 }
 655                 Some('*') => {
 656                     self.bump();
 657                     self.bump();
 658                     self.scan_block_comment()
 659                 }
 660                 _ => None,
 661             }
 662         } else if self.ch_is('#') {
 663             if self.nextch_is('!') {
 664
 665                 // Parse an inner attribute.
 666                 if self.nextnextch_is('[') {
 667                     return None;
 668                 }
 669
 670                 // I guess this is the only way to figure out if
 671                 // we're at the beginning of the file...
 672                 let smap = SourceMap::new(FilePathMapping::empty());
 673                 smap.files.borrow_mut().source_files.push(self.source_file.clone());
 674                 let loc = smap.lookup_char_pos_adj(self.pos);
 675                 debug!("Skipping a shebang");
 676                 if loc.line == 1 && loc.col == CharPos(0) {
 677                     // FIXME: Add shebang "token", return it
 678                     let start = self.pos;
 679                     while !self.ch_is('\n') && !self.is_eof() {
 680                         self.bump();
 681                     }
 682                     return Some(TokenAndSpan {
 683                         tok: token::Shebang(self.name_from(start)),
 684                         sp: self.mk_sp(start, self.pos),
 685                     });
 686                 }
 687             }
 688             None
 689         } else {
 690             None
 691         }
 692     }
 693
 694     /// If there is whitespace, shebang, or a comment, scan it. Otherwise,
 695     /// return `None`.
 696     fn scan_whitespace_or_comment(&mut self) -> Option<TokenAndSpan> {
 697         match self.ch.unwrap_or('\0') {
 698             // # to handle shebang at start of file -- this is the entry point
 699             // for skipping over all "junk"
 700             '/' | '#' => {
 701                 let c = self.scan_comment();
 702                 debug!("scanning a comment {:?}", c);
 703                 c
 704             },
 705             c if is_pattern_whitespace(Some(c)) => {
 706                 let start_bpos = self.pos;
 707                 while is_pattern_whitespace(self.ch) {
 708                     self.bump();
 709                 }
 710                 let c = Some(TokenAndSpan {
 711                     tok: token::Whitespace,
 712                     sp: self.mk_sp(start_bpos, self.pos),
 713                 });
 714                 debug!("scanning whitespace: {:?}", c);
 715                 c
 716             }
 717             _ => None,
 718         }
 719     }
 720
 721     /// Might return a sugared-doc-attr
 722     fn scan_block_comment(&mut self) -> Option<TokenAndSpan> {
 723         // block comments starting with "/**" or "/*!" are doc-comments
 724         let is_doc_comment = self.ch_is('*') || self.ch_is('!');
 725         let start_bpos = self.pos - BytePos(2);
 726
 727         let mut level: isize = 1;
 728         let mut has_cr = false;
 729         while level > 0 {
 730             if self.is_eof() {
 731                 let msg = if is_doc_comment {
 732                     "unterminated block doc-comment"
 733                 } else {
 734                     "unterminated block comment"
 735                 };
 736                 let last_bpos = self.pos;
 737                 self.fatal_span_(start_bpos, last_bpos, msg).raise();
 738             }
 739             let n = self.ch.unwrap();
 740             match n {
 741                 '/' if self.nextch_is('*') => {
 742                     level += 1;
 743                     self.bump();
 744                 }
 745                 '*' if self.nextch_is('/') => {
 746                     level -= 1;
 747                     self.bump();
 748                 }
 749                 '\r' => {
 750                     has_cr = true;
 751                 }
 752                 _ => (),
 753             }
 754             self.bump();
 755         }
 756
 757         self.with_str_from(start_bpos, |string| {
 758             // but comments with only "*"s between two "/"s are not
 759             let tok = if is_block_doc_comment(string) {
 760                 let string = if has_cr {
 761                     self.translate_crlf(start_bpos,
 762                                         string,
 763                                         "bare CR not allowed in block doc-comment")
 764                 } else {
 765                     string.into()
 766                 };
 767                 token::DocComment(Symbol::intern(&string[..]))
 768             } else {
 769                 token::Comment
 770             };
 771
 772             Some(TokenAndSpan {
 773                 tok,
 774                 sp: self.mk_sp(start_bpos, self.pos),
 775             })
 776         })
 777     }
 778
 779     /// Scan through any digits (base `scan_radix`) or underscores,
 780     /// and return how many digits there were.
 781     ///
 782     /// `real_radix` represents the true radix of the number we're
 783     /// interested in, and errors will be emitted for any digits
 784     /// between `real_radix` and `scan_radix`.
 785     fn scan_digits(&mut self, real_radix: u32, scan_radix: u32) -> usize {
 786         assert!(real_radix <= scan_radix);
 787         let mut len = 0;
 788
 789         loop {
 790             let c = self.ch;
 791             if c == Some('_') {
 792                 debug!("skipping a _");
 793                 self.bump();
 794                 continue;
 795             }
 796             match c.and_then(|cc| cc.to_digit(scan_radix)) {
 797                 Some(_) => {
 798                     debug!("{:?} in scan_digits", c);
 799                     // check that the hypothetical digit is actually
 800                     // in range for the true radix
 801                     if c.unwrap().to_digit(real_radix).is_none() {
 802                         self.err_span_(self.pos,
 803                                        self.next_pos,
 804                                        &format!("invalid digit for a base {} literal", real_radix));
 805                     }
 806                     len += 1;
 807                     self.bump();
 808                 }
 809                 _ => return len,
 810             }
 811         }
 812     }
 813
 814     /// Lex a LIT_INTEGER or a LIT_FLOAT
 815     fn scan_number(&mut self, c: char) -> token::Lit {
 816         let mut base = 10;
 817         let start_bpos = self.pos;
 818         self.bump();
 819
 820         let num_digits = if c == '0' {
 821             match self.ch.unwrap_or('\0') {
 822                 'b' => {
 823                     self.bump();
 824                     base = 2;
 825                     self.scan_digits(2, 10)
 826                 }
 827                 'o' => {
 828                     self.bump();
 829                     base = 8;
 830                     self.scan_digits(8, 10)
 831                 }
 832                 'x' => {
 833                     self.bump();
 834                     base = 16;
 835                     self.scan_digits(16, 16)
 836                 }
 837                 '0'..='9' | '_' | '.' | 'e' | 'E' => {
 838                     self.scan_digits(10, 10) + 1
 839                 }
 840                 _ => {
 841                     // just a 0
 842                     return token::Integer(self.name_from(start_bpos));
 843                 }
 844             }
 845         } else if c.is_digit(10) {
 846             self.scan_digits(10, 10) + 1
 847         } else {
 848             0
 849         };
 850
 851         if num_digits == 0 {
 852             self.err_span_(start_bpos, self.pos, "no valid digits found for number");
 853
 854             return token::Integer(Symbol::intern("0"));
 855         }
 856
 857         // might be a float, but don't be greedy if this is actually an
 858         // integer literal followed by field/method access or a range pattern
 859         // (`0..2` and `12.foo()`)
 860         if self.ch_is('.') && !self.nextch_is('.') &&
 861            !ident_start(self.nextch()) {
 862             // might have stuff after the ., and if it does, it needs to start
 863             // with a number
 864             self.bump();
 865             if self.ch.unwrap_or('\0').is_digit(10) {
 866                 self.scan_digits(10, 10);
 867                 self.scan_float_exponent();
 868             }
 869             let pos = self.pos;
 870             self.check_float_base(start_bpos, pos, base);
 871
 872             token::Float(self.name_from(start_bpos))
 873         } else {
 874             // it might be a float if it has an exponent
 875             if self.ch_is('e') || self.ch_is('E') {
 876                 self.scan_float_exponent();
 877                 let pos = self.pos;
 878                 self.check_float_base(start_bpos, pos, base);
 879                 return token::Float(self.name_from(start_bpos));
 880             }
 881             // but we certainly have an integer!
 882             token::Integer(self.name_from(start_bpos))
 883         }
 884     }
 885
 886     /// Scan over `n_digits` hex digits, stopping at `delim`, reporting an
 887     /// error if too many or too few digits are encountered.
 888     fn scan_hex_digits(&mut self, n_digits: usize, delim: char, below_0x7f_only: bool) -> bool {
 889         debug!("scanning {} digits until {:?}", n_digits, delim);
 890         let start_bpos = self.pos;
 891         let mut accum_int = 0;
 892
 893         let mut valid = true;
 894         for _ in 0..n_digits {
 895             if self.is_eof() {
 896                 let last_bpos = self.pos;
 897                 self.fatal_span_(start_bpos,
 898                                  last_bpos,
 899                                  "unterminated numeric character escape").raise();
 900             }
 901             if self.ch_is(delim) {
 902                 let last_bpos = self.pos;
 903                 self.err_span_(start_bpos,
 904                                last_bpos,
 905                                "numeric character escape is too short");
 906                 valid = false;
 907                 break;
 908             }
 909             let c = self.ch.unwrap_or('\x00');
 910             accum_int *= 16;
 911             accum_int += c.to_digit(16).unwrap_or_else(|| {
 912                 self.err_span_char(self.pos,
 913                                    self.next_pos,
 914                                    "invalid character in numeric character escape",
 915                                    c);
 916
 917                 valid = false;
 918                 0
 919             });
 920             self.bump();
 921         }
 922
 923         if below_0x7f_only && accum_int >= 0x80 {
 924             self.err_span_(start_bpos,
 925                            self.pos,
 926                            "this form of character escape may only be used with characters in \
 927                             the range [\\x00-\\x7f]");
 928             valid = false;
 929         }
 930
 931         match char::from_u32(accum_int) {
 932             Some(_) => valid,
 933             None => {
 934                 let last_bpos = self.pos;
 935                 self.err_span_(start_bpos, last_bpos, "invalid numeric character escape");
 936                 false
 937             }
 938         }
 939     }
 940
 941     /// Scan for a single (possibly escaped) byte or char
 942     /// in a byte, (non-raw) byte string, char, or (non-raw) string literal.
 943     /// `start` is the position of `first_source_char`, which is already consumed.
 944     ///
 945     /// Returns `true` if there was a valid char/byte.
 946     fn scan_char_or_byte(&mut self,
 947                          start: BytePos,
 948                          first_source_char: char,
 949                          ascii_only: bool,
 950                          delim: char)
 951                          -> bool
 952     {
 953         match first_source_char {
 954             '\\' => {
 955                 // '\X' for some X must be a character constant:
 956                 let escaped = self.ch;
 957                 let escaped_pos = self.pos;
 958                 self.bump();
 959                 match escaped {
 960                     None => {}  // EOF here is an error that will be checked later.
 961                     Some(e) => {
 962                         return match e {
 963                             'n' | 'r' | 't' | '\\' | '\'' | '"' | '0' => true,
 964                             'x' => self.scan_byte_escape(delim, !ascii_only),
 965                             'u' => {
 966                                 let valid = if self.ch_is('{') {
 967                                     self.scan_unicode_escape(delim) && !ascii_only
 968                                 } else {
 969                                     let span = self.mk_sp(start, self.pos);
 970                                     let mut suggestion = "\\u{".to_owned();
 971                                     let msg = "incorrect unicode escape sequence";
 972                                     let mut err = self.sess.span_diagnostic.struct_span_err(
 973                                         span,
 974                                         msg,
 975                                     );
 976                                     let mut i = 0;
 977                                     while let (Some(ch), true) = (self.ch, i < 6) {
 978                                         if ch.is_digit(16) {
 979                                             suggestion.push(ch);
 980                                             self.bump();
 981                                             i += 1;
 982                                         } else {
 983                                             break;
 984                                         }
 985                                     }
 986                                     if i != 0 {
 987                                         suggestion.push('}');
 988                                         err.span_suggestion(
 989                                             self.mk_sp(start, self.pos),
 990                                             "format of unicode escape sequences uses braces",
 991                                             suggestion,
 992                                             Applicability::MaybeIncorrect,
 993                                         );
 994                                     } else {
 995                                         err.span_label(span, msg);
 996                                         err.help(
 997                                             "format of unicode escape sequences is `\\u{...}`",
 998                                         );
 999                                     }
1000                                     err.emit();
1001                                     false
1002                                 };
1003                                 if ascii_only {
1004                                     self.err_span_(start,
1005                                                    self.pos,
1006                                                    "unicode escape sequences cannot be used as a \
1007                                                     byte or in a byte string");
1008                                 }
1009                                 valid
1010
1011                             }
1012                             '\n' if delim == '"' => {
1013                                 self.consume_whitespace();
1014                                 true
1015                             }
1016                             '\r' if delim == '"' && self.ch_is('\n') => {
1017                                 self.consume_whitespace();
1018                                 true
1019                             }
1020                             c => {
1021                                 let pos = self.pos;
1022                                 let msg = if ascii_only {
1023                                     "unknown byte escape"
1024                                 } else {
1025                                     "unknown character escape"
1026                                 };
1027                                 let mut err = self.struct_err_span_char(escaped_pos, pos, msg, c);
1028                                 err.span_label(self.mk_sp(escaped_pos, pos), msg);
1029                                 if e == '\r' {
1030                                     err.help(
1031                                         "this is an isolated carriage return; consider checking \
1032                                          your editor and version control settings",
1033                                     );
1034                                 }
1035                                 if (e == '{' || e == '}') && !ascii_only {
1036                                     err.help(
1037                                         "if used in a formatting string, curly braces are escaped \
1038                                          with `{{` and `}}`",
1039                                     );
1040                                 }
1041                                 err.emit();
1042                                 false
1043                             }
1044                         }
1045                     }
1046                 }
1047             }
1048             '\t' | '\n' | '\r' | '\'' if delim == '\'' => {
1049                 let pos = self.pos;
1050                 self.err_span_char(start,
1051                                    pos,
1052                                    if ascii_only {
1053                                        "byte constant must be escaped"
1054                                    } else {
1055                                        "character constant must be escaped"
1056                                    },
1057                                    first_source_char);
1058                 return false;
1059             }
1060             '\r' => {
1061                 if self.ch_is('\n') {
1062                     self.bump();
1063                     return true;
1064                 } else {
1065                     self.err_span_(start,
1066                                    self.pos,
1067                                    "bare CR not allowed in string, use \\r instead");
1068                     return false;
1069                 }
1070             }
1071             _ => {
1072                 if ascii_only && first_source_char > '\x7F' {
1073                     let pos = self.pos;
1074                     self.err_span_(start,
1075                                    pos,
1076                                    "byte constant must be ASCII. Use a \\xHH escape for a \
1077                                     non-ASCII byte");
1078                     return false;
1079                 }
1080             }
1081         }
1082         true
1083     }
1084
1085     /// Scan over a `\u{...}` escape
1086     ///
1087     /// At this point, we have already seen the `\` and the `u`, the `{` is the current character.
1088     /// We will read a hex number (with `_` separators), with 1 to 6 actual digits,
1089     /// and pass over the `}`.
1090     fn scan_unicode_escape(&mut self, delim: char) -> bool {
1091         self.bump(); // past the {
1092         let start_bpos = self.pos;
1093         let mut valid = true;
1094
1095         if let Some('_') = self.ch {
1096             // disallow leading `_`
1097             self.err_span_(self.pos,
1098                            self.next_pos,
1099                            "invalid start of unicode escape");
1100             valid = false;
1101         }
1102
1103         let count = self.scan_digits(16, 16);
1104
1105         if count > 6 {
1106             self.err_span_(start_bpos,
1107                            self.pos,
1108                            "overlong unicode escape (must have at most 6 hex digits)");
1109             valid = false;
1110         }
1111
1112         loop {
1113             match self.ch {
1114                 Some('}') => {
1115                     if valid && count == 0 {
1116                         self.err_span_(start_bpos,
1117                                        self.pos,
1118                                        "empty unicode escape (must have at least 1 hex digit)");
1119                         valid = false;
1120                     }
1121                     self.bump(); // past the ending `}`
1122                     break;
1123                 },
1124                 Some(c) => {
1125                     if c == delim {
1126                         self.err_span_(self.pos,
1127                                        self.pos,
1128                                        "unterminated unicode escape (needed a `}`)");
1129                         valid = false;
1130                         break;
1131                     } else if valid {
1132                         self.err_span_char(start_bpos,
1133                                            self.pos,
1134                                            "invalid character in unicode escape",
1135                                            c);
1136                         valid = false;
1137                     }
1138                 },
1139                 None => {
1140                     self.fatal_span_(start_bpos,
1141                                      self.pos,
1142                                      "unterminated unicode escape (found EOF)").raise();
1143                 }
1144             }
1145             self.bump();
1146         }
1147
1148         valid
1149     }
1150
1151     /// Scan over a float exponent.
1152     fn scan_float_exponent(&mut self) {
1153         if self.ch_is('e') || self.ch_is('E') {
1154             self.bump();
1155
1156             if self.ch_is('-') || self.ch_is('+') {
1157                 self.bump();
1158             }
1159
1160             if self.scan_digits(10, 10) == 0 {
1161                 let mut err = self.struct_span_fatal(
1162                     self.pos, self.next_pos,
1163                     "expected at least one digit in exponent"
1164                 );
1165                 if let Some(ch) = self.ch {
1166                     // check for e.g., Unicode minus '−' (Issue #49746)
1167                     if unicode_chars::check_for_substitution(self, ch, &mut err) {
1168                         self.bump();
1169                         self.scan_digits(10, 10);
1170                     }
1171                 }
1172                 err.emit();
1173             }
1174         }
1175     }
1176
1177     /// Checks that a base is valid for a floating literal, emitting a nice
1178     /// error if it isn't.
1179     fn check_float_base(&mut self, start_bpos: BytePos, last_bpos: BytePos, base: usize) {
1180         match base {
1181             16 => {
1182                 self.err_span_(start_bpos,
1183                                last_bpos,
1184                                "hexadecimal float literal is not supported")
1185             }
1186             8 => {
1187                 self.err_span_(start_bpos,
1188                                last_bpos,
1189                                "octal float literal is not supported")
1190             }
1191             2 => {
1192                 self.err_span_(start_bpos,
1193                                last_bpos,
1194                                "binary float literal is not supported")
1195             }
1196             _ => (),
1197         }
1198     }
1199
1200     fn binop(&mut self, op: token::BinOpToken) -> token::Token {
1201         self.bump();
1202         if self.ch_is('=') {
1203             self.bump();
1204             token::BinOpEq(op)
1205         } else {
1206             token::BinOp(op)
1207         }
1208     }
1209
1210     /// Returns the next token from the string, advances the input past that
1211     /// token, and updates the interner
1212     fn next_token_inner(&mut self) -> Result<token::Token, ()> {
1213         let c = self.ch;
1214
1215         if ident_start(c) {
1216             let (is_ident_start, is_raw_ident) =
1217                 match (c.unwrap(), self.nextch(), self.nextnextch()) {
1218                     // r# followed by an identifier starter is a raw identifier.
1219                     // This is an exception to the r# case below.
1220                     ('r', Some('#'), x) if ident_start(x) => (true, true),
1221                     // r as in r" or r#" is part of a raw string literal.
1222                     // b as in b' is part of a byte literal.
1223                     // They are not identifiers, and are handled further down.
1224                     ('r', Some('"'), _) |
1225                     ('r', Some('#'), _) |
1226                     ('b', Some('"'), _) |
1227                     ('b', Some('\''), _) |
1228                     ('b', Some('r'), Some('"')) |
1229                     ('b', Some('r'), Some('#')) => (false, false),
1230                     _ => (true, false),
1231                 };
1232
1233             if is_ident_start {
1234                 let raw_start = self.pos;
1235                 if is_raw_ident {
1236                     // Consume the 'r#' characters.
1237                     self.bump();
1238                     self.bump();
1239                 }
1240
1241                 let start = self.pos;
1242                 self.bump();
1243
1244                 while ident_continue(self.ch) {
1245                     self.bump();
1246                 }
1247
1248                 return Ok(self.with_str_from(start, |string| {
1249                     // FIXME: perform NFKC normalization here. (Issue #2253)
1250                     let ident = self.mk_ident(string);
1251
1252                     if is_raw_ident {
1253                         let span = self.mk_sp(raw_start, self.pos);
1254                         if !ident.can_be_raw() {
1255                             self.err_span(span, &format!("`{}` cannot be a raw identifier", ident));
1256                         }
1257                         self.sess.raw_identifier_spans.borrow_mut().push(span);
1258                     }
1259
1260                     token::Ident(ident, is_raw_ident)
1261                 }));
1262             }
1263         }
1264
1265         if is_dec_digit(c) {
1266             let num = self.scan_number(c.unwrap());
1267             let suffix = self.scan_optional_raw_name();
1268             debug!("next_token_inner: scanned number {:?}, {:?}", num, suffix);
1269             return Ok(token::Literal(num, suffix));
1270         }
1271
1272         match c.expect("next_token_inner called at EOF") {
1273             // One-byte tokens.
1274             ';' => {
1275                 self.bump();
1276                 Ok(token::Semi)
1277             }
1278             ',' => {
1279                 self.bump();
1280                 Ok(token::Comma)
1281             }
1282             '.' => {
1283                 self.bump();
1284                 if self.ch_is('.') {
1285                     self.bump();
1286                     if self.ch_is('.') {
1287                         self.bump();
1288                         Ok(token::DotDotDot)
1289                     } else if self.ch_is('=') {
1290                         self.bump();
1291                         Ok(token::DotDotEq)
1292                     } else {
1293                         Ok(token::DotDot)
1294                     }
1295                 } else {
1296                     Ok(token::Dot)
1297                 }
1298             }
1299             '(' => {
1300                 self.bump();
1301                 Ok(token::OpenDelim(token::Paren))
1302             }
1303             ')' => {
1304                 self.bump();
1305                 Ok(token::CloseDelim(token::Paren))
1306             }
1307             '{' => {
1308                 self.bump();
1309                 Ok(token::OpenDelim(token::Brace))
1310             }
1311             '}' => {
1312                 self.bump();
1313                 Ok(token::CloseDelim(token::Brace))
1314             }
1315             '[' => {
1316                 self.bump();
1317                 Ok(token::OpenDelim(token::Bracket))
1318             }
1319             ']' => {
1320                 self.bump();
1321                 Ok(token::CloseDelim(token::Bracket))
1322             }
1323             '@' => {
1324                 self.bump();
1325                 Ok(token::At)
1326             }
1327             '#' => {
1328                 self.bump();
1329                 Ok(token::Pound)
1330             }
1331             '~' => {
1332                 self.bump();
1333                 Ok(token::Tilde)
1334             }
1335             '?' => {
1336                 self.bump();
1337                 Ok(token::Question)
1338             }
1339             ':' => {
1340                 self.bump();
1341                 if self.ch_is(':') {
1342                     self.bump();
1343                     Ok(token::ModSep)
1344                 } else {
1345                     Ok(token::Colon)
1346                 }
1347             }
1348
1349             '$' => {
1350                 self.bump();
1351                 Ok(token::Dollar)
1352             }
1353
1354             // Multi-byte tokens.
1355             '=' => {
1356                 self.bump();
1357                 if self.ch_is('=') {
1358                     self.bump();
1359                     Ok(token::EqEq)
1360                 } else if self.ch_is('>') {
1361                     self.bump();
1362                     Ok(token::FatArrow)
1363                 } else {
1364                     Ok(token::Eq)
1365                 }
1366             }
1367             '!' => {
1368                 self.bump();
1369                 if self.ch_is('=') {
1370                     self.bump();
1371                     Ok(token::Ne)
1372                 } else {
1373                     Ok(token::Not)
1374                 }
1375             }
1376             '<' => {
1377                 self.bump();
1378                 match self.ch.unwrap_or('\x00') {
1379                     '=' => {
1380                         self.bump();
1381                         Ok(token::Le)
1382                     }
1383                     '<' => {
1384                         Ok(self.binop(token::Shl))
1385                     }
1386                     '-' => {
1387                         self.bump();
1388                         Ok(token::LArrow)
1389                     }
1390                     _ => {
1391                         Ok(token::Lt)
1392                     }
1393                 }
1394             }
1395             '>' => {
1396                 self.bump();
1397                 match self.ch.unwrap_or('\x00') {
1398                     '=' => {
1399                         self.bump();
1400                         Ok(token::Ge)
1401                     }
1402                     '>' => {
1403                         Ok(self.binop(token::Shr))
1404                     }
1405                     _ => {
1406                         Ok(token::Gt)
1407                     }
1408                 }
1409             }
1410             '\'' => {
1411                 // Either a character constant 'a' OR a lifetime name 'abc
1412                 let start_with_quote = self.pos;
1413                 self.bump();
1414                 let start = self.pos;
1415
1416                 // the eof will be picked up by the final `'` check below
1417                 let c2 = self.ch.unwrap_or('\x00');
1418                 self.bump();
1419
1420                 // If the character is an ident start not followed by another single
1421                 // quote, then this is a lifetime name:
1422                 if (ident_start(Some(c2)) || c2.is_numeric()) && !self.ch_is('\'') {
1423                     while ident_continue(self.ch) {
1424                         self.bump();
1425                     }
1426                     // lifetimes shouldn't end with a single quote
1427                     // if we find one, then this is an invalid character literal
1428                     if self.ch_is('\'') {
1429                         self.err_span_(
1430                             start_with_quote,
1431                             self.next_pos,
1432                             "character literal may only contain one codepoint");
1433                         self.bump();
1434                         return Ok(token::Literal(token::Err(Symbol::intern("??")), None))
1435
1436                     }
1437
1438                     // Include the leading `'` in the real identifier, for macro
1439                     // expansion purposes. See #12512 for the gory details of why
1440                     // this is necessary.
1441                     let ident = self.with_str_from(start, |lifetime_name| {
1442                         self.mk_ident(&format!("'{}", lifetime_name))
1443                     });
1444
1445                     if c2.is_numeric() {
1446                         // this is a recovered lifetime written `'1`, error but accept it
1447                         self.err_span_(
1448                             start_with_quote,
1449                             self.pos,
1450                             "lifetimes cannot start with a number",
1451                         );
1452                     }
1453
1454                     return Ok(token::Lifetime(ident));
1455                 }
1456
1457                 let valid = self.scan_char_or_byte(start, c2, /* ascii_only */ false, '\'');
1458
1459                 if !self.ch_is('\'') {
1460                     let pos = self.pos;
1461
1462                     loop {
1463                         self.bump();
1464                         if self.ch_is('\'') {
1465                             let start = self.src_index(start);
1466                             let end = self.src_index(self.pos);
1467                             self.bump();
1468                             let span = self.mk_sp(start_with_quote, self.pos);
1469                             self.sess.span_diagnostic
1470                                 .struct_span_err(span,
1471                                                  "character literal may only contain one codepoint")
1472                                 .span_suggestion(
1473                                     span,
1474                                     "if you meant to write a `str` literal, use double quotes",
1475                                     format!("\"{}\"", &self.src[start..end]),
1476                                     Applicability::MachineApplicable
1477                                 ).emit();
1478                             return Ok(token::Literal(token::Err(Symbol::intern("??")), None))
1479                         }
1480                         if self.ch_is('\n') || self.is_eof() || self.ch_is('/') {
1481                             // Only attempt to infer single line string literals. If we encounter
1482                             // a slash, bail out in order to avoid nonsensical suggestion when
1483                             // involving comments.
1484                             break;
1485                         }
1486                     }
1487
1488                     self.fatal_span_verbose(start_with_quote, pos,
1489                         String::from("character literal may only contain one codepoint")).raise();
1490                 }
1491
1492                 let id = if valid {
1493                     self.name_from(start)
1494                 } else {
1495                     Symbol::intern("0")
1496                 };
1497
1498                 self.bump(); // advance ch past token
1499                 let suffix = self.scan_optional_raw_name();
1500
1501                 Ok(token::Literal(token::Char(id), suffix))
1502             }
1503             'b' => {
1504                 self.bump();
1505                 let lit = match self.ch {
1506                     Some('\'') => self.scan_byte(),
1507                     Some('"') => self.scan_byte_string(),
1508                     Some('r') => self.scan_raw_byte_string(),
1509                     _ => unreachable!(),  // Should have been a token::Ident above.
1510                 };
1511                 let suffix = self.scan_optional_raw_name();
1512
1513                 Ok(token::Literal(lit, suffix))
1514             }
1515             '"' => {
1516                 let start_bpos = self.pos;
1517                 let mut valid = true;
1518                 self.bump();
1519
1520                 while !self.ch_is('"') {
1521                     if self.is_eof() {
1522                         let last_bpos = self.pos;
1523                         self.fatal_span_(start_bpos,
1524                                          last_bpos,
1525                                          "unterminated double quote string").raise();
1526                     }
1527
1528                     let ch_start = self.pos;
1529                     let ch = self.ch.unwrap();
1530                     self.bump();
1531                     valid &= self.scan_char_or_byte(ch_start, ch, /* ascii_only */ false, '"');
1532                 }
1533                 // adjust for the ASCII " at the start of the literal
1534                 let id = if valid {
1535                     self.name_from(start_bpos + BytePos(1))
1536                 } else {
1537                     Symbol::intern("??")
1538                 };
1539                 self.bump();
1540                 let suffix = self.scan_optional_raw_name();
1541
1542                 Ok(token::Literal(token::Str_(id), suffix))
1543             }
1544             'r' => {
1545                 let start_bpos = self.pos;
1546                 self.bump();
1547                 let mut hash_count: u16 = 0;
1548                 while self.ch_is('#') {
1549                     if hash_count == 65535 {
1550                         let bpos = self.next_pos;
1551                         self.fatal_span_(start_bpos,
1552                                          bpos,
1553                                          "too many `#` symbols: raw strings may be \
1554                                          delimited by up to 65535 `#` symbols").raise();
1555                     }
1556                     self.bump();
1557                     hash_count += 1;
1558                 }
1559
1560                 if self.is_eof() {
1561                     self.fail_unterminated_raw_string(start_bpos, hash_count);
1562                 } else if !self.ch_is('"') {
1563                     let last_bpos = self.pos;
1564                     let curr_char = self.ch.unwrap();
1565                     self.fatal_span_char(start_bpos,
1566                                          last_bpos,
1567                                          "found invalid character; only `#` is allowed \
1568                                          in raw string delimitation",
1569                                          curr_char).raise();
1570                 }
1571                 self.bump();
1572                 let content_start_bpos = self.pos;
1573                 let mut content_end_bpos;
1574                 let mut valid = true;
1575                 'outer: loop {
1576                     if self.is_eof() {
1577                         self.fail_unterminated_raw_string(start_bpos, hash_count);
1578                     }
1579                     // if self.ch_is('"') {
1580                     // content_end_bpos = self.pos;
1581                     // for _ in 0..hash_count {
1582                     // self.bump();
1583                     // if !self.ch_is('#') {
1584                     // continue 'outer;
1585                     let c = self.ch.unwrap();
1586                     match c {
1587                         '"' => {
1588                             content_end_bpos = self.pos;
1589                             for _ in 0..hash_count {
1590                                 self.bump();
1591                                 if !self.ch_is('#') {
1592                                     continue 'outer;
1593                                 }
1594                             }
1595                             break;
1596                         }
1597                         '\r' => {
1598                             if !self.nextch_is('\n') {
1599                                 let last_bpos = self.pos;
1600                                 self.err_span_(start_bpos,
1601                                                last_bpos,
1602                                                "bare CR not allowed in raw string, use \\r \
1603                                                 instead");
1604                                 valid = false;
1605                             }
1606                         }
1607                         _ => (),
1608                     }
1609                     self.bump();
1610                 }
1611
1612                 self.bump();
1613                 let id = if valid {
1614                     self.name_from_to(content_start_bpos, content_end_bpos)
1615                 } else {
1616                     Symbol::intern("??")
1617                 };
1618                 let suffix = self.scan_optional_raw_name();
1619
1620                 Ok(token::Literal(token::StrRaw(id, hash_count), suffix))
1621             }
1622             '-' => {
1623                 if self.nextch_is('>') {
1624                     self.bump();
1625                     self.bump();
1626                     Ok(token::RArrow)
1627                 } else {
1628                     Ok(self.binop(token::Minus))
1629                 }
1630             }
1631             '&' => {
1632                 if self.nextch_is('&') {
1633                     self.bump();
1634                     self.bump();
1635                     Ok(token::AndAnd)
1636                 } else {
1637                     Ok(self.binop(token::And))
1638                 }
1639             }
1640             '|' => {
1641                 match self.nextch() {
1642                     Some('|') => {
1643                         self.bump();
1644                         self.bump();
1645                         Ok(token::OrOr)
1646                     }
1647                     _ => {
1648                         Ok(self.binop(token::Or))
1649                     }
1650                 }
1651             }
1652             '+' => {
1653                 Ok(self.binop(token::Plus))
1654             }
1655             '*' => {
1656                 Ok(self.binop(token::Star))
1657             }
1658             '/' => {
1659                 Ok(self.binop(token::Slash))
1660             }
1661             '^' => {
1662                 Ok(self.binop(token::Caret))
1663             }
1664             '%' => {
1665                 Ok(self.binop(token::Percent))
1666             }
1667             c => {
1668                 let last_bpos = self.pos;
1669                 let bpos = self.next_pos;
1670                 let mut err = self.struct_fatal_span_char(last_bpos,
1671                                                           bpos,
1672                                                           "unknown start of token",
1673                                                           c);
1674                 unicode_chars::check_for_substitution(self, c, &mut err);
1675                 self.fatal_errs.push(err);
1676
1677                 Err(())
1678             }
1679         }
1680     }
1681
1682     fn consume_whitespace(&mut self) {
1683         while is_pattern_whitespace(self.ch) && !self.is_eof() {
1684             self.bump();
1685         }
1686     }
1687
1688     fn read_to_eol(&mut self) -> String {
1689         let mut val = String::new();
1690         while !self.ch_is('\n') && !self.is_eof() {
1691             val.push(self.ch.unwrap());
1692             self.bump();
1693         }
1694
1695         if self.ch_is('\n') {
1696             self.bump();
1697         }
1698
1699         val
1700     }
1701
1702     fn read_one_line_comment(&mut self) -> String {
1703         let val = self.read_to_eol();
1704         assert!((val.as_bytes()[0] == b'/' && val.as_bytes()[1] == b'/') ||
1705                 (val.as_bytes()[0] == b'#' && val.as_bytes()[1] == b'!'));
1706         val
1707     }
1708
1709     fn consume_non_eol_whitespace(&mut self) {
1710         while is_pattern_whitespace(self.ch) && !self.ch_is('\n') && !self.is_eof() {
1711             self.bump();
1712         }
1713     }
1714
1715     fn peeking_at_comment(&self) -> bool {
1716         (self.ch_is('/') && self.nextch_is('/')) || (self.ch_is('/') && self.nextch_is('*')) ||
1717         // consider shebangs comments, but not inner attributes
1718         (self.ch_is('#') && self.nextch_is('!') && !self.nextnextch_is('['))
1719     }
1720
1721     fn scan_byte(&mut self) -> token::Lit {
1722         self.bump();
1723         let start = self.pos;
1724
1725         // the eof will be picked up by the final `'` check below
1726         let c2 = self.ch.unwrap_or('\x00');
1727         self.bump();
1728
1729         let valid = self.scan_char_or_byte(start,
1730                                            c2,
1731                                            // ascii_only =
1732                                            true,
1733                                            '\'');
1734         if !self.ch_is('\'') {
1735             // Byte offsetting here is okay because the
1736             // character before position `start` are an
1737             // ascii single quote and ascii 'b'.
1738             let pos = self.pos;
1739             self.fatal_span_verbose(start - BytePos(2),
1740                                     pos,
1741                                     "unterminated byte constant".to_string()).raise();
1742         }
1743
1744         let id = if valid {
1745             self.name_from(start)
1746         } else {
1747             Symbol::intern("?")
1748         };
1749         self.bump(); // advance ch past token
1750
1751         token::Byte(id)
1752     }
1753
1754     #[inline]
1755     fn scan_byte_escape(&mut self, delim: char, below_0x7f_only: bool) -> bool {
1756         self.scan_hex_digits(2, delim, below_0x7f_only)
1757     }
1758
1759     fn scan_byte_string(&mut self) -> token::Lit {
1760         self.bump();
1761         let start = self.pos;
1762         let mut valid = true;
1763
1764         while !self.ch_is('"') {
1765             if self.is_eof() {
1766                 let pos = self.pos;
1767                 self.fatal_span_(start, pos, "unterminated double quote byte string").raise();
1768             }
1769
1770             let ch_start = self.pos;
1771             let ch = self.ch.unwrap();
1772             self.bump();
1773             valid &= self.scan_char_or_byte(ch_start,
1774                                             ch,
1775                                             // ascii_only =
1776                                             true,
1777                                             '"');
1778         }
1779
1780         let id = if valid {
1781             self.name_from(start)
1782         } else {
1783             Symbol::intern("??")
1784         };
1785         self.bump();
1786
1787         token::ByteStr(id)
1788     }
1789
1790     fn scan_raw_byte_string(&mut self) -> token::Lit {
1791         let start_bpos = self.pos;
1792         self.bump();
1793         let mut hash_count = 0;
1794         while self.ch_is('#') {
1795             if hash_count == 65535 {
1796                 let bpos = self.next_pos;
1797                 self.fatal_span_(start_bpos,
1798                                  bpos,
1799                                  "too many `#` symbols: raw byte strings may be \
1800                                  delimited by up to 65535 `#` symbols").raise();
1801             }
1802             self.bump();
1803             hash_count += 1;
1804         }
1805
1806         if self.is_eof() {
1807             self.fail_unterminated_raw_string(start_bpos, hash_count);
1808         } else if !self.ch_is('"') {
1809             let pos = self.pos;
1810             let ch = self.ch.unwrap();
1811             self.fatal_span_char(start_bpos,
1812                                         pos,
1813                                         "found invalid character; only `#` is allowed in raw \
1814                                          string delimitation",
1815                                         ch).raise();
1816         }
1817         self.bump();
1818         let content_start_bpos = self.pos;
1819         let mut content_end_bpos;
1820         'outer: loop {
1821             match self.ch {
1822                 None => {
1823                     self.fail_unterminated_raw_string(start_bpos, hash_count);
1824                 }
1825                 Some('"') => {
1826                     content_end_bpos = self.pos;
1827                     for _ in 0..hash_count {
1828                         self.bump();
1829                         if !self.ch_is('#') {
1830                             continue 'outer;
1831                         }
1832                     }
1833                     break;
1834                 }
1835                 Some(c) => {
1836                     if c > '\x7F' {
1837                         let pos = self.pos;
1838                         self.err_span_char(pos, pos, "raw byte string must be ASCII", c);
1839                     }
1840                 }
1841             }
1842             self.bump();
1843         }
1844
1845         self.bump();
1846
1847         token::ByteStrRaw(self.name_from_to(content_start_bpos, content_end_bpos), hash_count)
1848     }
1849 }
1850
1851 // This tests the character for the unicode property 'PATTERN_WHITE_SPACE' which
1852 // is guaranteed to be forward compatible. http://unicode.org/reports/tr31/#R3
1853 #[inline]
1854 crate fn is_pattern_whitespace(c: Option<char>) -> bool {
1855     c.map_or(false, Pattern_White_Space)
1856 }
1857
1858 #[inline]
1859 fn in_range(c: Option<char>, lo: char, hi: char) -> bool {
1860     c.map_or(false, |c| lo <= c && c <= hi)
1861 }
1862
1863 #[inline]
1864 fn is_dec_digit(c: Option<char>) -> bool {
1865     in_range(c, '0', '9')
1866 }
1867
1868 fn is_doc_comment(s: &str) -> bool {
1869     let res = (s.starts_with("///") && *s.as_bytes().get(3).unwrap_or(&b' ') != b'/') ||
1870               s.starts_with("//!");
1871     debug!("is {:?} a doc comment? {}", s, res);
1872     res
1873 }
1874
1875 fn is_block_doc_comment(s: &str) -> bool {
1876     // Prevent `/**/` from being parsed as a doc comment
1877     let res = ((s.starts_with("/**") && *s.as_bytes().get(3).unwrap_or(&b' ') != b'*') ||
1878                s.starts_with("/*!")) && s.len() >= 5;
1879     debug!("is {:?} a doc comment? {}", s, res);
1880     res
1881 }
1882
1883 /// Determine whether `c` is a valid start for an ident.
1884 fn ident_start(c: Option<char>) -> bool {
1885     let c = match c {
1886         Some(c) => c,
1887         None => return false,
1888     };
1889
1890     (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c > '\x7f' && c.is_xid_start())
1891 }
1892
1893 fn ident_continue(c: Option<char>) -> bool {
1894     let c = match c {
1895         Some(c) => c,
1896         None => return false,
1897     };
1898
1899     (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' ||
1900     (c > '\x7f' && c.is_xid_continue())
1901 }
1902
1903 #[inline]
1904 fn char_at(s: &str, byte: usize) -> char {
1905     s[byte..].chars().next().unwrap()
1906 }
1907
1908 #[cfg(test)]
1909 mod tests {
1910     use super::*;
1911
1912     use crate::ast::{Ident, CrateConfig};
1913     use crate::symbol::Symbol;
1914     use crate::source_map::SourceMap;
1915     use crate::feature_gate::UnstableFeatures;
1916     use crate::parse::token;
1917     use crate::diagnostics::plugin::ErrorMap;
1918     use crate::with_globals;
1919     use std::io;
1920     use std::path::PathBuf;
1921     use syntax_pos::{BytePos, Span, NO_EXPANSION};
1922     use rustc_data_structures::fx::FxHashSet;
1923     use rustc_data_structures::sync::Lock;
1924
1925     fn mk_sess(sm: Lrc<SourceMap>) -> ParseSess {
1926         let emitter = errors::emitter::EmitterWriter::new(Box::new(io::sink()),
1927                                                           Some(sm.clone()),
1928                                                           false,
1929                                                           false);
1930         ParseSess {
1931             span_diagnostic: errors::Handler::with_emitter(true, None, Box::new(emitter)),
1932             unstable_features: UnstableFeatures::from_environment(),
1933             config: CrateConfig::default(),
1934             included_mod_stack: Lock::new(Vec::new()),
1935             source_map: sm,
1936             missing_fragment_specifiers: Lock::new(FxHashSet::default()),
1937             raw_identifier_spans: Lock::new(Vec::new()),
1938             registered_diagnostics: Lock::new(ErrorMap::new()),
1939             buffered_lints: Lock::new(vec![]),
1940         }
1941     }
1942
1943     // open a string reader for the given string
1944     fn setup<'a>(sm: &SourceMap,
1945                  sess: &'a ParseSess,
1946                  teststr: String)
1947                  -> StringReader<'a> {
1948         let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr);
1949         StringReader::new(sess, sf, None)
1950     }
1951
1952     #[test]
1953     fn t1() {
1954         with_globals(|| {
1955             let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
1956             let sh = mk_sess(sm.clone());
1957             let mut string_reader = setup(&sm,
1958                                         &sh,
1959                                         "/* my source file */ fn main() { println!(\"zebra\"); }\n"
1960                                             .to_string());
1961             let id = Ident::from_str("fn");
1962             assert_eq!(string_reader.next_token().tok, token::Comment);
1963             assert_eq!(string_reader.next_token().tok, token::Whitespace);
1964             let tok1 = string_reader.next_token();
1965             let tok2 = TokenAndSpan {
1966                 tok: token::Ident(id, false),
1967                 sp: Span::new(BytePos(21), BytePos(23), NO_EXPANSION),
1968             };
1969             assert_eq!(tok1.tok, tok2.tok);
1970             assert_eq!(tok1.sp, tok2.sp);
1971             assert_eq!(string_reader.next_token().tok, token::Whitespace);
1972             // the 'main' id is already read:
1973             assert_eq!(string_reader.pos.clone(), BytePos(28));
1974             // read another token:
1975             let tok3 = string_reader.next_token();
1976             let tok4 = TokenAndSpan {
1977                 tok: mk_ident("main"),
1978                 sp: Span::new(BytePos(24), BytePos(28), NO_EXPANSION),
1979             };
1980             assert_eq!(tok3.tok, tok4.tok);
1981             assert_eq!(tok3.sp, tok4.sp);
1982             // the lparen is already read:
1983             assert_eq!(string_reader.pos.clone(), BytePos(29))
1984         })
1985     }
1986
1987     // check that the given reader produces the desired stream
1988     // of tokens (stop checking after exhausting the expected vec)
1989     fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<token::Token>) {
1990         for expected_tok in &expected {
1991             assert_eq!(&string_reader.next_token().tok, expected_tok);
1992         }
1993     }
1994
1995     // make the identifier by looking up the string in the interner
1996     fn mk_ident(id: &str) -> token::Token {
1997         token::Token::from_ast_ident(Ident::from_str(id))
1998     }
1999
2000     #[test]
2001     fn doublecolonparsing() {
2002         with_globals(|| {
2003             let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
2004             let sh = mk_sess(sm.clone());
2005             check_tokenization(setup(&sm, &sh, "a b".to_string()),
2006                             vec![mk_ident("a"), token::Whitespace, mk_ident("b")]);
2007         })
2008     }
2009
2010     #[test]
2011     fn dcparsing_2() {
2012         with_globals(|| {
2013             let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
2014             let sh = mk_sess(sm.clone());
2015             check_tokenization(setup(&sm, &sh, "a::b".to_string()),
2016                             vec![mk_ident("a"), token::ModSep, mk_ident("b")]);
2017         })
2018     }
2019
2020     #[test]
2021     fn dcparsing_3() {
2022         with_globals(|| {
2023             let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
2024             let sh = mk_sess(sm.clone());
2025             check_tokenization(setup(&sm, &sh, "a ::b".to_string()),
2026                             vec![mk_ident("a"), token::Whitespace, token::ModSep, mk_ident("b")]);
2027         })
2028     }
2029
2030     #[test]
2031     fn dcparsing_4() {
2032         with_globals(|| {
2033             let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
2034             let sh = mk_sess(sm.clone());
2035             check_tokenization(setup(&sm, &sh, "a:: b".to_string()),
2036                             vec![mk_ident("a"), token::ModSep, token::Whitespace, mk_ident("b")]);
2037         })
2038     }
2039
2040     #[test]
2041     fn character_a() {
2042         with_globals(|| {
2043             let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
2044             let sh = mk_sess(sm.clone());
2045             assert_eq!(setup(&sm, &sh, "'a'".to_string()).next_token().tok,
2046                     token::Literal(token::Char(Symbol::intern("a")), None));
2047         })
2048     }
2049
2050     #[test]
2051     fn character_space() {
2052         with_globals(|| {
2053             let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
2054             let sh = mk_sess(sm.clone());
2055             assert_eq!(setup(&sm, &sh, "' '".to_string()).next_token().tok,
2056                     token::Literal(token::Char(Symbol::intern(" ")), None));
2057         })
2058     }
2059
2060     #[test]
2061     fn character_escaped() {
2062         with_globals(|| {
2063             let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
2064             let sh = mk_sess(sm.clone());
2065             assert_eq!(setup(&sm, &sh, "'\\n'".to_string()).next_token().tok,
2066                     token::Literal(token::Char(Symbol::intern("\\n")), None));
2067         })
2068     }
2069
2070     #[test]
2071     fn lifetime_name() {
2072         with_globals(|| {
2073             let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
2074             let sh = mk_sess(sm.clone());
2075             assert_eq!(setup(&sm, &sh, "'abc".to_string()).next_token().tok,
2076                     token::Lifetime(Ident::from_str("'abc")));
2077         })
2078     }
2079
2080     #[test]
2081     fn raw_string() {
2082         with_globals(|| {
2083             let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
2084             let sh = mk_sess(sm.clone());
2085             assert_eq!(setup(&sm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string())
2086                         .next_token()
2087                         .tok,
2088                     token::Literal(token::StrRaw(Symbol::intern("\"#a\\b\x00c\""), 3), None));
2089         })
2090     }
2091
2092     #[test]
2093     fn literal_suffixes() {
2094         with_globals(|| {
2095             let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
2096             let sh = mk_sess(sm.clone());
2097             macro_rules! test {
2098                 ($input: expr, $tok_type: ident, $tok_contents: expr) => {{
2099                     assert_eq!(setup(&sm, &sh, format!("{}suffix", $input)).next_token().tok,
2100                             token::Literal(token::$tok_type(Symbol::intern($tok_contents)),
2101                                             Some(Symbol::intern("suffix"))));
2102                     // with a whitespace separator:
2103                     assert_eq!(setup(&sm, &sh, format!("{} suffix", $input)).next_token().tok,
2104                             token::Literal(token::$tok_type(Symbol::intern($tok_contents)),
2105                                             None));
2106                 }}
2107             }
2108
2109             test!("'a'", Char, "a");
2110             test!("b'a'", Byte, "a");
2111             test!("\"a\"", Str_, "a");
2112             test!("b\"a\"", ByteStr, "a");
2113             test!("1234", Integer, "1234");
2114             test!("0b101", Integer, "0b101");
2115             test!("0xABC", Integer, "0xABC");
2116             test!("1.0", Float, "1.0");
2117             test!("1.0e10", Float, "1.0e10");
2118
2119             assert_eq!(setup(&sm, &sh, "2us".to_string()).next_token().tok,
2120                     token::Literal(token::Integer(Symbol::intern("2")),
2121                                     Some(Symbol::intern("us"))));
2122             assert_eq!(setup(&sm, &sh, "r###\"raw\"###suffix".to_string()).next_token().tok,
2123                     token::Literal(token::StrRaw(Symbol::intern("raw"), 3),
2124                                     Some(Symbol::intern("suffix"))));
2125             assert_eq!(setup(&sm, &sh, "br###\"raw\"###suffix".to_string()).next_token().tok,
2126                     token::Literal(token::ByteStrRaw(Symbol::intern("raw"), 3),
2127                                     Some(Symbol::intern("suffix"))));
2128         })
2129     }
2130
2131     #[test]
2132     fn line_doc_comments() {
2133         assert!(is_doc_comment("///"));
2134         assert!(is_doc_comment("/// blah"));
2135         assert!(!is_doc_comment("////"));
2136     }
2137
2138     #[test]
2139     fn nested_block_comments() {
2140         with_globals(|| {
2141             let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
2142             let sh = mk_sess(sm.clone());
2143             let mut lexer = setup(&sm, &sh, "/* /* */ */'a'".to_string());
2144             match lexer.next_token().tok {
2145                 token::Comment => {}
2146                 _ => panic!("expected a comment!"),
2147             }
2148             assert_eq!(lexer.next_token().tok,
2149                     token::Literal(token::Char(Symbol::intern("a")), None));
2150         })
2151     }
2152
2153     #[test]
2154     fn crlf_comments() {
2155         with_globals(|| {
2156             let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
2157             let sh = mk_sess(sm.clone());
2158             let mut lexer = setup(&sm, &sh, "// test\r\n/// test\r\n".to_string());
2159             let comment = lexer.next_token();
2160             assert_eq!(comment.tok, token::Comment);
2161             assert_eq!((comment.sp.lo(), comment.sp.hi()), (BytePos(0), BytePos(7)));
2162             assert_eq!(lexer.next_token().tok, token::Whitespace);
2163             assert_eq!(lexer.next_token().tok,
2164                     token::DocComment(Symbol::intern("/// test")));
2165         })
2166     }
2167 }