self.pos,
"unknown start of token",
c);
- if let Some(t) = unicode_chars::check_for_substitution(self, start, c, &mut err) {
- err.emit();
- return Ok(t);
+ // FIXME: the lexer could be used to turn the ASCII version of unicode homoglyphs,
+ // instead of keeping a table in `check_for_substitution`into the token. Ideally,
+ // this should be inside `rustc_lexer`. However, we should first remove compound
+ // tokens like `<<` from `rustc_lexer`, and then add fancier error recovery to it,
+ // as there will be less overall work to do this way.
+ return match unicode_chars::check_for_substitution(self, start, c, &mut err) {
+ Some(token) => {
+ err.emit();
+ Ok(token)
+ }
+ None => Err(err),
}
- return Err(err)
}
};
Ok(kind)
use super::StringReader;
use errors::{Applicability, DiagnosticBuilder};
-use syntax_pos::{BytePos, Pos, Span, NO_EXPANSION};
+use syntax_pos::{BytePos, Pos, Span, NO_EXPANSION, symbol::kw};
use crate::parse::token;
#[rustfmt::skip] // for line breaks
('>', "Fullwidth Greater-Than Sign", '>'),
];
+// FIXME: the lexer could be used to turn the ASCII version of unicode homoglyphs, instead of
+// keeping the substitution token in this table. Ideally, this should be inside `rustc_lexer`.
+// However, we should first remove compound tokens like `<<` from `rustc_lexer`, and then add
+// fancier error recovery to it, as there will be less overall work to do this way.
const ASCII_ARRAY: &[(char, &str, Option<token::TokenKind>)] = &[
(' ', "Space", Some(token::Whitespace)),
- ('_', "Underscore", None),
+ ('_', "Underscore", Some(token::Ident(kw::Underscore, false))),
('-', "Minus/Hyphen", Some(token::BinOp(token::Minus))),
(',', "Comma", Some(token::Comma)),
(';', "Semicolon", Some(token::Semi)),
('!', "Exclamation Mark", Some(token::Not)),
('?', "Question Mark", Some(token::Question)),
('.', "Period", Some(token::Dot)),
- ('\'', "Single Quote", None), // Literals are already lexed by this point, so we can't recover
- ('"', "Quotation Mark", None), // gracefully just by spitting the correct token out.
('(', "Left Parenthesis", Some(token::OpenDelim(token::Paren))),
(')', "Right Parenthesis", Some(token::CloseDelim(token::Paren))),
('[', "Left Square Bracket", Some(token::OpenDelim(token::Bracket))),
('<', "Less-Than Sign", Some(token::Lt)),
('=', "Equals Sign", Some(token::Eq)),
('>', "Greater-Than Sign", Some(token::Gt)),
+ // FIXME: Literals are already lexed by this point, so we can't recover gracefully just by
+ // spitting the correct token out.
+ ('\'', "Single Quote", None),
+ ('"', "Quotation Mark", None),
];
crate fn check_for_substitution<'a>(