1 //! Code related to parsing literals.
3 use crate::ast::{self, Lit, LitKind};
4 use crate::token::{self, Token};
6 use rustc_lexer::unescape::{unescape_byte, unescape_char};
7 use rustc_lexer::unescape::{unescape_byte_literal, unescape_literal, Mode};
8 use rustc_span::symbol::{kw, sym, Symbol};
24 /// Converts literal token into a semantic literal.
25 pub fn from_token_lit(lit: token::Lit) -> Result<LitKind, LitError> {
26 let token::Lit { kind, symbol, suffix } = lit;
27 if suffix.is_some() && !kind.may_have_suffix() {
28 return Err(LitError::InvalidSuffix);
33 assert!(symbol.is_bool_lit());
34 LitKind::Bool(symbol == kw::True)
37 return unescape_byte(symbol.as_str())
39 .map_err(|_| LitError::LexerError);
42 return unescape_char(symbol.as_str())
44 .map_err(|_| LitError::LexerError);
47 // There are some valid suffixes for integer and float literals,
48 // so all the handling is done internally.
49 token::Integer => return integer_lit(symbol, suffix),
50 token::Float => return float_lit(symbol, suffix),
53 // If there are no characters requiring special treatment we can
54 // reuse the symbol from the token. Otherwise, we must generate a
55 // new symbol because the string in the LitKind is different to the
56 // string in the token.
57 let s = symbol.as_str();
58 let symbol = if s.contains(&['\\', '\r']) {
59 let mut buf = String::with_capacity(s.len());
60 let mut error = Ok(());
61 // Force-inlining here is aggressive but the closure is
62 // called on every char in the string, so it can be
63 // hot in programs with many long strings.
67 &mut #[inline(always)]
68 |_, unescaped_char| match unescaped_char {
72 error = Err(LitError::LexerError);
82 LitKind::Str(symbol, ast::StrStyle::Cooked)
86 let s = symbol.as_str();
89 let mut buf = String::with_capacity(s.len());
90 let mut error = Ok(());
91 unescape_literal(&s, Mode::RawStr, &mut |_, unescaped_char| {
92 match unescaped_char {
96 error = Err(LitError::LexerError);
106 LitKind::Str(symbol, ast::StrStyle::Raw(n))
109 let s = symbol.as_str();
110 let mut buf = Vec::with_capacity(s.len());
111 let mut error = Ok(());
112 unescape_byte_literal(&s, Mode::ByteStr, &mut |_, unescaped_byte| {
113 match unescaped_byte {
114 Ok(c) => buf.push(c),
117 error = Err(LitError::LexerError);
123 LitKind::ByteStr(buf.into())
125 token::ByteStrRaw(_) => {
126 let s = symbol.as_str();
127 let bytes = if s.contains('\r') {
128 let mut buf = Vec::with_capacity(s.len());
129 let mut error = Ok(());
130 unescape_byte_literal(&s, Mode::RawByteStr, &mut |_, unescaped_byte| {
131 match unescaped_byte {
132 Ok(c) => buf.push(c),
135 error = Err(LitError::LexerError);
143 symbol.to_string().into_bytes()
146 LitKind::ByteStr(bytes.into())
148 token::Err => LitKind::Err,
152 /// Attempts to recover a token from semantic literal.
153 /// This function is used when the original token doesn't exist (e.g. the literal is created
154 /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
155 pub fn to_token_lit(&self) -> token::Lit {
156 let (kind, symbol, suffix) = match *self {
157 LitKind::Str(symbol, ast::StrStyle::Cooked) => {
158 // Don't re-intern unless the escaped string is different.
159 let s = symbol.as_str();
160 let escaped = s.escape_default().to_string();
161 let symbol = if s == escaped { symbol } else { Symbol::intern(&escaped) };
162 (token::Str, symbol, None)
164 LitKind::Str(symbol, ast::StrStyle::Raw(n)) => (token::StrRaw(n), symbol, None),
165 LitKind::ByteStr(ref bytes) => {
166 let string = bytes.escape_ascii().to_string();
167 (token::ByteStr, Symbol::intern(&string), None)
169 LitKind::Byte(byte) => {
170 let string: String = ascii::escape_default(byte).map(Into::<char>::into).collect();
171 (token::Byte, Symbol::intern(&string), None)
173 LitKind::Char(ch) => {
174 let string: String = ch.escape_default().map(Into::<char>::into).collect();
175 (token::Char, Symbol::intern(&string), None)
177 LitKind::Int(n, ty) => {
178 let suffix = match ty {
179 ast::LitIntType::Unsigned(ty) => Some(ty.name()),
180 ast::LitIntType::Signed(ty) => Some(ty.name()),
181 ast::LitIntType::Unsuffixed => None,
183 (token::Integer, sym::integer(n), suffix)
185 LitKind::Float(symbol, ty) => {
186 let suffix = match ty {
187 ast::LitFloatType::Suffixed(ty) => Some(ty.name()),
188 ast::LitFloatType::Unsuffixed => None,
190 (token::Float, symbol, suffix)
192 LitKind::Bool(value) => {
193 let symbol = if value { kw::True } else { kw::False };
194 (token::Bool, symbol, None)
196 // This only shows up in places like `-Zunpretty=hir` output, so we
197 // don't bother to produce something useful.
198 LitKind::Err => (token::Err, Symbol::intern("<bad-literal>"), None),
201 token::Lit::new(kind, symbol, suffix)
206 /// Converts literal token into an AST literal.
207 pub fn from_token_lit(token_lit: token::Lit, span: Span) -> Result<Lit, LitError> {
208 Ok(Lit { token_lit, kind: LitKind::from_token_lit(token_lit)?, span })
211 /// Converts arbitrary token into an AST literal.
213 /// Keep this in sync with `Token::can_begin_literal_or_bool` excluding unary negation.
214 pub fn from_token(token: &Token) -> Result<Lit, LitError> {
215 let lit = match token.uninterpolate().kind {
216 token::Ident(name, false) if name.is_bool_lit() => {
217 token::Lit::new(token::Bool, name, None)
219 token::Literal(lit) => lit,
220 token::Interpolated(ref nt) => {
221 if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt
222 && let ast::ExprKind::Lit(lit) = &expr.kind
224 return Ok(lit.clone());
226 return Err(LitError::NotLiteral);
228 _ => return Err(LitError::NotLiteral),
231 Lit::from_token_lit(lit, token.span)
234 /// Attempts to recover an AST literal from semantic literal.
235 /// This function is used when the original token doesn't exist (e.g. the literal is created
236 /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
237 pub fn from_lit_kind(kind: LitKind, span: Span) -> Lit {
238 Lit { token_lit: kind.to_token_lit(), kind, span }
241 /// Losslessly convert an AST literal into a token.
242 pub fn to_token(&self) -> Token {
243 let kind = match self.token_lit.kind {
244 token::Bool => token::Ident(self.token_lit.symbol, false),
245 _ => token::Literal(self.token_lit),
247 Token::new(kind, self.span)
251 fn strip_underscores(symbol: Symbol) -> Symbol {
252 // Do not allocate a new string unless necessary.
253 let s = symbol.as_str();
255 let mut s = s.to_string();
256 s.retain(|c| c != '_');
257 return Symbol::intern(&s);
262 fn filtered_float_lit(
264 suffix: Option<Symbol>,
266 ) -> Result<LitKind, LitError> {
267 debug!("filtered_float_lit: {:?}, {:?}, {:?}", symbol, suffix, base);
269 return Err(LitError::NonDecimalFloat(base));
272 Some(suf) => LitKind::Float(
274 ast::LitFloatType::Suffixed(match suf {
275 sym::f32 => ast::FloatTy::F32,
276 sym::f64 => ast::FloatTy::F64,
277 _ => return Err(LitError::InvalidFloatSuffix),
280 None => LitKind::Float(symbol, ast::LitFloatType::Unsuffixed),
284 fn float_lit(symbol: Symbol, suffix: Option<Symbol>) -> Result<LitKind, LitError> {
285 debug!("float_lit: {:?}, {:?}", symbol, suffix);
286 filtered_float_lit(strip_underscores(symbol), suffix, 10)
289 fn integer_lit(symbol: Symbol, suffix: Option<Symbol>) -> Result<LitKind, LitError> {
290 debug!("integer_lit: {:?}, {:?}", symbol, suffix);
291 let symbol = strip_underscores(symbol);
292 let s = symbol.as_str();
294 let base = match s.as_bytes() {
295 [b'0', b'x', ..] => 16,
296 [b'0', b'o', ..] => 8,
297 [b'0', b'b', ..] => 2,
301 let ty = match suffix {
302 Some(suf) => match suf {
303 sym::isize => ast::LitIntType::Signed(ast::IntTy::Isize),
304 sym::i8 => ast::LitIntType::Signed(ast::IntTy::I8),
305 sym::i16 => ast::LitIntType::Signed(ast::IntTy::I16),
306 sym::i32 => ast::LitIntType::Signed(ast::IntTy::I32),
307 sym::i64 => ast::LitIntType::Signed(ast::IntTy::I64),
308 sym::i128 => ast::LitIntType::Signed(ast::IntTy::I128),
309 sym::usize => ast::LitIntType::Unsigned(ast::UintTy::Usize),
310 sym::u8 => ast::LitIntType::Unsigned(ast::UintTy::U8),
311 sym::u16 => ast::LitIntType::Unsigned(ast::UintTy::U16),
312 sym::u32 => ast::LitIntType::Unsigned(ast::UintTy::U32),
313 sym::u64 => ast::LitIntType::Unsigned(ast::UintTy::U64),
314 sym::u128 => ast::LitIntType::Unsigned(ast::UintTy::U128),
315 // `1f64` and `2f32` etc. are valid float literals, and
316 // `fxxx` looks more like an invalid float literal than invalid integer literal.
317 _ if suf.as_str().starts_with('f') => return filtered_float_lit(symbol, suffix, base),
318 _ => return Err(LitError::InvalidIntSuffix),
320 _ => ast::LitIntType::Unsuffixed,
323 let s = &s[if base != 10 { 2 } else { 0 }..];
324 u128::from_str_radix(s, base).map(|i| LitKind::Int(i, ty)).map_err(|_| {
325 // Small bases are lexed as if they were base 10, e.g, the string
326 // might be `0b10201`. This will cause the conversion above to fail,
327 // but these kinds of errors are already reported by the lexer.
329 base < 10 && s.chars().any(|c| c.to_digit(10).map_or(false, |d| d >= base));
330 if from_lexer { LitError::LexerError } else { LitError::IntTooLarge }