1 //! Code related to parsing literals.
3 use crate::ast::{self, Lit, LitKind};
4 use crate::symbol::{kw, sym, Symbol};
5 use crate::token::{self, Token};
6 use crate::tokenstream::TokenTree;
9 use rustc_data_structures::sync::Lrc;
11 use rustc_lexer::unescape::{unescape_char, unescape_byte};
12 use rustc_lexer::unescape::{unescape_str, unescape_byte_str};
13 use rustc_lexer::unescape::{unescape_raw_str, unescape_raw_byte_str};
28 /// Converts literal token into a semantic literal.
29 fn from_lit_token(lit: token::Lit) -> Result<LitKind, LitError> {
30 let token::Lit { kind, symbol, suffix } = lit;
31 if suffix.is_some() && !kind.may_have_suffix() {
32 return Err(LitError::InvalidSuffix);
37 assert!(symbol.is_bool_lit());
38 LitKind::Bool(symbol == kw::True)
40 token::Byte => return unescape_byte(&symbol.as_str())
41 .map(LitKind::Byte).map_err(|_| LitError::LexerError),
42 token::Char => return unescape_char(&symbol.as_str())
43 .map(LitKind::Char).map_err(|_| LitError::LexerError),
45 // There are some valid suffixes for integer and float literals,
46 // so all the handling is done internally.
47 token::Integer => return integer_lit(symbol, suffix),
48 token::Float => return float_lit(symbol, suffix),
51 // If there are no characters requiring special treatment we can
52 // reuse the symbol from the token. Otherwise, we must generate a
53 // new symbol because the string in the LitKind is different to the
54 // string in the token.
55 let s = symbol.as_str();
56 let symbol = if s.contains(&['\\', '\r'][..]) {
57 let mut buf = String::with_capacity(s.len());
58 let mut error = Ok(());
59 unescape_str(&s, &mut |_, unescaped_char| {
60 match unescaped_char {
62 Err(_) => error = Err(LitError::LexerError),
70 LitKind::Str(symbol, ast::StrStyle::Cooked)
74 let s = symbol.as_str();
75 let symbol = if s.contains('\r') {
76 let mut buf = String::with_capacity(s.len());
77 let mut error = Ok(());
78 unescape_raw_str(&s, &mut |_, unescaped_char| {
79 match unescaped_char {
81 Err(_) => error = Err(LitError::LexerError),
90 LitKind::Str(symbol, ast::StrStyle::Raw(n))
93 let s = symbol.as_str();
94 let mut buf = Vec::with_capacity(s.len());
95 let mut error = Ok(());
96 unescape_byte_str(&s, &mut |_, unescaped_byte| {
97 match unescaped_byte {
99 Err(_) => error = Err(LitError::LexerError),
104 LitKind::ByteStr(Lrc::new(buf))
106 token::ByteStrRaw(_) => {
107 let s = symbol.as_str();
108 let bytes = if s.contains('\r') {
109 let mut buf = Vec::with_capacity(s.len());
110 let mut error = Ok(());
111 unescape_raw_byte_str(&s, &mut |_, unescaped_byte| {
112 match unescaped_byte {
113 Ok(c) => buf.push(c),
114 Err(_) => error = Err(LitError::LexerError),
121 symbol.to_string().into_bytes()
124 LitKind::ByteStr(Lrc::new(bytes))
126 token::Err => LitKind::Err(symbol),
130 /// Attempts to recover a token from semantic literal.
131 /// This function is used when the original token doesn't exist (e.g. the literal is created
132 /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
133 pub fn to_lit_token(&self) -> token::Lit {
134 let (kind, symbol, suffix) = match *self {
135 LitKind::Str(symbol, ast::StrStyle::Cooked) => {
136 // Don't re-intern unless the escaped string is different.
137 let s = symbol.as_str();
138 let escaped = s.escape_default().to_string();
139 let symbol = if s == escaped { symbol } else { Symbol::intern(&escaped) };
140 (token::Str, symbol, None)
142 LitKind::Str(symbol, ast::StrStyle::Raw(n)) => {
143 (token::StrRaw(n), symbol, None)
145 LitKind::ByteStr(ref bytes) => {
146 let string = bytes.iter().cloned().flat_map(ascii::escape_default)
147 .map(Into::<char>::into).collect::<String>();
148 (token::ByteStr, Symbol::intern(&string), None)
150 LitKind::Byte(byte) => {
151 let string: String = ascii::escape_default(byte).map(Into::<char>::into).collect();
152 (token::Byte, Symbol::intern(&string), None)
154 LitKind::Char(ch) => {
155 let string: String = ch.escape_default().map(Into::<char>::into).collect();
156 (token::Char, Symbol::intern(&string), None)
158 LitKind::Int(n, ty) => {
159 let suffix = match ty {
160 ast::LitIntType::Unsigned(ty) => Some(ty.name()),
161 ast::LitIntType::Signed(ty) => Some(ty.name()),
162 ast::LitIntType::Unsuffixed => None,
164 (token::Integer, sym::integer(n), suffix)
166 LitKind::Float(symbol, ty) => {
167 let suffix = match ty {
168 ast::LitFloatType::Suffixed(ty) => Some(ty.name()),
169 ast::LitFloatType::Unsuffixed => None,
171 (token::Float, symbol, suffix)
173 LitKind::Bool(value) => {
174 let symbol = if value { kw::True } else { kw::False };
175 (token::Bool, symbol, None)
177 LitKind::Err(symbol) => {
178 (token::Err, symbol, None)
182 token::Lit::new(kind, symbol, suffix)
187 /// Converts literal token into an AST literal.
188 crate fn from_lit_token(token: token::Lit, span: Span) -> Result<Lit, LitError> {
189 Ok(Lit { token, kind: LitKind::from_lit_token(token)?, span })
192 /// Converts arbitrary token into an AST literal.
193 crate fn from_token(token: &Token) -> Result<Lit, LitError> {
194 let lit = match token.kind {
195 token::Ident(name, false) if name.is_bool_lit() =>
196 token::Lit::new(token::Bool, name, None),
197 token::Literal(lit) =>
199 token::Interpolated(ref nt) => {
200 if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt {
201 if let ast::ExprKind::Lit(lit) = &expr.kind {
202 return Ok(lit.clone());
205 return Err(LitError::NotLiteral);
207 _ => return Err(LitError::NotLiteral)
210 Lit::from_lit_token(lit, token.span)
213 /// Attempts to recover an AST literal from semantic literal.
214 /// This function is used when the original token doesn't exist (e.g. the literal is created
215 /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
216 pub fn from_lit_kind(kind: LitKind, span: Span) -> Lit {
217 Lit { token: kind.to_lit_token(), kind, span }
220 /// Losslessly convert an AST literal into a token tree.
221 crate fn token_tree(&self) -> TokenTree {
222 let token = match self.token.kind {
223 token::Bool => token::Ident(self.token.symbol, false),
224 _ => token::Literal(self.token),
226 TokenTree::token(token, self.span)
230 fn strip_underscores(symbol: Symbol) -> Symbol {
231 // Do not allocate a new string unless necessary.
232 let s = symbol.as_str();
234 let mut s = s.to_string();
235 s.retain(|c| c != '_');
236 return Symbol::intern(&s);
241 fn filtered_float_lit(symbol: Symbol, suffix: Option<Symbol>, base: u32)
242 -> Result<LitKind, LitError> {
243 debug!("filtered_float_lit: {:?}, {:?}, {:?}", symbol, suffix, base);
245 return Err(LitError::NonDecimalFloat(base));
248 Some(suf) => LitKind::Float(symbol, ast::LitFloatType::Suffixed(match suf {
249 sym::f32 => ast::FloatTy::F32,
250 sym::f64 => ast::FloatTy::F64,
251 _ => return Err(LitError::InvalidFloatSuffix),
253 None => LitKind::Float(symbol, ast::LitFloatType::Unsuffixed)
257 fn float_lit(symbol: Symbol, suffix: Option<Symbol>) -> Result<LitKind, LitError> {
258 debug!("float_lit: {:?}, {:?}", symbol, suffix);
259 filtered_float_lit(strip_underscores(symbol), suffix, 10)
262 fn integer_lit(symbol: Symbol, suffix: Option<Symbol>) -> Result<LitKind, LitError> {
263 debug!("integer_lit: {:?}, {:?}", symbol, suffix);
264 let symbol = strip_underscores(symbol);
265 let s = symbol.as_str();
267 let base = match s.as_bytes() {
268 [b'0', b'x', ..] => 16,
269 [b'0', b'o', ..] => 8,
270 [b'0', b'b', ..] => 2,
274 let ty = match suffix {
275 Some(suf) => match suf {
276 sym::isize => ast::LitIntType::Signed(ast::IntTy::Isize),
277 sym::i8 => ast::LitIntType::Signed(ast::IntTy::I8),
278 sym::i16 => ast::LitIntType::Signed(ast::IntTy::I16),
279 sym::i32 => ast::LitIntType::Signed(ast::IntTy::I32),
280 sym::i64 => ast::LitIntType::Signed(ast::IntTy::I64),
281 sym::i128 => ast::LitIntType::Signed(ast::IntTy::I128),
282 sym::usize => ast::LitIntType::Unsigned(ast::UintTy::Usize),
283 sym::u8 => ast::LitIntType::Unsigned(ast::UintTy::U8),
284 sym::u16 => ast::LitIntType::Unsigned(ast::UintTy::U16),
285 sym::u32 => ast::LitIntType::Unsigned(ast::UintTy::U32),
286 sym::u64 => ast::LitIntType::Unsigned(ast::UintTy::U64),
287 sym::u128 => ast::LitIntType::Unsigned(ast::UintTy::U128),
288 // `1f64` and `2f32` etc. are valid float literals, and
289 // `fxxx` looks more like an invalid float literal than invalid integer literal.
290 _ if suf.as_str().starts_with('f') => return filtered_float_lit(symbol, suffix, base),
291 _ => return Err(LitError::InvalidIntSuffix),
293 _ => ast::LitIntType::Unsuffixed
296 let s = &s[if base != 10 { 2 } else { 0 } ..];
297 u128::from_str_radix(s, base).map(|i| LitKind::Int(i, ty)).map_err(|_| {
298 // Small bases are lexed as if they were base 10, e.g, the string
299 // might be `0b10201`. This will cause the conversion above to fail,
300 // but these kinds of errors are already reported by the lexer.
302 base < 10 && s.chars().any(|c| c.to_digit(10).map_or(false, |d| d >= base));
303 if from_lexer { LitError::LexerError } else { LitError::IntTooLarge }