crates/ra_syntax/src/ast/tokens.rs

   1 //! There are many AstNodes, but only a few tokens, so we hand-write them here.
   2
   3 use std::convert::{TryFrom, TryInto};
   4
   5 use crate::{
   6     ast::{AstToken, Comment, RawString, String, Whitespace},
   7     TextRange, TextSize,
   8 };
   9
  10 impl Comment {
  11     pub fn kind(&self) -> CommentKind {
  12         kind_by_prefix(self.text())
  13     }
  14
  15     pub fn prefix(&self) -> &'static str {
  16         for (prefix, k) in COMMENT_PREFIX_TO_KIND.iter() {
  17             if *k == self.kind() && self.text().starts_with(prefix) {
  18                 return prefix;
  19             }
  20         }
  21         unreachable!()
  22     }
  23 }
  24
  25 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
  26 pub struct CommentKind {
  27     pub shape: CommentShape,
  28     pub doc: Option<CommentPlacement>,
  29 }
  30
  31 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
  32 pub enum CommentShape {
  33     Line,
  34     Block,
  35 }
  36
  37 impl CommentShape {
  38     pub fn is_line(self) -> bool {
  39         self == CommentShape::Line
  40     }
  41
  42     pub fn is_block(self) -> bool {
  43         self == CommentShape::Block
  44     }
  45 }
  46
  47 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
  48 pub enum CommentPlacement {
  49     Inner,
  50     Outer,
  51 }
  52
  53 const COMMENT_PREFIX_TO_KIND: &[(&str, CommentKind)] = {
  54     use {CommentPlacement::*, CommentShape::*};
  55     &[
  56         ("////", CommentKind { shape: Line, doc: None }),
  57         ("///", CommentKind { shape: Line, doc: Some(Outer) }),
  58         ("//!", CommentKind { shape: Line, doc: Some(Inner) }),
  59         ("/**", CommentKind { shape: Block, doc: Some(Outer) }),
  60         ("/*!", CommentKind { shape: Block, doc: Some(Inner) }),
  61         ("//", CommentKind { shape: Line, doc: None }),
  62         ("/*", CommentKind { shape: Block, doc: None }),
  63     ]
  64 };
  65
  66 fn kind_by_prefix(text: &str) -> CommentKind {
  67     if text == "/**/" {
  68         return CommentKind { shape: CommentShape::Block, doc: None };
  69     }
  70     for (prefix, kind) in COMMENT_PREFIX_TO_KIND.iter() {
  71         if text.starts_with(prefix) {
  72             return *kind;
  73         }
  74     }
  75     panic!("bad comment text: {:?}", text)
  76 }
  77
  78 impl Whitespace {
  79     pub fn spans_multiple_lines(&self) -> bool {
  80         let text = self.text();
  81         text.find('\n').map_or(false, |idx| text[idx + 1..].contains('\n'))
  82     }
  83 }
  84
  85 pub struct QuoteOffsets {
  86     pub quotes: [TextRange; 2],
  87     pub contents: TextRange,
  88 }
  89
  90 impl QuoteOffsets {
  91     fn new(literal: &str) -> Option<QuoteOffsets> {
  92         let left_quote = literal.find('"')?;
  93         let right_quote = literal.rfind('"')?;
  94         if left_quote == right_quote {
  95             // `literal` only contains one quote
  96             return None;
  97         }
  98
  99         let start = TextSize::from(0);
 100         let left_quote = TextSize::try_from(left_quote).unwrap() + TextSize::of('"');
 101         let right_quote = TextSize::try_from(right_quote).unwrap();
 102         let end = TextSize::of(literal);
 103
 104         let res = QuoteOffsets {
 105             quotes: [TextRange::new(start, left_quote), TextRange::new(right_quote, end)],
 106             contents: TextRange::new(left_quote, right_quote),
 107         };
 108         Some(res)
 109     }
 110 }
 111
 112 pub trait HasQuotes: AstToken {
 113     fn quote_offsets(&self) -> Option<QuoteOffsets> {
 114         let text = self.text().as_str();
 115         let offsets = QuoteOffsets::new(text)?;
 116         let o = self.syntax().text_range().start();
 117         let offsets = QuoteOffsets {
 118             quotes: [offsets.quotes[0] + o, offsets.quotes[1] + o],
 119             contents: offsets.contents + o,
 120         };
 121         Some(offsets)
 122     }
 123     fn open_quote_text_range(&self) -> Option<TextRange> {
 124         self.quote_offsets().map(|it| it.quotes[0])
 125     }
 126
 127     fn close_quote_text_range(&self) -> Option<TextRange> {
 128         self.quote_offsets().map(|it| it.quotes[1])
 129     }
 130
 131     fn text_range_between_quotes(&self) -> Option<TextRange> {
 132         self.quote_offsets().map(|it| it.contents)
 133     }
 134 }
 135
 136 impl HasQuotes for String {}
 137 impl HasQuotes for RawString {}
 138
 139 pub trait HasStringValue: HasQuotes {
 140     fn value(&self) -> Option<std::string::String>;
 141 }
 142
 143 impl HasStringValue for String {
 144     fn value(&self) -> Option<std::string::String> {
 145         let text = self.text().as_str();
 146         let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
 147
 148         let mut buf = std::string::String::with_capacity(text.len());
 149         let mut has_error = false;
 150         rustc_lexer::unescape::unescape_str(text, &mut |_, unescaped_char| match unescaped_char {
 151             Ok(c) => buf.push(c),
 152             Err(_) => has_error = true,
 153         });
 154
 155         if has_error {
 156             return None;
 157         }
 158         Some(buf)
 159     }
 160 }
 161
 162 impl HasStringValue for RawString {
 163     fn value(&self) -> Option<std::string::String> {
 164         let text = self.text().as_str();
 165         let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
 166         Some(text.to_string())
 167     }
 168 }
 169
 170 impl RawString {
 171     pub fn map_range_up(&self, range: TextRange) -> Option<TextRange> {
 172         let contents_range = self.text_range_between_quotes()?;
 173         assert!(TextRange::up_to(contents_range.len()).contains_range(range));
 174         Some(range + contents_range.start())
 175     }
 176 }
 177
 178 #[derive(Debug)]
 179 pub enum FormatSpecifier {
 180     Open,
 181     Close,
 182     Integer,
 183     Identifier,
 184     Colon,
 185     Fill,
 186     Align,
 187     Sign,
 188     NumberSign,
 189     Zero,
 190     DollarSign,
 191     Dot,
 192     Asterisk,
 193     QuestionMark,
 194 }
 195
 196 pub trait HasFormatSpecifier: AstToken {
 197     fn char_ranges(
 198         &self,
 199     ) -> Option<Vec<(TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>>;
 200
 201     fn lex_format_specifier<F>(&self, mut callback: F)
 202     where
 203         F: FnMut(TextRange, FormatSpecifier),
 204     {
 205         let char_ranges = if let Some(char_ranges) = self.char_ranges() {
 206             char_ranges
 207         } else {
 208             return;
 209         };
 210         let mut chars = char_ranges.iter().peekable();
 211
 212         while let Some((range, first_char)) = chars.next() {
 213             match first_char {
 214                 Ok('{') => {
 215                     // Format specifier, see syntax at https://doc.rust-lang.org/std/fmt/index.html#syntax
 216                     if let Some((_, Ok('{'))) = chars.peek() {
 217                         // Escaped format specifier, `{{`
 218                         chars.next();
 219                         continue;
 220                     }
 221
 222                     callback(*range, FormatSpecifier::Open);
 223
 224                     // check for integer/identifier
 225                     match chars
 226                         .peek()
 227                         .and_then(|next| next.1.as_ref().ok())
 228                         .copied()
 229                         .unwrap_or_default()
 230                     {
 231                         '0'..='9' => {
 232                             // integer
 233                             read_integer(&mut chars, &mut callback);
 234                         }
 235                         c if c == '_' || c.is_alphabetic() => {
 236                             // identifier
 237                             read_identifier(&mut chars, &mut callback);
 238                         }
 239                         _ => {}
 240                     }
 241
 242                     if let Some((_, Ok(':'))) = chars.peek() {
 243                         skip_char_and_emit(&mut chars, FormatSpecifier::Colon, &mut callback);
 244
 245                         // check for fill/align
 246                         let mut cloned = chars.clone().take(2);
 247                         let first = cloned
 248                             .next()
 249                             .and_then(|next| next.1.as_ref().ok())
 250                             .copied()
 251                             .unwrap_or_default();
 252                         let second = cloned
 253                             .next()
 254                             .and_then(|next| next.1.as_ref().ok())
 255                             .copied()
 256                             .unwrap_or_default();
 257                         match second {
 258                             '<' | '^' | '>' => {
 259                                 // alignment specifier, first char specifies fillment
 260                                 skip_char_and_emit(
 261                                     &mut chars,
 262                                     FormatSpecifier::Fill,
 263                                     &mut callback,
 264                                 );
 265                                 skip_char_and_emit(
 266                                     &mut chars,
 267                                     FormatSpecifier::Align,
 268                                     &mut callback,
 269                                 );
 270                             }
 271                             _ => match first {
 272                                 '<' | '^' | '>' => {
 273                                     skip_char_and_emit(
 274                                         &mut chars,
 275                                         FormatSpecifier::Align,
 276                                         &mut callback,
 277                                     );
 278                                 }
 279                                 _ => {}
 280                             },
 281                         }
 282
 283                         // check for sign
 284                         match chars
 285                             .peek()
 286                             .and_then(|next| next.1.as_ref().ok())
 287                             .copied()
 288                             .unwrap_or_default()
 289                         {
 290                             '+' | '-' => {
 291                                 skip_char_and_emit(
 292                                     &mut chars,
 293                                     FormatSpecifier::Sign,
 294                                     &mut callback,
 295                                 );
 296                             }
 297                             _ => {}
 298                         }
 299
 300                         // check for `#`
 301                         if let Some((_, Ok('#'))) = chars.peek() {
 302                             skip_char_and_emit(
 303                                 &mut chars,
 304                                 FormatSpecifier::NumberSign,
 305                                 &mut callback,
 306                             );
 307                         }
 308
 309                         // check for `0`
 310                         let mut cloned = chars.clone().take(2);
 311                         let first = cloned.next().and_then(|next| next.1.as_ref().ok()).copied();
 312                         let second = cloned.next().and_then(|next| next.1.as_ref().ok()).copied();
 313
 314                         if first == Some('0') && second != Some('$') {
 315                             skip_char_and_emit(&mut chars, FormatSpecifier::Zero, &mut callback);
 316                         }
 317
 318                         // width
 319                         match chars
 320                             .peek()
 321                             .and_then(|next| next.1.as_ref().ok())
 322                             .copied()
 323                             .unwrap_or_default()
 324                         {
 325                             '0'..='9' => {
 326                                 read_integer(&mut chars, &mut callback);
 327                                 if let Some((_, Ok('$'))) = chars.peek() {
 328                                     skip_char_and_emit(
 329                                         &mut chars,
 330                                         FormatSpecifier::DollarSign,
 331                                         &mut callback,
 332                                     );
 333                                 }
 334                             }
 335                             c if c == '_' || c.is_alphabetic() => {
 336                                 read_identifier(&mut chars, &mut callback);
 337                                 if chars.peek().and_then(|next| next.1.as_ref().ok()).copied()
 338                                     != Some('$')
 339                                 {
 340                                     continue;
 341                                 }
 342                                 skip_char_and_emit(
 343                                     &mut chars,
 344                                     FormatSpecifier::DollarSign,
 345                                     &mut callback,
 346                                 );
 347                             }
 348                             _ => {}
 349                         }
 350
 351                         // precision
 352                         if let Some((_, Ok('.'))) = chars.peek() {
 353                             skip_char_and_emit(&mut chars, FormatSpecifier::Dot, &mut callback);
 354
 355                             match chars
 356                                 .peek()
 357                                 .and_then(|next| next.1.as_ref().ok())
 358                                 .copied()
 359                                 .unwrap_or_default()
 360                             {
 361                                 '*' => {
 362                                     skip_char_and_emit(
 363                                         &mut chars,
 364                                         FormatSpecifier::Asterisk,
 365                                         &mut callback,
 366                                     );
 367                                 }
 368                                 '0'..='9' => {
 369                                     read_integer(&mut chars, &mut callback);
 370                                     if let Some((_, Ok('$'))) = chars.peek() {
 371                                         skip_char_and_emit(
 372                                             &mut chars,
 373                                             FormatSpecifier::DollarSign,
 374                                             &mut callback,
 375                                         );
 376                                     }
 377                                 }
 378                                 c if c == '_' || c.is_alphabetic() => {
 379                                     read_identifier(&mut chars, &mut callback);
 380                                     if chars.peek().and_then(|next| next.1.as_ref().ok()).copied()
 381                                         != Some('$')
 382                                     {
 383                                         continue;
 384                                     }
 385                                     skip_char_and_emit(
 386                                         &mut chars,
 387                                         FormatSpecifier::DollarSign,
 388                                         &mut callback,
 389                                     );
 390                                 }
 391                                 _ => {
 392                                     continue;
 393                                 }
 394                             }
 395                         }
 396
 397                         // type
 398                         match chars
 399                             .peek()
 400                             .and_then(|next| next.1.as_ref().ok())
 401                             .copied()
 402                             .unwrap_or_default()
 403                         {
 404                             '?' => {
 405                                 skip_char_and_emit(
 406                                     &mut chars,
 407                                     FormatSpecifier::QuestionMark,
 408                                     &mut callback,
 409                                 );
 410                             }
 411                             c if c == '_' || c.is_alphabetic() => {
 412                                 read_identifier(&mut chars, &mut callback);
 413                             }
 414                             _ => {}
 415                         }
 416                     }
 417
 418                     let mut cloned = chars.clone().take(2);
 419                     let first = cloned.next().and_then(|next| next.1.as_ref().ok()).copied();
 420                     let second = cloned.next().and_then(|next| next.1.as_ref().ok()).copied();
 421                     if first != Some('}') {
 422                         continue;
 423                     }
 424                     if second == Some('}') {
 425                         // Escaped format end specifier, `}}`
 426                         continue;
 427                     }
 428                     skip_char_and_emit(&mut chars, FormatSpecifier::Close, &mut callback);
 429                 }
 430                 _ => {
 431                     while let Some((_, Ok(next_char))) = chars.peek() {
 432                         match next_char {
 433                             '{' => break,
 434                             _ => {}
 435                         }
 436                         chars.next();
 437                     }
 438                 }
 439             };
 440         }
 441
 442         fn skip_char_and_emit<'a, I, F>(
 443             chars: &mut std::iter::Peekable<I>,
 444             emit: FormatSpecifier,
 445             callback: &mut F,
 446         ) where
 447             I: Iterator<Item = &'a (TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>,
 448             F: FnMut(TextRange, FormatSpecifier),
 449         {
 450             let (range, _) = chars.next().unwrap();
 451             callback(*range, emit);
 452         }
 453
 454         fn read_integer<'a, I, F>(chars: &mut std::iter::Peekable<I>, callback: &mut F)
 455         where
 456             I: Iterator<Item = &'a (TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>,
 457             F: FnMut(TextRange, FormatSpecifier),
 458         {
 459             let (mut range, c) = chars.next().unwrap();
 460             assert!(c.as_ref().unwrap().is_ascii_digit());
 461             while let Some((r, Ok(next_char))) = chars.peek() {
 462                 if next_char.is_ascii_digit() {
 463                     chars.next();
 464                     range = range.cover(*r);
 465                 } else {
 466                     break;
 467                 }
 468             }
 469             callback(range, FormatSpecifier::Integer);
 470         }
 471
 472         fn read_identifier<'a, I, F>(chars: &mut std::iter::Peekable<I>, callback: &mut F)
 473         where
 474             I: Iterator<Item = &'a (TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>,
 475             F: FnMut(TextRange, FormatSpecifier),
 476         {
 477             let (mut range, c) = chars.next().unwrap();
 478             assert!(c.as_ref().unwrap().is_alphabetic() || *c.as_ref().unwrap() == '_');
 479             while let Some((r, Ok(next_char))) = chars.peek() {
 480                 if *next_char == '_' || next_char.is_ascii_digit() || next_char.is_alphabetic() {
 481                     chars.next();
 482                     range = range.cover(*r);
 483                 } else {
 484                     break;
 485                 }
 486             }
 487             callback(range, FormatSpecifier::Identifier);
 488         }
 489     }
 490 }
 491
 492 impl HasFormatSpecifier for String {
 493     fn char_ranges(
 494         &self,
 495     ) -> Option<Vec<(TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>> {
 496         let text = self.text().as_str();
 497         let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
 498         let offset = self.text_range_between_quotes()?.start() - self.syntax().text_range().start();
 499
 500         let mut res = Vec::with_capacity(text.len());
 501         rustc_lexer::unescape::unescape_str(text, &mut |range, unescaped_char| {
 502             res.push((
 503                 TextRange::new(range.start.try_into().unwrap(), range.end.try_into().unwrap())
 504                     + offset,
 505                 unescaped_char,
 506             ))
 507         });
 508
 509         Some(res)
 510     }
 511 }
 512
 513 impl HasFormatSpecifier for RawString {
 514     fn char_ranges(
 515         &self,
 516     ) -> Option<Vec<(TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>> {
 517         let text = self.text().as_str();
 518         let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
 519         let offset = self.text_range_between_quotes()?.start() - self.syntax().text_range().start();
 520
 521         let mut res = Vec::with_capacity(text.len());
 522         for (idx, c) in text.char_indices() {
 523             res.push((TextRange::at(idx.try_into().unwrap(), TextSize::of(c)) + offset, Ok(c)));
 524         }
 525         Some(res)
 526     }
 527 }