src/tools/rustfmt/src/comment.rs

   1 // Formatting and tools for comments.
   2
   3 use std::{self, borrow::Cow, iter};
   4
   5 use itertools::{multipeek, MultiPeek};
   6 use rustc_span::Span;
   7
   8 use crate::config::Config;
   9 use crate::rewrite::RewriteContext;
  10 use crate::shape::{Indent, Shape};
  11 use crate::string::{rewrite_string, StringFormat};
  12 use crate::utils::{
  13     count_newlines, first_line_width, last_line_width, trim_left_preserve_layout, unicode_str_width,
  14 };
  15 use crate::{ErrorKind, FormattingError};
  16
  17 fn is_custom_comment(comment: &str) -> bool {
  18     if !comment.starts_with("//") {
  19         false
  20     } else if let Some(c) = comment.chars().nth(2) {
  21         !c.is_alphanumeric() && !c.is_whitespace()
  22     } else {
  23         false
  24     }
  25 }
  26
  27 #[derive(Copy, Clone, PartialEq, Eq)]
  28 pub(crate) enum CommentStyle<'a> {
  29     DoubleSlash,
  30     TripleSlash,
  31     Doc,
  32     SingleBullet,
  33     DoubleBullet,
  34     Exclamation,
  35     Custom(&'a str),
  36 }
  37
  38 fn custom_opener(s: &str) -> &str {
  39     s.lines().next().map_or("", |first_line| {
  40         first_line
  41             .find(' ')
  42             .map_or(first_line, |space_index| &first_line[0..=space_index])
  43     })
  44 }
  45
  46 impl<'a> CommentStyle<'a> {
  47     /// Returns `true` if the commenting style covers a line only.
  48     pub(crate) fn is_line_comment(&self) -> bool {
  49         match *self {
  50             CommentStyle::DoubleSlash
  51             | CommentStyle::TripleSlash
  52             | CommentStyle::Doc
  53             | CommentStyle::Custom(_) => true,
  54             _ => false,
  55         }
  56     }
  57
  58     /// Returns `true` if the commenting style can span over multiple lines.
  59     pub(crate) fn is_block_comment(&self) -> bool {
  60         match *self {
  61             CommentStyle::SingleBullet | CommentStyle::DoubleBullet | CommentStyle::Exclamation => {
  62                 true
  63             }
  64             _ => false,
  65         }
  66     }
  67
  68     /// Returns `true` if the commenting style is for documentation.
  69     pub(crate) fn is_doc_comment(&self) -> bool {
  70         match *self {
  71             CommentStyle::TripleSlash | CommentStyle::Doc => true,
  72             _ => false,
  73         }
  74     }
  75
  76     pub(crate) fn opener(&self) -> &'a str {
  77         match *self {
  78             CommentStyle::DoubleSlash => "// ",
  79             CommentStyle::TripleSlash => "/// ",
  80             CommentStyle::Doc => "//! ",
  81             CommentStyle::SingleBullet => "/* ",
  82             CommentStyle::DoubleBullet => "/** ",
  83             CommentStyle::Exclamation => "/*! ",
  84             CommentStyle::Custom(opener) => opener,
  85         }
  86     }
  87
  88     pub(crate) fn closer(&self) -> &'a str {
  89         match *self {
  90             CommentStyle::DoubleSlash
  91             | CommentStyle::TripleSlash
  92             | CommentStyle::Custom(..)
  93             | CommentStyle::Doc => "",
  94             CommentStyle::SingleBullet | CommentStyle::DoubleBullet | CommentStyle::Exclamation => {
  95                 " */"
  96             }
  97         }
  98     }
  99
 100     pub(crate) fn line_start(&self) -> &'a str {
 101         match *self {
 102             CommentStyle::DoubleSlash => "// ",
 103             CommentStyle::TripleSlash => "/// ",
 104             CommentStyle::Doc => "//! ",
 105             CommentStyle::SingleBullet | CommentStyle::DoubleBullet | CommentStyle::Exclamation => {
 106                 " * "
 107             }
 108             CommentStyle::Custom(opener) => opener,
 109         }
 110     }
 111
 112     pub(crate) fn to_str_tuplet(&self) -> (&'a str, &'a str, &'a str) {
 113         (self.opener(), self.closer(), self.line_start())
 114     }
 115 }
 116
 117 pub(crate) fn comment_style(orig: &str, normalize_comments: bool) -> CommentStyle<'_> {
 118     if !normalize_comments {
 119         if orig.starts_with("/**") && !orig.starts_with("/**/") {
 120             CommentStyle::DoubleBullet
 121         } else if orig.starts_with("/*!") {
 122             CommentStyle::Exclamation
 123         } else if orig.starts_with("/*") {
 124             CommentStyle::SingleBullet
 125         } else if orig.starts_with("///") && orig.chars().nth(3).map_or(true, |c| c != '/') {
 126             CommentStyle::TripleSlash
 127         } else if orig.starts_with("//!") {
 128             CommentStyle::Doc
 129         } else if is_custom_comment(orig) {
 130             CommentStyle::Custom(custom_opener(orig))
 131         } else {
 132             CommentStyle::DoubleSlash
 133         }
 134     } else if (orig.starts_with("///") && orig.chars().nth(3).map_or(true, |c| c != '/'))
 135         || (orig.starts_with("/**") && !orig.starts_with("/**/"))
 136     {
 137         CommentStyle::TripleSlash
 138     } else if orig.starts_with("//!") || orig.starts_with("/*!") {
 139         CommentStyle::Doc
 140     } else if is_custom_comment(orig) {
 141         CommentStyle::Custom(custom_opener(orig))
 142     } else {
 143         CommentStyle::DoubleSlash
 144     }
 145 }
 146
 147 /// Returns true if the last line of the passed string finishes with a block-comment.
 148 pub(crate) fn is_last_comment_block(s: &str) -> bool {
 149     s.trim_end().ends_with("*/")
 150 }
 151
 152 /// Combine `prev_str` and `next_str` into a single `String`. `span` may contain
 153 /// comments between two strings. If there are such comments, then that will be
 154 /// recovered. If `allow_extend` is true and there is no comment between the two
 155 /// strings, then they will be put on a single line as long as doing so does not
 156 /// exceed max width.
 157 pub(crate) fn combine_strs_with_missing_comments(
 158     context: &RewriteContext<'_>,
 159     prev_str: &str,
 160     next_str: &str,
 161     span: Span,
 162     shape: Shape,
 163     allow_extend: bool,
 164 ) -> Option<String> {
 165     trace!(
 166         "combine_strs_with_missing_comments `{}` `{}` {:?} {:?}",
 167         prev_str,
 168         next_str,
 169         span,
 170         shape
 171     );
 172
 173     let mut result =
 174         String::with_capacity(prev_str.len() + next_str.len() + shape.indent.width() + 128);
 175     result.push_str(prev_str);
 176     let mut allow_one_line = !prev_str.contains('\n') && !next_str.contains('\n');
 177     let first_sep = if prev_str.is_empty() || next_str.is_empty() {
 178         ""
 179     } else {
 180         " "
 181     };
 182     let mut one_line_width =
 183         last_line_width(prev_str) + first_line_width(next_str) + first_sep.len();
 184
 185     let config = context.config;
 186     let indent = shape.indent;
 187     let missing_comment = rewrite_missing_comment(span, shape, context)?;
 188
 189     if missing_comment.is_empty() {
 190         if allow_extend && prev_str.len() + first_sep.len() + next_str.len() <= shape.width {
 191             result.push_str(first_sep);
 192         } else if !prev_str.is_empty() {
 193             result.push_str(&indent.to_string_with_newline(config))
 194         }
 195         result.push_str(next_str);
 196         return Some(result);
 197     }
 198
 199     // We have a missing comment between the first expression and the second expression.
 200
 201     // Peek the the original source code and find out whether there is a newline between the first
 202     // expression and the second expression or the missing comment. We will preserve the original
 203     // layout whenever possible.
 204     let original_snippet = context.snippet(span);
 205     let prefer_same_line = if let Some(pos) = original_snippet.find('/') {
 206         !original_snippet[..pos].contains('\n')
 207     } else {
 208         !original_snippet.contains('\n')
 209     };
 210
 211     one_line_width -= first_sep.len();
 212     let first_sep = if prev_str.is_empty() || missing_comment.is_empty() {
 213         Cow::from("")
 214     } else {
 215         let one_line_width = last_line_width(prev_str) + first_line_width(&missing_comment) + 1;
 216         if prefer_same_line && one_line_width <= shape.width {
 217             Cow::from(" ")
 218         } else {
 219             indent.to_string_with_newline(config)
 220         }
 221     };
 222     result.push_str(&first_sep);
 223     result.push_str(&missing_comment);
 224
 225     let second_sep = if missing_comment.is_empty() || next_str.is_empty() {
 226         Cow::from("")
 227     } else if missing_comment.starts_with("//") {
 228         indent.to_string_with_newline(config)
 229     } else {
 230         one_line_width += missing_comment.len() + first_sep.len() + 1;
 231         allow_one_line &= !missing_comment.starts_with("//") && !missing_comment.contains('\n');
 232         if prefer_same_line && allow_one_line && one_line_width <= shape.width {
 233             Cow::from(" ")
 234         } else {
 235             indent.to_string_with_newline(config)
 236         }
 237     };
 238     result.push_str(&second_sep);
 239     result.push_str(next_str);
 240
 241     Some(result)
 242 }
 243
 244 pub(crate) fn rewrite_doc_comment(orig: &str, shape: Shape, config: &Config) -> Option<String> {
 245     identify_comment(orig, false, shape, config, true)
 246 }
 247
 248 pub(crate) fn rewrite_comment(
 249     orig: &str,
 250     block_style: bool,
 251     shape: Shape,
 252     config: &Config,
 253 ) -> Option<String> {
 254     identify_comment(orig, block_style, shape, config, false)
 255 }
 256
 257 fn identify_comment(
 258     orig: &str,
 259     block_style: bool,
 260     shape: Shape,
 261     config: &Config,
 262     is_doc_comment: bool,
 263 ) -> Option<String> {
 264     let style = comment_style(orig, false);
 265
 266     // Computes the byte length of line taking into account a newline if the line is part of a
 267     // paragraph.
 268     fn compute_len(orig: &str, line: &str) -> usize {
 269         if orig.len() > line.len() {
 270             if orig.as_bytes()[line.len()] == b'\r' {
 271                 line.len() + 2
 272             } else {
 273                 line.len() + 1
 274             }
 275         } else {
 276             line.len()
 277         }
 278     }
 279
 280     // Get the first group of line comments having the same commenting style.
 281     //
 282     // Returns a tuple with:
 283     // - a boolean indicating if there is a blank line
 284     // - a number indicating the size of the first group of comments
 285     fn consume_same_line_comments(
 286         style: CommentStyle<'_>,
 287         orig: &str,
 288         line_start: &str,
 289     ) -> (bool, usize) {
 290         let mut first_group_ending = 0;
 291         let mut hbl = false;
 292
 293         for line in orig.lines() {
 294             let trimmed_line = line.trim_start();
 295             if trimmed_line.is_empty() {
 296                 hbl = true;
 297                 break;
 298             } else if trimmed_line.starts_with(line_start)
 299                 || comment_style(trimmed_line, false) == style
 300             {
 301                 first_group_ending += compute_len(&orig[first_group_ending..], line);
 302             } else {
 303                 break;
 304             }
 305         }
 306         (hbl, first_group_ending)
 307     }
 308
 309     let (has_bare_lines, first_group_ending) = match style {
 310         CommentStyle::DoubleSlash | CommentStyle::TripleSlash | CommentStyle::Doc => {
 311             let line_start = style.line_start().trim_start();
 312             consume_same_line_comments(style, orig, line_start)
 313         }
 314         CommentStyle::Custom(opener) => {
 315             let trimmed_opener = opener.trim_end();
 316             consume_same_line_comments(style, orig, trimmed_opener)
 317         }
 318         // for a block comment, search for the closing symbol
 319         CommentStyle::DoubleBullet | CommentStyle::SingleBullet | CommentStyle::Exclamation => {
 320             let closer = style.closer().trim_start();
 321             let mut count = orig.matches(closer).count();
 322             let mut closing_symbol_offset = 0;
 323             let mut hbl = false;
 324             let mut first = true;
 325             for line in orig.lines() {
 326                 closing_symbol_offset += compute_len(&orig[closing_symbol_offset..], line);
 327                 let mut trimmed_line = line.trim_start();
 328                 if !trimmed_line.starts_with('*')
 329                     && !trimmed_line.starts_with("//")
 330                     && !trimmed_line.starts_with("/*")
 331                 {
 332                     hbl = true;
 333                 }
 334
 335                 // Remove opener from consideration when searching for closer
 336                 if first {
 337                     let opener = style.opener().trim_end();
 338                     trimmed_line = &trimmed_line[opener.len()..];
 339                     first = false;
 340                 }
 341                 if trimmed_line.ends_with(closer) {
 342                     count -= 1;
 343                     if count == 0 {
 344                         break;
 345                     }
 346                 }
 347             }
 348             (hbl, closing_symbol_offset)
 349         }
 350     };
 351
 352     let (first_group, rest) = orig.split_at(first_group_ending);
 353     let rewritten_first_group =
 354         if !config.normalize_comments() && has_bare_lines && style.is_block_comment() {
 355             trim_left_preserve_layout(first_group, shape.indent, config)?
 356         } else if !config.normalize_comments()
 357             && !config.wrap_comments()
 358             && !config.format_code_in_doc_comments()
 359         {
 360             light_rewrite_comment(first_group, shape.indent, config, is_doc_comment)
 361         } else {
 362             rewrite_comment_inner(
 363                 first_group,
 364                 block_style,
 365                 style,
 366                 shape,
 367                 config,
 368                 is_doc_comment || style.is_doc_comment(),
 369             )?
 370         };
 371     if rest.is_empty() {
 372         Some(rewritten_first_group)
 373     } else {
 374         identify_comment(
 375             rest.trim_start(),
 376             block_style,
 377             shape,
 378             config,
 379             is_doc_comment,
 380         )
 381         .map(|rest_str| {
 382             format!(
 383                 "{}\n{}{}{}",
 384                 rewritten_first_group,
 385                 // insert back the blank line
 386                 if has_bare_lines && style.is_line_comment() {
 387                     "\n"
 388                 } else {
 389                     ""
 390                 },
 391                 shape.indent.to_string(config),
 392                 rest_str
 393             )
 394         })
 395     }
 396 }
 397
 398 /// Attributes for code blocks in rustdoc.
 399 /// See https://doc.rust-lang.org/rustdoc/print.html#attributes
 400 enum CodeBlockAttribute {
 401     Rust,
 402     Ignore,
 403     Text,
 404     ShouldPanic,
 405     NoRun,
 406     CompileFail,
 407 }
 408
 409 impl CodeBlockAttribute {
 410     fn new(attribute: &str) -> CodeBlockAttribute {
 411         match attribute {
 412             "rust" | "" => CodeBlockAttribute::Rust,
 413             "ignore" => CodeBlockAttribute::Ignore,
 414             "text" => CodeBlockAttribute::Text,
 415             "should_panic" => CodeBlockAttribute::ShouldPanic,
 416             "no_run" => CodeBlockAttribute::NoRun,
 417             "compile_fail" => CodeBlockAttribute::CompileFail,
 418             _ => CodeBlockAttribute::Text,
 419         }
 420     }
 421 }
 422
 423 /// Block that is formatted as an item.
 424 ///
 425 /// An item starts with either a star `*` or a dash `-`. Different level of indentation are
 426 /// handled by shrinking the shape accordingly.
 427 struct ItemizedBlock {
 428     /// the lines that are identified as part of an itemized block
 429     lines: Vec<String>,
 430     /// the number of whitespaces up to the item sigil
 431     indent: usize,
 432     /// the string that marks the start of an item
 433     opener: String,
 434     /// sequence of whitespaces to prefix new lines that are part of the item
 435     line_start: String,
 436 }
 437
 438 impl ItemizedBlock {
 439     /// Returns `true` if the line is formatted as an item
 440     fn is_itemized_line(line: &str) -> bool {
 441         let trimmed = line.trim_start();
 442         trimmed.starts_with("* ") || trimmed.starts_with("- ")
 443     }
 444
 445     /// Creates a new ItemizedBlock described with the given line.
 446     /// The `is_itemized_line` needs to be called first.
 447     fn new(line: &str) -> ItemizedBlock {
 448         let space_to_sigil = line.chars().take_while(|c| c.is_whitespace()).count();
 449         let indent = space_to_sigil + 2;
 450         ItemizedBlock {
 451             lines: vec![line[indent..].to_string()],
 452             indent,
 453             opener: line[..indent].to_string(),
 454             line_start: " ".repeat(indent),
 455         }
 456     }
 457
 458     /// Returns a `StringFormat` used for formatting the content of an item.
 459     fn create_string_format<'a>(&'a self, fmt: &'a StringFormat<'_>) -> StringFormat<'a> {
 460         StringFormat {
 461             opener: "",
 462             closer: "",
 463             line_start: "",
 464             line_end: "",
 465             shape: Shape::legacy(fmt.shape.width.saturating_sub(self.indent), Indent::empty()),
 466             trim_end: true,
 467             config: fmt.config,
 468         }
 469     }
 470
 471     /// Returns `true` if the line is part of the current itemized block.
 472     /// If it is, then it is added to the internal lines list.
 473     fn add_line(&mut self, line: &str) -> bool {
 474         if !ItemizedBlock::is_itemized_line(line)
 475             && self.indent <= line.chars().take_while(|c| c.is_whitespace()).count()
 476         {
 477             self.lines.push(line.to_string());
 478             return true;
 479         }
 480         false
 481     }
 482
 483     /// Returns the block as a string, with each line trimmed at the start.
 484     fn trimmed_block_as_string(&self) -> String {
 485         self.lines
 486             .iter()
 487             .map(|line| format!("{} ", line.trim_start()))
 488             .collect::<String>()
 489     }
 490
 491     /// Returns the block as a string under its original form.
 492     fn original_block_as_string(&self) -> String {
 493         self.lines.join("\n")
 494     }
 495 }
 496
 497 struct CommentRewrite<'a> {
 498     result: String,
 499     code_block_buffer: String,
 500     is_prev_line_multi_line: bool,
 501     code_block_attr: Option<CodeBlockAttribute>,
 502     item_block: Option<ItemizedBlock>,
 503     comment_line_separator: String,
 504     indent_str: String,
 505     max_width: usize,
 506     fmt_indent: Indent,
 507     fmt: StringFormat<'a>,
 508
 509     opener: String,
 510     closer: String,
 511     line_start: String,
 512 }
 513
 514 impl<'a> CommentRewrite<'a> {
 515     fn new(
 516         orig: &'a str,
 517         block_style: bool,
 518         shape: Shape,
 519         config: &'a Config,
 520     ) -> CommentRewrite<'a> {
 521         let (opener, closer, line_start) = if block_style {
 522             CommentStyle::SingleBullet.to_str_tuplet()
 523         } else {
 524             comment_style(orig, config.normalize_comments()).to_str_tuplet()
 525         };
 526
 527         let max_width = shape
 528             .width
 529             .checked_sub(closer.len() + opener.len())
 530             .unwrap_or(1);
 531         let indent_str = shape.indent.to_string_with_newline(config).to_string();
 532
 533         let mut cr = CommentRewrite {
 534             result: String::with_capacity(orig.len() * 2),
 535             code_block_buffer: String::with_capacity(128),
 536             is_prev_line_multi_line: false,
 537             code_block_attr: None,
 538             item_block: None,
 539             comment_line_separator: format!("{}{}", indent_str, line_start),
 540             max_width,
 541             indent_str,
 542             fmt_indent: shape.indent,
 543
 544             fmt: StringFormat {
 545                 opener: "",
 546                 closer: "",
 547                 line_start,
 548                 line_end: "",
 549                 shape: Shape::legacy(max_width, shape.indent),
 550                 trim_end: true,
 551                 config,
 552             },
 553
 554             opener: opener.to_owned(),
 555             closer: closer.to_owned(),
 556             line_start: line_start.to_owned(),
 557         };
 558         cr.result.push_str(opener);
 559         cr
 560     }
 561
 562     fn join_block(s: &str, sep: &str) -> String {
 563         let mut result = String::with_capacity(s.len() + 128);
 564         let mut iter = s.lines().peekable();
 565         while let Some(line) = iter.next() {
 566             result.push_str(line);
 567             result.push_str(match iter.peek() {
 568                 Some(next_line) if next_line.is_empty() => sep.trim_end(),
 569                 Some(..) => &sep,
 570                 None => "",
 571             });
 572         }
 573         result
 574     }
 575
 576     fn finish(mut self) -> String {
 577         if !self.code_block_buffer.is_empty() {
 578             // There is a code block that is not properly enclosed by backticks.
 579             // We will leave them untouched.
 580             self.result.push_str(&self.comment_line_separator);
 581             self.result.push_str(&Self::join_block(
 582                 &trim_custom_comment_prefix(&self.code_block_buffer),
 583                 &self.comment_line_separator,
 584             ));
 585         }
 586
 587         if let Some(ref ib) = self.item_block {
 588             // the last few lines are part of an itemized block
 589             self.fmt.shape = Shape::legacy(self.max_width, self.fmt_indent);
 590             let item_fmt = ib.create_string_format(&self.fmt);
 591             self.result.push_str(&self.comment_line_separator);
 592             self.result.push_str(&ib.opener);
 593             match rewrite_string(
 594                 &ib.trimmed_block_as_string(),
 595                 &item_fmt,
 596                 self.max_width.saturating_sub(ib.indent),
 597             ) {
 598                 Some(s) => self.result.push_str(&Self::join_block(
 599                     &s,
 600                     &format!("{}{}", self.comment_line_separator, ib.line_start),
 601                 )),
 602                 None => self.result.push_str(&Self::join_block(
 603                     &ib.original_block_as_string(),
 604                     &self.comment_line_separator,
 605                 )),
 606             };
 607         }
 608
 609         self.result.push_str(&self.closer);
 610         if self.result.ends_with(&self.opener) && self.opener.ends_with(' ') {
 611             // Trailing space.
 612             self.result.pop();
 613         }
 614
 615         self.result
 616     }
 617
 618     fn handle_line(
 619         &mut self,
 620         orig: &'a str,
 621         i: usize,
 622         line: &'a str,
 623         has_leading_whitespace: bool,
 624     ) -> bool {
 625         let is_last = i == count_newlines(orig);
 626
 627         if let Some(ref mut ib) = self.item_block {
 628             if ib.add_line(&line) {
 629                 return false;
 630             }
 631             self.is_prev_line_multi_line = false;
 632             self.fmt.shape = Shape::legacy(self.max_width, self.fmt_indent);
 633             let item_fmt = ib.create_string_format(&self.fmt);
 634             self.result.push_str(&self.comment_line_separator);
 635             self.result.push_str(&ib.opener);
 636             match rewrite_string(
 637                 &ib.trimmed_block_as_string(),
 638                 &item_fmt,
 639                 self.max_width.saturating_sub(ib.indent),
 640             ) {
 641                 Some(s) => self.result.push_str(&Self::join_block(
 642                     &s,
 643                     &format!("{}{}", self.comment_line_separator, ib.line_start),
 644                 )),
 645                 None => self.result.push_str(&Self::join_block(
 646                     &ib.original_block_as_string(),
 647                     &self.comment_line_separator,
 648                 )),
 649             };
 650         } else if self.code_block_attr.is_some() {
 651             if line.starts_with("```") {
 652                 let code_block = match self.code_block_attr.as_ref().unwrap() {
 653                     CodeBlockAttribute::Ignore | CodeBlockAttribute::Text => {
 654                         trim_custom_comment_prefix(&self.code_block_buffer)
 655                     }
 656                     _ if self.code_block_buffer.is_empty() => String::new(),
 657                     _ => {
 658                         let mut config = self.fmt.config.clone();
 659                         config.set().wrap_comments(false);
 660                         if config.format_code_in_doc_comments() {
 661                             if let Some(s) =
 662                                 crate::format_code_block(&self.code_block_buffer, &config, false)
 663                             {
 664                                 trim_custom_comment_prefix(&s.snippet)
 665                             } else {
 666                                 trim_custom_comment_prefix(&self.code_block_buffer)
 667                             }
 668                         } else {
 669                             trim_custom_comment_prefix(&self.code_block_buffer)
 670                         }
 671                     }
 672                 };
 673                 if !code_block.is_empty() {
 674                     self.result.push_str(&self.comment_line_separator);
 675                     self.result
 676                         .push_str(&Self::join_block(&code_block, &self.comment_line_separator));
 677                 }
 678                 self.code_block_buffer.clear();
 679                 self.result.push_str(&self.comment_line_separator);
 680                 self.result.push_str(line);
 681                 self.code_block_attr = None;
 682             } else {
 683                 self.code_block_buffer
 684                     .push_str(&hide_sharp_behind_comment(line));
 685                 self.code_block_buffer.push('\n');
 686             }
 687             return false;
 688         }
 689
 690         self.code_block_attr = None;
 691         self.item_block = None;
 692         if line.starts_with("```") {
 693             self.code_block_attr = Some(CodeBlockAttribute::new(&line[3..]))
 694         } else if self.fmt.config.wrap_comments() && ItemizedBlock::is_itemized_line(&line) {
 695             let ib = ItemizedBlock::new(&line);
 696             self.item_block = Some(ib);
 697             return false;
 698         }
 699
 700         if self.result == self.opener {
 701             let force_leading_whitespace = &self.opener == "/* " && count_newlines(orig) == 0;
 702             if !has_leading_whitespace && !force_leading_whitespace && self.result.ends_with(' ') {
 703                 self.result.pop();
 704             }
 705             if line.is_empty() {
 706                 return false;
 707             }
 708         } else if self.is_prev_line_multi_line && !line.is_empty() {
 709             self.result.push(' ')
 710         } else if is_last && line.is_empty() {
 711             // trailing blank lines are unwanted
 712             if !self.closer.is_empty() {
 713                 self.result.push_str(&self.indent_str);
 714             }
 715             return true;
 716         } else {
 717             self.result.push_str(&self.comment_line_separator);
 718             if !has_leading_whitespace && self.result.ends_with(' ') {
 719                 self.result.pop();
 720             }
 721         }
 722
 723         if self.fmt.config.wrap_comments()
 724             && unicode_str_width(line) > self.fmt.shape.width
 725             && !has_url(line)
 726         {
 727             match rewrite_string(line, &self.fmt, self.max_width) {
 728                 Some(ref s) => {
 729                     self.is_prev_line_multi_line = s.contains('\n');
 730                     self.result.push_str(s);
 731                 }
 732                 None if self.is_prev_line_multi_line => {
 733                     // We failed to put the current `line` next to the previous `line`.
 734                     // Remove the trailing space, then start rewrite on the next line.
 735                     self.result.pop();
 736                     self.result.push_str(&self.comment_line_separator);
 737                     self.fmt.shape = Shape::legacy(self.max_width, self.fmt_indent);
 738                     match rewrite_string(line, &self.fmt, self.max_width) {
 739                         Some(ref s) => {
 740                             self.is_prev_line_multi_line = s.contains('\n');
 741                             self.result.push_str(s);
 742                         }
 743                         None => {
 744                             self.is_prev_line_multi_line = false;
 745                             self.result.push_str(line);
 746                         }
 747                     }
 748                 }
 749                 None => {
 750                     self.is_prev_line_multi_line = false;
 751                     self.result.push_str(line);
 752                 }
 753             }
 754
 755             self.fmt.shape = if self.is_prev_line_multi_line {
 756                 // 1 = " "
 757                 let offset = 1 + last_line_width(&self.result) - self.line_start.len();
 758                 Shape {
 759                     width: self.max_width.saturating_sub(offset),
 760                     indent: self.fmt_indent,
 761                     offset: self.fmt.shape.offset + offset,
 762                 }
 763             } else {
 764                 Shape::legacy(self.max_width, self.fmt_indent)
 765             };
 766         } else {
 767             if line.is_empty() && self.result.ends_with(' ') && !is_last {
 768                 // Remove space if this is an empty comment or a doc comment.
 769                 self.result.pop();
 770             }
 771             self.result.push_str(line);
 772             self.fmt.shape = Shape::legacy(self.max_width, self.fmt_indent);
 773             self.is_prev_line_multi_line = false;
 774         }
 775
 776         false
 777     }
 778 }
 779
 780 fn rewrite_comment_inner(
 781     orig: &str,
 782     block_style: bool,
 783     style: CommentStyle<'_>,
 784     shape: Shape,
 785     config: &Config,
 786     is_doc_comment: bool,
 787 ) -> Option<String> {
 788     let mut rewriter = CommentRewrite::new(orig, block_style, shape, config);
 789
 790     let line_breaks = count_newlines(orig.trim_end());
 791     let lines = orig
 792         .lines()
 793         .enumerate()
 794         .map(|(i, mut line)| {
 795             line = trim_end_unless_two_whitespaces(line.trim_start(), is_doc_comment);
 796             // Drop old closer.
 797             if i == line_breaks && line.ends_with("*/") && !line.starts_with("//") {
 798                 line = line[..(line.len() - 2)].trim_end();
 799             }
 800
 801             line
 802         })
 803         .map(|s| left_trim_comment_line(s, &style))
 804         .map(|(line, has_leading_whitespace)| {
 805             if orig.starts_with("/*") && line_breaks == 0 {
 806                 (
 807                     line.trim_start(),
 808                     has_leading_whitespace || config.normalize_comments(),
 809                 )
 810             } else {
 811                 (line, has_leading_whitespace || config.normalize_comments())
 812             }
 813         });
 814
 815     for (i, (line, has_leading_whitespace)) in lines.enumerate() {
 816         if rewriter.handle_line(orig, i, line, has_leading_whitespace) {
 817             break;
 818         }
 819     }
 820
 821     Some(rewriter.finish())
 822 }
 823
 824 const RUSTFMT_CUSTOM_COMMENT_PREFIX: &str = "//#### ";
 825
 826 fn hide_sharp_behind_comment(s: &str) -> Cow<'_, str> {
 827     let s_trimmed = s.trim();
 828     if s_trimmed.starts_with("# ") || s_trimmed == "#" {
 829         Cow::from(format!("{}{}", RUSTFMT_CUSTOM_COMMENT_PREFIX, s))
 830     } else {
 831         Cow::from(s)
 832     }
 833 }
 834
 835 fn trim_custom_comment_prefix(s: &str) -> String {
 836     s.lines()
 837         .map(|line| {
 838             let left_trimmed = line.trim_start();
 839             if left_trimmed.starts_with(RUSTFMT_CUSTOM_COMMENT_PREFIX) {
 840                 left_trimmed.trim_start_matches(RUSTFMT_CUSTOM_COMMENT_PREFIX)
 841             } else {
 842                 line
 843             }
 844         })
 845         .collect::<Vec<_>>()
 846         .join("\n")
 847 }
 848
 849 /// Returns `true` if the given string MAY include URLs or alike.
 850 fn has_url(s: &str) -> bool {
 851     // This function may return false positive, but should get its job done in most cases.
 852     s.contains("https://") || s.contains("http://") || s.contains("ftp://") || s.contains("file://")
 853 }
 854
 855 /// Given the span, rewrite the missing comment inside it if available.
 856 /// Note that the given span must only include comments (or leading/trailing whitespaces).
 857 pub(crate) fn rewrite_missing_comment(
 858     span: Span,
 859     shape: Shape,
 860     context: &RewriteContext<'_>,
 861 ) -> Option<String> {
 862     let missing_snippet = context.snippet(span);
 863     let trimmed_snippet = missing_snippet.trim();
 864     // check the span starts with a comment
 865     let pos = trimmed_snippet.find('/');
 866     if !trimmed_snippet.is_empty() && pos.is_some() {
 867         rewrite_comment(trimmed_snippet, false, shape, context.config)
 868     } else {
 869         Some(String::new())
 870     }
 871 }
 872
 873 /// Recover the missing comments in the specified span, if available.
 874 /// The layout of the comments will be preserved as long as it does not break the code
 875 /// and its total width does not exceed the max width.
 876 pub(crate) fn recover_missing_comment_in_span(
 877     span: Span,
 878     shape: Shape,
 879     context: &RewriteContext<'_>,
 880     used_width: usize,
 881 ) -> Option<String> {
 882     let missing_comment = rewrite_missing_comment(span, shape, context)?;
 883     if missing_comment.is_empty() {
 884         Some(String::new())
 885     } else {
 886         let missing_snippet = context.snippet(span);
 887         let pos = missing_snippet.find('/')?;
 888         // 1 = ` `
 889         let total_width = missing_comment.len() + used_width + 1;
 890         let force_new_line_before_comment =
 891             missing_snippet[..pos].contains('\n') || total_width > context.config.max_width();
 892         let sep = if force_new_line_before_comment {
 893             shape.indent.to_string_with_newline(context.config)
 894         } else {
 895             Cow::from(" ")
 896         };
 897         Some(format!("{}{}", sep, missing_comment))
 898     }
 899 }
 900
 901 /// Trim trailing whitespaces unless they consist of two or more whitespaces.
 902 fn trim_end_unless_two_whitespaces(s: &str, is_doc_comment: bool) -> &str {
 903     if is_doc_comment && s.ends_with("  ") {
 904         s
 905     } else {
 906         s.trim_end()
 907     }
 908 }
 909
 910 /// Trims whitespace and aligns to indent, but otherwise does not change comments.
 911 fn light_rewrite_comment(
 912     orig: &str,
 913     offset: Indent,
 914     config: &Config,
 915     is_doc_comment: bool,
 916 ) -> String {
 917     let lines: Vec<&str> = orig
 918         .lines()
 919         .map(|l| {
 920             // This is basically just l.trim(), but in the case that a line starts
 921             // with `*` we want to leave one space before it, so it aligns with the
 922             // `*` in `/*`.
 923             let first_non_whitespace = l.find(|c| !char::is_whitespace(c));
 924             let left_trimmed = if let Some(fnw) = first_non_whitespace {
 925                 if l.as_bytes()[fnw] == b'*' && fnw > 0 {
 926                     &l[fnw - 1..]
 927                 } else {
 928                     &l[fnw..]
 929                 }
 930             } else {
 931                 ""
 932             };
 933             // Preserve markdown's double-space line break syntax in doc comment.
 934             trim_end_unless_two_whitespaces(left_trimmed, is_doc_comment)
 935         })
 936         .collect();
 937     lines.join(&format!("\n{}", offset.to_string(config)))
 938 }
 939
 940 /// Trims comment characters and possibly a single space from the left of a string.
 941 /// Does not trim all whitespace. If a single space is trimmed from the left of the string,
 942 /// this function returns true.
 943 fn left_trim_comment_line<'a>(line: &'a str, style: &CommentStyle<'_>) -> (&'a str, bool) {
 944     if line.starts_with("//! ")
 945         || line.starts_with("/// ")
 946         || line.starts_with("/*! ")
 947         || line.starts_with("/** ")
 948     {
 949         (&line[4..], true)
 950     } else if let CommentStyle::Custom(opener) = *style {
 951         if line.starts_with(opener) {
 952             (&line[opener.len()..], true)
 953         } else {
 954             (&line[opener.trim_end().len()..], false)
 955         }
 956     } else if line.starts_with("/* ")
 957         || line.starts_with("// ")
 958         || line.starts_with("//!")
 959         || line.starts_with("///")
 960         || line.starts_with("** ")
 961         || line.starts_with("/*!")
 962         || (line.starts_with("/**") && !line.starts_with("/**/"))
 963     {
 964         (&line[3..], line.chars().nth(2).unwrap() == ' ')
 965     } else if line.starts_with("/*")
 966         || line.starts_with("* ")
 967         || line.starts_with("//")
 968         || line.starts_with("**")
 969     {
 970         (&line[2..], line.chars().nth(1).unwrap() == ' ')
 971     } else if line.starts_with('*') {
 972         (&line[1..], false)
 973     } else {
 974         (line, line.starts_with(' '))
 975     }
 976 }
 977
 978 pub(crate) trait FindUncommented {
 979     fn find_uncommented(&self, pat: &str) -> Option<usize>;
 980     fn find_last_uncommented(&self, pat: &str) -> Option<usize>;
 981 }
 982
 983 impl FindUncommented for str {
 984     fn find_uncommented(&self, pat: &str) -> Option<usize> {
 985         let mut needle_iter = pat.chars();
 986         for (kind, (i, b)) in CharClasses::new(self.char_indices()) {
 987             match needle_iter.next() {
 988                 None => {
 989                     return Some(i - pat.len());
 990                 }
 991                 Some(c) => match kind {
 992                     FullCodeCharKind::Normal | FullCodeCharKind::InString if b == c => {}
 993                     _ => {
 994                         needle_iter = pat.chars();
 995                     }
 996                 },
 997             }
 998         }
 999
1000         // Handle case where the pattern is a suffix of the search string
1001         match needle_iter.next() {
1002             Some(_) => None,
1003             None => Some(self.len() - pat.len()),
1004         }
1005     }
1006
1007     fn find_last_uncommented(&self, pat: &str) -> Option<usize> {
1008         if let Some(left) = self.find_uncommented(pat) {
1009             let mut result = left;
1010             // add 1 to use find_last_uncommented for &str after pat
1011             while let Some(next) = self[(result + 1)..].find_last_uncommented(pat) {
1012                 result += next + 1;
1013             }
1014             Some(result)
1015         } else {
1016             None
1017         }
1018     }
1019 }
1020
1021 // Returns the first byte position after the first comment. The given string
1022 // is expected to be prefixed by a comment, including delimiters.
1023 // Good: `/* /* inner */ outer */ code();`
1024 // Bad:  `code(); // hello\n world!`
1025 pub(crate) fn find_comment_end(s: &str) -> Option<usize> {
1026     let mut iter = CharClasses::new(s.char_indices());
1027     for (kind, (i, _c)) in &mut iter {
1028         if kind == FullCodeCharKind::Normal || kind == FullCodeCharKind::InString {
1029             return Some(i);
1030         }
1031     }
1032
1033     // Handle case where the comment ends at the end of `s`.
1034     if iter.status == CharClassesStatus::Normal {
1035         Some(s.len())
1036     } else {
1037         None
1038     }
1039 }
1040
1041 /// Returns `true` if text contains any comment.
1042 pub(crate) fn contains_comment(text: &str) -> bool {
1043     CharClasses::new(text.chars()).any(|(kind, _)| kind.is_comment())
1044 }
1045
1046 pub(crate) struct CharClasses<T>
1047 where
1048     T: Iterator,
1049     T::Item: RichChar,
1050 {
1051     base: MultiPeek<T>,
1052     status: CharClassesStatus,
1053 }
1054
1055 pub(crate) trait RichChar {
1056     fn get_char(&self) -> char;
1057 }
1058
1059 impl RichChar for char {
1060     fn get_char(&self) -> char {
1061         *self
1062     }
1063 }
1064
1065 impl RichChar for (usize, char) {
1066     fn get_char(&self) -> char {
1067         self.1
1068     }
1069 }
1070
1071 #[derive(PartialEq, Eq, Debug, Clone, Copy)]
1072 enum CharClassesStatus {
1073     Normal,
1074     /// Character is within a string
1075     LitString,
1076     LitStringEscape,
1077     /// Character is within a raw string
1078     LitRawString(u32),
1079     RawStringPrefix(u32),
1080     RawStringSuffix(u32),
1081     LitChar,
1082     LitCharEscape,
1083     /// Character inside a block comment, with the integer indicating the nesting deepness of the
1084     /// comment
1085     BlockComment(u32),
1086     /// Character inside a block-commented string, with the integer indicating the nesting deepness
1087     /// of the comment
1088     StringInBlockComment(u32),
1089     /// Status when the '/' has been consumed, but not yet the '*', deepness is
1090     /// the new deepness (after the comment opening).
1091     BlockCommentOpening(u32),
1092     /// Status when the '*' has been consumed, but not yet the '/', deepness is
1093     /// the new deepness (after the comment closing).
1094     BlockCommentClosing(u32),
1095     /// Character is within a line comment
1096     LineComment,
1097 }
1098
1099 /// Distinguish between functional part of code and comments
1100 #[derive(PartialEq, Eq, Debug, Clone, Copy)]
1101 pub(crate) enum CodeCharKind {
1102     Normal,
1103     Comment,
1104 }
1105
1106 /// Distinguish between functional part of code and comments,
1107 /// describing opening and closing of comments for ease when chunking
1108 /// code from tagged characters
1109 #[derive(PartialEq, Eq, Debug, Clone, Copy)]
1110 pub(crate) enum FullCodeCharKind {
1111     Normal,
1112     /// The first character of a comment, there is only one for a comment (always '/')
1113     StartComment,
1114     /// Any character inside a comment including the second character of comment
1115     /// marks ("//", "/*")
1116     InComment,
1117     /// Last character of a comment, '\n' for a line comment, '/' for a block comment.
1118     EndComment,
1119     /// Start of a mutlitine string inside a comment
1120     StartStringCommented,
1121     /// End of a mutlitine string inside a comment
1122     EndStringCommented,
1123     /// Inside a commented string
1124     InStringCommented,
1125     /// Start of a mutlitine string
1126     StartString,
1127     /// End of a mutlitine string
1128     EndString,
1129     /// Inside a string.
1130     InString,
1131 }
1132
1133 impl FullCodeCharKind {
1134     pub(crate) fn is_comment(self) -> bool {
1135         match self {
1136             FullCodeCharKind::StartComment
1137             | FullCodeCharKind::InComment
1138             | FullCodeCharKind::EndComment
1139             | FullCodeCharKind::StartStringCommented
1140             | FullCodeCharKind::InStringCommented
1141             | FullCodeCharKind::EndStringCommented => true,
1142             _ => false,
1143         }
1144     }
1145
1146     /// Returns true if the character is inside a comment
1147     pub(crate) fn inside_comment(self) -> bool {
1148         match self {
1149             FullCodeCharKind::InComment
1150             | FullCodeCharKind::StartStringCommented
1151             | FullCodeCharKind::InStringCommented
1152             | FullCodeCharKind::EndStringCommented => true,
1153             _ => false,
1154         }
1155     }
1156
1157     pub(crate) fn is_string(self) -> bool {
1158         self == FullCodeCharKind::InString || self == FullCodeCharKind::StartString
1159     }
1160
1161     /// Returns true if the character is within a commented string
1162     pub(crate) fn is_commented_string(self) -> bool {
1163         self == FullCodeCharKind::InStringCommented
1164             || self == FullCodeCharKind::StartStringCommented
1165     }
1166
1167     fn to_codecharkind(self) -> CodeCharKind {
1168         if self.is_comment() {
1169             CodeCharKind::Comment
1170         } else {
1171             CodeCharKind::Normal
1172         }
1173     }
1174 }
1175
1176 impl<T> CharClasses<T>
1177 where
1178     T: Iterator,
1179     T::Item: RichChar,
1180 {
1181     pub(crate) fn new(base: T) -> CharClasses<T> {
1182         CharClasses {
1183             base: multipeek(base),
1184             status: CharClassesStatus::Normal,
1185         }
1186     }
1187 }
1188
1189 fn is_raw_string_suffix<T>(iter: &mut MultiPeek<T>, count: u32) -> bool
1190 where
1191     T: Iterator,
1192     T::Item: RichChar,
1193 {
1194     for _ in 0..count {
1195         match iter.peek() {
1196             Some(c) if c.get_char() == '#' => continue,
1197             _ => return false,
1198         }
1199     }
1200     true
1201 }
1202
1203 impl<T> Iterator for CharClasses<T>
1204 where
1205     T: Iterator,
1206     T::Item: RichChar,
1207 {
1208     type Item = (FullCodeCharKind, T::Item);
1209
1210     fn next(&mut self) -> Option<(FullCodeCharKind, T::Item)> {
1211         let item = self.base.next()?;
1212         let chr = item.get_char();
1213         let mut char_kind = FullCodeCharKind::Normal;
1214         self.status = match self.status {
1215             CharClassesStatus::LitRawString(sharps) => {
1216                 char_kind = FullCodeCharKind::InString;
1217                 match chr {
1218                     '"' => {
1219                         if sharps == 0 {
1220                             char_kind = FullCodeCharKind::Normal;
1221                             CharClassesStatus::Normal
1222                         } else if is_raw_string_suffix(&mut self.base, sharps) {
1223                             CharClassesStatus::RawStringSuffix(sharps)
1224                         } else {
1225                             CharClassesStatus::LitRawString(sharps)
1226                         }
1227                     }
1228                     _ => CharClassesStatus::LitRawString(sharps),
1229                 }
1230             }
1231             CharClassesStatus::RawStringPrefix(sharps) => {
1232                 char_kind = FullCodeCharKind::InString;
1233                 match chr {
1234                     '#' => CharClassesStatus::RawStringPrefix(sharps + 1),
1235                     '"' => CharClassesStatus::LitRawString(sharps),
1236                     _ => CharClassesStatus::Normal, // Unreachable.
1237                 }
1238             }
1239             CharClassesStatus::RawStringSuffix(sharps) => {
1240                 match chr {
1241                     '#' => {
1242                         if sharps == 1 {
1243                             CharClassesStatus::Normal
1244                         } else {
1245                             char_kind = FullCodeCharKind::InString;
1246                             CharClassesStatus::RawStringSuffix(sharps - 1)
1247                         }
1248                     }
1249                     _ => CharClassesStatus::Normal, // Unreachable
1250                 }
1251             }
1252             CharClassesStatus::LitString => {
1253                 char_kind = FullCodeCharKind::InString;
1254                 match chr {
1255                     '"' => CharClassesStatus::Normal,
1256                     '\\' => CharClassesStatus::LitStringEscape,
1257                     _ => CharClassesStatus::LitString,
1258                 }
1259             }
1260             CharClassesStatus::LitStringEscape => {
1261                 char_kind = FullCodeCharKind::InString;
1262                 CharClassesStatus::LitString
1263             }
1264             CharClassesStatus::LitChar => match chr {
1265                 '\\' => CharClassesStatus::LitCharEscape,
1266                 '\'' => CharClassesStatus::Normal,
1267                 _ => CharClassesStatus::LitChar,
1268             },
1269             CharClassesStatus::LitCharEscape => CharClassesStatus::LitChar,
1270             CharClassesStatus::Normal => match chr {
1271                 'r' => match self.base.peek().map(RichChar::get_char) {
1272                     Some('#') | Some('"') => {
1273                         char_kind = FullCodeCharKind::InString;
1274                         CharClassesStatus::RawStringPrefix(0)
1275                     }
1276                     _ => CharClassesStatus::Normal,
1277                 },
1278                 '"' => {
1279                     char_kind = FullCodeCharKind::InString;
1280                     CharClassesStatus::LitString
1281                 }
1282                 '\'' => {
1283                     // HACK: Work around mut borrow.
1284                     match self.base.peek() {
1285                         Some(next) if next.get_char() == '\\' => {
1286                             self.status = CharClassesStatus::LitChar;
1287                             return Some((char_kind, item));
1288                         }
1289                         _ => (),
1290                     }
1291
1292                     match self.base.peek() {
1293                         Some(next) if next.get_char() == '\'' => CharClassesStatus::LitChar,
1294                         _ => CharClassesStatus::Normal,
1295                     }
1296                 }
1297                 '/' => match self.base.peek() {
1298                     Some(next) if next.get_char() == '*' => {
1299                         self.status = CharClassesStatus::BlockCommentOpening(1);
1300                         return Some((FullCodeCharKind::StartComment, item));
1301                     }
1302                     Some(next) if next.get_char() == '/' => {
1303                         self.status = CharClassesStatus::LineComment;
1304                         return Some((FullCodeCharKind::StartComment, item));
1305                     }
1306                     _ => CharClassesStatus::Normal,
1307                 },
1308                 _ => CharClassesStatus::Normal,
1309             },
1310             CharClassesStatus::StringInBlockComment(deepness) => {
1311                 char_kind = FullCodeCharKind::InStringCommented;
1312                 if chr == '"' {
1313                     CharClassesStatus::BlockComment(deepness)
1314                 } else if chr == '*' && self.base.peek().map(RichChar::get_char) == Some('/') {
1315                     char_kind = FullCodeCharKind::InComment;
1316                     CharClassesStatus::BlockCommentClosing(deepness - 1)
1317                 } else {
1318                     CharClassesStatus::StringInBlockComment(deepness)
1319                 }
1320             }
1321             CharClassesStatus::BlockComment(deepness) => {
1322                 assert_ne!(deepness, 0);
1323                 char_kind = FullCodeCharKind::InComment;
1324                 match self.base.peek() {
1325                     Some(next) if next.get_char() == '/' && chr == '*' => {
1326                         CharClassesStatus::BlockCommentClosing(deepness - 1)
1327                     }
1328                     Some(next) if next.get_char() == '*' && chr == '/' => {
1329                         CharClassesStatus::BlockCommentOpening(deepness + 1)
1330                     }
1331                     _ if chr == '"' => CharClassesStatus::StringInBlockComment(deepness),
1332                     _ => self.status,
1333                 }
1334             }
1335             CharClassesStatus::BlockCommentOpening(deepness) => {
1336                 assert_eq!(chr, '*');
1337                 self.status = CharClassesStatus::BlockComment(deepness);
1338                 return Some((FullCodeCharKind::InComment, item));
1339             }
1340             CharClassesStatus::BlockCommentClosing(deepness) => {
1341                 assert_eq!(chr, '/');
1342                 if deepness == 0 {
1343                     self.status = CharClassesStatus::Normal;
1344                     return Some((FullCodeCharKind::EndComment, item));
1345                 } else {
1346                     self.status = CharClassesStatus::BlockComment(deepness);
1347                     return Some((FullCodeCharKind::InComment, item));
1348                 }
1349             }
1350             CharClassesStatus::LineComment => match chr {
1351                 '\n' => {
1352                     self.status = CharClassesStatus::Normal;
1353                     return Some((FullCodeCharKind::EndComment, item));
1354                 }
1355                 _ => {
1356                     self.status = CharClassesStatus::LineComment;
1357                     return Some((FullCodeCharKind::InComment, item));
1358                 }
1359             },
1360         };
1361         Some((char_kind, item))
1362     }
1363 }
1364
1365 /// An iterator over the lines of a string, paired with the char kind at the
1366 /// end of the line.
1367 pub(crate) struct LineClasses<'a> {
1368     base: iter::Peekable<CharClasses<std::str::Chars<'a>>>,
1369     kind: FullCodeCharKind,
1370 }
1371
1372 impl<'a> LineClasses<'a> {
1373     pub(crate) fn new(s: &'a str) -> Self {
1374         LineClasses {
1375             base: CharClasses::new(s.chars()).peekable(),
1376             kind: FullCodeCharKind::Normal,
1377         }
1378     }
1379 }
1380
1381 impl<'a> Iterator for LineClasses<'a> {
1382     type Item = (FullCodeCharKind, String);
1383
1384     fn next(&mut self) -> Option<Self::Item> {
1385         self.base.peek()?;
1386
1387         let mut line = String::new();
1388
1389         let start_kind = match self.base.peek() {
1390             Some((kind, _)) => *kind,
1391             None => unreachable!(),
1392         };
1393
1394         while let Some((kind, c)) = self.base.next() {
1395             // needed to set the kind of the ending character on the last line
1396             self.kind = kind;
1397             if c == '\n' {
1398                 self.kind = match (start_kind, kind) {
1399                     (FullCodeCharKind::Normal, FullCodeCharKind::InString) => {
1400                         FullCodeCharKind::StartString
1401                     }
1402                     (FullCodeCharKind::InString, FullCodeCharKind::Normal) => {
1403                         FullCodeCharKind::EndString
1404                     }
1405                     (FullCodeCharKind::InComment, FullCodeCharKind::InStringCommented) => {
1406                         FullCodeCharKind::StartStringCommented
1407                     }
1408                     (FullCodeCharKind::InStringCommented, FullCodeCharKind::InComment) => {
1409                         FullCodeCharKind::EndStringCommented
1410                     }
1411                     _ => kind,
1412                 };
1413                 break;
1414             }
1415             line.push(c);
1416         }
1417
1418         // Workaround for CRLF newline.
1419         if line.ends_with('\r') {
1420             line.pop();
1421         }
1422
1423         Some((self.kind, line))
1424     }
1425 }
1426
1427 /// Iterator over functional and commented parts of a string. Any part of a string is either
1428 /// functional code, either *one* block comment, either *one* line comment. Whitespace between
1429 /// comments is functional code. Line comments contain their ending newlines.
1430 struct UngroupedCommentCodeSlices<'a> {
1431     slice: &'a str,
1432     iter: iter::Peekable<CharClasses<std::str::CharIndices<'a>>>,
1433 }
1434
1435 impl<'a> UngroupedCommentCodeSlices<'a> {
1436     fn new(code: &'a str) -> UngroupedCommentCodeSlices<'a> {
1437         UngroupedCommentCodeSlices {
1438             slice: code,
1439             iter: CharClasses::new(code.char_indices()).peekable(),
1440         }
1441     }
1442 }
1443
1444 impl<'a> Iterator for UngroupedCommentCodeSlices<'a> {
1445     type Item = (CodeCharKind, usize, &'a str);
1446
1447     fn next(&mut self) -> Option<Self::Item> {
1448         let (kind, (start_idx, _)) = self.iter.next()?;
1449         match kind {
1450             FullCodeCharKind::Normal | FullCodeCharKind::InString => {
1451                 // Consume all the Normal code
1452                 while let Some(&(char_kind, _)) = self.iter.peek() {
1453                     if char_kind.is_comment() {
1454                         break;
1455                     }
1456                     let _ = self.iter.next();
1457                 }
1458             }
1459             FullCodeCharKind::StartComment => {
1460                 // Consume the whole comment
1461                 loop {
1462                     match self.iter.next() {
1463                         Some((kind, ..)) if kind.inside_comment() => continue,
1464                         _ => break,
1465                     }
1466                 }
1467             }
1468             _ => panic!(),
1469         }
1470         let slice = match self.iter.peek() {
1471             Some(&(_, (end_idx, _))) => &self.slice[start_idx..end_idx],
1472             None => &self.slice[start_idx..],
1473         };
1474         Some((
1475             if kind.is_comment() {
1476                 CodeCharKind::Comment
1477             } else {
1478                 CodeCharKind::Normal
1479             },
1480             start_idx,
1481             slice,
1482         ))
1483     }
1484 }
1485
1486 /// Iterator over an alternating sequence of functional and commented parts of
1487 /// a string. The first item is always a, possibly zero length, subslice of
1488 /// functional text. Line style comments contain their ending newlines.
1489 pub(crate) struct CommentCodeSlices<'a> {
1490     slice: &'a str,
1491     last_slice_kind: CodeCharKind,
1492     last_slice_end: usize,
1493 }
1494
1495 impl<'a> CommentCodeSlices<'a> {
1496     pub(crate) fn new(slice: &'a str) -> CommentCodeSlices<'a> {
1497         CommentCodeSlices {
1498             slice,
1499             last_slice_kind: CodeCharKind::Comment,
1500             last_slice_end: 0,
1501         }
1502     }
1503 }
1504
1505 impl<'a> Iterator for CommentCodeSlices<'a> {
1506     type Item = (CodeCharKind, usize, &'a str);
1507
1508     fn next(&mut self) -> Option<Self::Item> {
1509         if self.last_slice_end == self.slice.len() {
1510             return None;
1511         }
1512
1513         let mut sub_slice_end = self.last_slice_end;
1514         let mut first_whitespace = None;
1515         let subslice = &self.slice[self.last_slice_end..];
1516         let mut iter = CharClasses::new(subslice.char_indices());
1517
1518         for (kind, (i, c)) in &mut iter {
1519             let is_comment_connector = self.last_slice_kind == CodeCharKind::Normal
1520                 && &subslice[..2] == "//"
1521                 && [' ', '\t'].contains(&c);
1522
1523             if is_comment_connector && first_whitespace.is_none() {
1524                 first_whitespace = Some(i);
1525             }
1526
1527             if kind.to_codecharkind() == self.last_slice_kind && !is_comment_connector {
1528                 let last_index = match first_whitespace {
1529                     Some(j) => j,
1530                     None => i,
1531                 };
1532                 sub_slice_end = self.last_slice_end + last_index;
1533                 break;
1534             }
1535
1536             if !is_comment_connector {
1537                 first_whitespace = None;
1538             }
1539         }
1540
1541         if let (None, true) = (iter.next(), sub_slice_end == self.last_slice_end) {
1542             // This was the last subslice.
1543             sub_slice_end = match first_whitespace {
1544                 Some(i) => self.last_slice_end + i,
1545                 None => self.slice.len(),
1546             };
1547         }
1548
1549         let kind = match self.last_slice_kind {
1550             CodeCharKind::Comment => CodeCharKind::Normal,
1551             CodeCharKind::Normal => CodeCharKind::Comment,
1552         };
1553         let res = (
1554             kind,
1555             self.last_slice_end,
1556             &self.slice[self.last_slice_end..sub_slice_end],
1557         );
1558         self.last_slice_end = sub_slice_end;
1559         self.last_slice_kind = kind;
1560
1561         Some(res)
1562     }
1563 }
1564
1565 /// Checks is `new` didn't miss any comment from `span`, if it removed any, return previous text
1566 /// (if it fits in the width/offset, else return `None`), else return `new`
1567 pub(crate) fn recover_comment_removed(
1568     new: String,
1569     span: Span,
1570     context: &RewriteContext<'_>,
1571 ) -> Option<String> {
1572     let snippet = context.snippet(span);
1573     if snippet != new && changed_comment_content(snippet, &new) {
1574         // We missed some comments. Warn and keep the original text.
1575         if context.config.error_on_unformatted() {
1576             context.report.append(
1577                 context.parse_sess.span_to_filename(span),
1578                 vec![FormattingError::from_span(
1579                     span,
1580                     &context.parse_sess,
1581                     ErrorKind::LostComment,
1582                 )],
1583             );
1584         }
1585         Some(snippet.to_owned())
1586     } else {
1587         Some(new)
1588     }
1589 }
1590
1591 pub(crate) fn filter_normal_code(code: &str) -> String {
1592     let mut buffer = String::with_capacity(code.len());
1593     LineClasses::new(code).for_each(|(kind, line)| match kind {
1594         FullCodeCharKind::Normal
1595         | FullCodeCharKind::StartString
1596         | FullCodeCharKind::InString
1597         | FullCodeCharKind::EndString => {
1598             buffer.push_str(&line);
1599             buffer.push('\n');
1600         }
1601         _ => (),
1602     });
1603     if !code.ends_with('\n') && buffer.ends_with('\n') {
1604         buffer.pop();
1605     }
1606     buffer
1607 }
1608
1609 /// Returns `true` if the two strings of code have the same payload of comments.
1610 /// The payload of comments is everything in the string except:
1611 /// - actual code (not comments),
1612 /// - comment start/end marks,
1613 /// - whitespace,
1614 /// - '*' at the beginning of lines in block comments.
1615 fn changed_comment_content(orig: &str, new: &str) -> bool {
1616     // Cannot write this as a fn since we cannot return types containing closures.
1617     let code_comment_content = |code| {
1618         let slices = UngroupedCommentCodeSlices::new(code);
1619         slices
1620             .filter(|&(ref kind, _, _)| *kind == CodeCharKind::Comment)
1621             .flat_map(|(_, _, s)| CommentReducer::new(s))
1622     };
1623     let res = code_comment_content(orig).ne(code_comment_content(new));
1624     debug!(
1625         "comment::changed_comment_content: {}\norig: '{}'\nnew: '{}'\nraw_old: {}\nraw_new: {}",
1626         res,
1627         orig,
1628         new,
1629         code_comment_content(orig).collect::<String>(),
1630         code_comment_content(new).collect::<String>()
1631     );
1632     res
1633 }
1634
1635 /// Iterator over the 'payload' characters of a comment.
1636 /// It skips whitespace, comment start/end marks, and '*' at the beginning of lines.
1637 /// The comment must be one comment, ie not more than one start mark (no multiple line comments,
1638 /// for example).
1639 struct CommentReducer<'a> {
1640     is_block: bool,
1641     at_start_line: bool,
1642     iter: std::str::Chars<'a>,
1643 }
1644
1645 impl<'a> CommentReducer<'a> {
1646     fn new(comment: &'a str) -> CommentReducer<'a> {
1647         let is_block = comment.starts_with("/*");
1648         let comment = remove_comment_header(comment);
1649         CommentReducer {
1650             is_block,
1651             // There are no supplementary '*' on the first line.
1652             at_start_line: false,
1653             iter: comment.chars(),
1654         }
1655     }
1656 }
1657
1658 impl<'a> Iterator for CommentReducer<'a> {
1659     type Item = char;
1660
1661     fn next(&mut self) -> Option<Self::Item> {
1662         loop {
1663             let mut c = self.iter.next()?;
1664             if self.is_block && self.at_start_line {
1665                 while c.is_whitespace() {
1666                     c = self.iter.next()?;
1667                 }
1668                 // Ignore leading '*'.
1669                 if c == '*' {
1670                     c = self.iter.next()?;
1671                 }
1672             } else if c == '\n' {
1673                 self.at_start_line = true;
1674             }
1675             if !c.is_whitespace() {
1676                 return Some(c);
1677             }
1678         }
1679     }
1680 }
1681
1682 fn remove_comment_header(comment: &str) -> &str {
1683     if comment.starts_with("///") || comment.starts_with("//!") {
1684         &comment[3..]
1685     } else if comment.starts_with("//") {
1686         &comment[2..]
1687     } else if (comment.starts_with("/**") && !comment.starts_with("/**/"))
1688         || comment.starts_with("/*!")
1689     {
1690         &comment[3..comment.len() - 2]
1691     } else {
1692         assert!(
1693             comment.starts_with("/*"),
1694             "string '{}' is not a comment",
1695             comment
1696         );
1697         &comment[2..comment.len() - 2]
1698     }
1699 }
1700
1701 #[cfg(test)]
1702 mod test {
1703     use super::*;
1704     use crate::shape::{Indent, Shape};
1705
1706     #[test]
1707     fn char_classes() {
1708         let mut iter = CharClasses::new("//\n\n".chars());
1709
1710         assert_eq!((FullCodeCharKind::StartComment, '/'), iter.next().unwrap());
1711         assert_eq!((FullCodeCharKind::InComment, '/'), iter.next().unwrap());
1712         assert_eq!((FullCodeCharKind::EndComment, '\n'), iter.next().unwrap());
1713         assert_eq!((FullCodeCharKind::Normal, '\n'), iter.next().unwrap());
1714         assert_eq!(None, iter.next());
1715     }
1716
1717     #[test]
1718     fn comment_code_slices() {
1719         let input = "code(); /* test */ 1 + 1";
1720         let mut iter = CommentCodeSlices::new(input);
1721
1722         assert_eq!((CodeCharKind::Normal, 0, "code(); "), iter.next().unwrap());
1723         assert_eq!(
1724             (CodeCharKind::Comment, 8, "/* test */"),
1725             iter.next().unwrap()
1726         );
1727         assert_eq!((CodeCharKind::Normal, 18, " 1 + 1"), iter.next().unwrap());
1728         assert_eq!(None, iter.next());
1729     }
1730
1731     #[test]
1732     fn comment_code_slices_two() {
1733         let input = "// comment\n    test();";
1734         let mut iter = CommentCodeSlices::new(input);
1735
1736         assert_eq!((CodeCharKind::Normal, 0, ""), iter.next().unwrap());
1737         assert_eq!(
1738             (CodeCharKind::Comment, 0, "// comment\n"),
1739             iter.next().unwrap()
1740         );
1741         assert_eq!(
1742             (CodeCharKind::Normal, 11, "    test();"),
1743             iter.next().unwrap()
1744         );
1745         assert_eq!(None, iter.next());
1746     }
1747
1748     #[test]
1749     fn comment_code_slices_three() {
1750         let input = "1 // comment\n    // comment2\n\n";
1751         let mut iter = CommentCodeSlices::new(input);
1752
1753         assert_eq!((CodeCharKind::Normal, 0, "1 "), iter.next().unwrap());
1754         assert_eq!(
1755             (CodeCharKind::Comment, 2, "// comment\n    // comment2\n"),
1756             iter.next().unwrap()
1757         );
1758         assert_eq!((CodeCharKind::Normal, 29, "\n"), iter.next().unwrap());
1759         assert_eq!(None, iter.next());
1760     }
1761
1762     #[test]
1763     #[rustfmt::skip]
1764     fn format_doc_comments() {
1765         let mut wrap_normalize_config: crate::config::Config = Default::default();
1766         wrap_normalize_config.set().wrap_comments(true);
1767         wrap_normalize_config.set().normalize_comments(true);
1768
1769         let mut wrap_config: crate::config::Config = Default::default();
1770         wrap_config.set().wrap_comments(true);
1771
1772         let comment = rewrite_comment(" //test",
1773                                       true,
1774                                       Shape::legacy(100, Indent::new(0, 100)),
1775                                       &wrap_normalize_config).unwrap();
1776         assert_eq!("/* test */", comment);
1777
1778         let comment = rewrite_comment("// comment on a",
1779                                       false,
1780                                       Shape::legacy(10, Indent::empty()),
1781                                       &wrap_normalize_config).unwrap();
1782         assert_eq!("// comment\n// on a", comment);
1783
1784         let comment = rewrite_comment("//  A multi line comment\n             // between args.",
1785                                       false,
1786                                       Shape::legacy(60, Indent::new(0, 12)),
1787                                       &wrap_normalize_config).unwrap();
1788         assert_eq!("//  A multi line comment\n            // between args.", comment);
1789
1790         let input = "// comment";
1791         let expected =
1792             "/* comment */";
1793         let comment = rewrite_comment(input,
1794                                       true,
1795                                       Shape::legacy(9, Indent::new(0, 69)),
1796                                       &wrap_normalize_config).unwrap();
1797         assert_eq!(expected, comment);
1798
1799         let comment = rewrite_comment("/*   trimmed    */",
1800                                       true,
1801                                       Shape::legacy(100, Indent::new(0, 100)),
1802                                       &wrap_normalize_config).unwrap();
1803         assert_eq!("/* trimmed */", comment);
1804
1805         // Check that different comment style are properly recognised.
1806         let comment = rewrite_comment(r#"/// test1
1807                                          /// test2
1808                                          /*
1809                                           * test3
1810                                           */"#,
1811                                       false,
1812                                       Shape::legacy(100, Indent::new(0, 0)),
1813                                       &wrap_normalize_config).unwrap();
1814         assert_eq!("/// test1\n/// test2\n// test3", comment);
1815
1816         // Check that the blank line marks the end of a commented paragraph.
1817         let comment = rewrite_comment(r#"// test1
1818
1819                                          // test2"#,
1820                                       false,
1821                                       Shape::legacy(100, Indent::new(0, 0)),
1822                                       &wrap_normalize_config).unwrap();
1823         assert_eq!("// test1\n\n// test2", comment);
1824
1825         // Check that the blank line marks the end of a custom-commented paragraph.
1826         let comment = rewrite_comment(r#"//@ test1
1827
1828                                          //@ test2"#,
1829                                       false,
1830                                       Shape::legacy(100, Indent::new(0, 0)),
1831                                       &wrap_normalize_config).unwrap();
1832         assert_eq!("//@ test1\n\n//@ test2", comment);
1833
1834         // Check that bare lines are just indented but otherwise left unchanged.
1835         let comment = rewrite_comment(r#"// test1
1836                                          /*
1837                                            a bare line!
1838
1839                                                 another bare line!
1840                                           */"#,
1841                                       false,
1842                                       Shape::legacy(100, Indent::new(0, 0)),
1843                                       &wrap_config).unwrap();
1844         assert_eq!("// test1\n/*\n a bare line!\n\n      another bare line!\n*/", comment);
1845     }
1846
1847     // This is probably intended to be a non-test fn, but it is not used.
1848     // We should keep this around unless it helps us test stuff to remove it.
1849     fn uncommented(text: &str) -> String {
1850         CharClasses::new(text.chars())
1851             .filter_map(|(s, c)| match s {
1852                 FullCodeCharKind::Normal | FullCodeCharKind::InString => Some(c),
1853                 _ => None,
1854             })
1855             .collect()
1856     }
1857
1858     #[test]
1859     fn test_uncommented() {
1860         assert_eq!(&uncommented("abc/*...*/"), "abc");
1861         assert_eq!(
1862             &uncommented("// .... /* \n../* /* *** / */ */a/* // */c\n"),
1863             "..ac\n"
1864         );
1865         assert_eq!(&uncommented("abc \" /* */\" qsdf"), "abc \" /* */\" qsdf");
1866     }
1867
1868     #[test]
1869     fn test_contains_comment() {
1870         assert_eq!(contains_comment("abc"), false);
1871         assert_eq!(contains_comment("abc // qsdf"), true);
1872         assert_eq!(contains_comment("abc /* kqsdf"), true);
1873         assert_eq!(contains_comment("abc \" /* */\" qsdf"), false);
1874     }
1875
1876     #[test]
1877     fn test_find_uncommented() {
1878         fn check(haystack: &str, needle: &str, expected: Option<usize>) {
1879             assert_eq!(expected, haystack.find_uncommented(needle));
1880         }
1881
1882         check("/*/ */test", "test", Some(6));
1883         check("//test\ntest", "test", Some(7));
1884         check("/* comment only */", "whatever", None);
1885         check(
1886             "/* comment */ some text /* more commentary */ result",
1887             "result",
1888             Some(46),
1889         );
1890         check("sup // sup", "p", Some(2));
1891         check("sup", "x", None);
1892         check(r#"π? /**/ π is nice!"#, r#"π is nice"#, Some(9));
1893         check("/*sup yo? \n sup*/ sup", "p", Some(20));
1894         check("hel/*lohello*/lo", "hello", None);
1895         check("acb", "ab", None);
1896         check(",/*A*/ ", ",", Some(0));
1897         check("abc", "abc", Some(0));
1898         check("/* abc */", "abc", None);
1899         check("/**/abc/* */", "abc", Some(4));
1900         check("\"/* abc */\"", "abc", Some(4));
1901         check("\"/* abc", "abc", Some(4));
1902     }
1903
1904     #[test]
1905     fn test_filter_normal_code() {
1906         let s = r#"
1907 fn main() {
1908     println!("hello, world");
1909 }
1910 "#;
1911         assert_eq!(s, filter_normal_code(s));
1912         let s_with_comment = r#"
1913 fn main() {
1914     // hello, world
1915     println!("hello, world");
1916 }
1917 "#;
1918         assert_eq!(s, filter_normal_code(s_with_comment));
1919     }
1920 }