src/comment.rs

   1 // Copyright 2015 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 // Formatting and tools for comments.
  12
  13 use std::{self, borrow::Cow, iter};
  14
  15 use itertools::{multipeek, MultiPeek};
  16 use syntax::source_map::Span;
  17
  18 use config::Config;
  19 use rewrite::RewriteContext;
  20 use shape::{Indent, Shape};
  21 use string::{rewrite_string, StringFormat};
  22 use utils::{count_newlines, first_line_width, last_line_width, trim_left_preserve_layout};
  23 use {ErrorKind, FormattingError};
  24
  25 fn is_custom_comment(comment: &str) -> bool {
  26     if !comment.starts_with("//") {
  27         false
  28     } else if let Some(c) = comment.chars().nth(2) {
  29         !c.is_alphanumeric() && !c.is_whitespace()
  30     } else {
  31         false
  32     }
  33 }
  34
  35 #[derive(Copy, Clone, PartialEq, Eq)]
  36 pub enum CommentStyle<'a> {
  37     DoubleSlash,
  38     TripleSlash,
  39     Doc,
  40     SingleBullet,
  41     DoubleBullet,
  42     Exclamation,
  43     Custom(&'a str),
  44 }
  45
  46 fn custom_opener(s: &str) -> &str {
  47     s.lines().next().map_or("", |first_line| {
  48         first_line
  49             .find(' ')
  50             .map_or(first_line, |space_index| &first_line[0..=space_index])
  51     })
  52 }
  53
  54 impl<'a> CommentStyle<'a> {
  55     /// Returns true if the commenting style covers a line only.
  56     pub fn is_line_comment(&self) -> bool {
  57         match *self {
  58             CommentStyle::DoubleSlash
  59             | CommentStyle::TripleSlash
  60             | CommentStyle::Doc
  61             | CommentStyle::Custom(_) => true,
  62             _ => false,
  63         }
  64     }
  65
  66     /// Returns true if the commenting style can span over multiple lines.
  67     pub fn is_block_comment(&self) -> bool {
  68         match *self {
  69             CommentStyle::SingleBullet | CommentStyle::DoubleBullet | CommentStyle::Exclamation => {
  70                 true
  71             }
  72             _ => false,
  73         }
  74     }
  75
  76     /// Returns true if the commenting style is for documentation.
  77     pub fn is_doc_comment(&self) -> bool {
  78         match *self {
  79             CommentStyle::TripleSlash | CommentStyle::Doc => true,
  80             _ => false,
  81         }
  82     }
  83
  84     pub fn opener(&self) -> &'a str {
  85         match *self {
  86             CommentStyle::DoubleSlash => "// ",
  87             CommentStyle::TripleSlash => "/// ",
  88             CommentStyle::Doc => "//! ",
  89             CommentStyle::SingleBullet => "/* ",
  90             CommentStyle::DoubleBullet => "/** ",
  91             CommentStyle::Exclamation => "/*! ",
  92             CommentStyle::Custom(opener) => opener,
  93         }
  94     }
  95
  96     pub fn closer(&self) -> &'a str {
  97         match *self {
  98             CommentStyle::DoubleSlash
  99             | CommentStyle::TripleSlash
 100             | CommentStyle::Custom(..)
 101             | CommentStyle::Doc => "",
 102             CommentStyle::DoubleBullet => " **/",
 103             CommentStyle::SingleBullet | CommentStyle::Exclamation => " */",
 104         }
 105     }
 106
 107     pub fn line_start(&self) -> &'a str {
 108         match *self {
 109             CommentStyle::DoubleSlash => "// ",
 110             CommentStyle::TripleSlash => "/// ",
 111             CommentStyle::Doc => "//! ",
 112             CommentStyle::SingleBullet | CommentStyle::Exclamation => " * ",
 113             CommentStyle::DoubleBullet => " ** ",
 114             CommentStyle::Custom(opener) => opener,
 115         }
 116     }
 117
 118     pub fn to_str_tuplet(&self) -> (&'a str, &'a str, &'a str) {
 119         (self.opener(), self.closer(), self.line_start())
 120     }
 121 }
 122
 123 fn comment_style(orig: &str, normalize_comments: bool) -> CommentStyle {
 124     if !normalize_comments {
 125         if orig.starts_with("/**") && !orig.starts_with("/**/") {
 126             CommentStyle::DoubleBullet
 127         } else if orig.starts_with("/*!") {
 128             CommentStyle::Exclamation
 129         } else if orig.starts_with("/*") {
 130             CommentStyle::SingleBullet
 131         } else if orig.starts_with("///") && orig.chars().nth(3).map_or(true, |c| c != '/') {
 132             CommentStyle::TripleSlash
 133         } else if orig.starts_with("//!") {
 134             CommentStyle::Doc
 135         } else if is_custom_comment(orig) {
 136             CommentStyle::Custom(custom_opener(orig))
 137         } else {
 138             CommentStyle::DoubleSlash
 139         }
 140     } else if (orig.starts_with("///") && orig.chars().nth(3).map_or(true, |c| c != '/'))
 141         || (orig.starts_with("/**") && !orig.starts_with("/**/"))
 142     {
 143         CommentStyle::TripleSlash
 144     } else if orig.starts_with("//!") || orig.starts_with("/*!") {
 145         CommentStyle::Doc
 146     } else if is_custom_comment(orig) {
 147         CommentStyle::Custom(custom_opener(orig))
 148     } else {
 149         CommentStyle::DoubleSlash
 150     }
 151 }
 152
 153 /// Combine `prev_str` and `next_str` into a single `String`. `span` may contain
 154 /// comments between two strings. If there are such comments, then that will be
 155 /// recovered. If `allow_extend` is true and there is no comment between the two
 156 /// strings, then they will be put on a single line as long as doing so does not
 157 /// exceed max width.
 158 pub fn combine_strs_with_missing_comments(
 159     context: &RewriteContext,
 160     prev_str: &str,
 161     next_str: &str,
 162     span: Span,
 163     shape: Shape,
 164     allow_extend: bool,
 165 ) -> Option<String> {
 166     let mut result =
 167         String::with_capacity(prev_str.len() + next_str.len() + shape.indent.width() + 128);
 168     result.push_str(prev_str);
 169     let mut allow_one_line = !prev_str.contains('\n') && !next_str.contains('\n');
 170     let first_sep = if prev_str.is_empty() || next_str.is_empty() {
 171         ""
 172     } else {
 173         " "
 174     };
 175     let mut one_line_width =
 176         last_line_width(prev_str) + first_line_width(next_str) + first_sep.len();
 177
 178     let config = context.config;
 179     let indent = shape.indent;
 180     let missing_comment = rewrite_missing_comment(span, shape, context)?;
 181
 182     if missing_comment.is_empty() {
 183         if allow_extend && prev_str.len() + first_sep.len() + next_str.len() <= shape.width {
 184             result.push_str(first_sep);
 185         } else if !prev_str.is_empty() {
 186             result.push_str(&indent.to_string_with_newline(config))
 187         }
 188         result.push_str(next_str);
 189         return Some(result);
 190     }
 191
 192     // We have a missing comment between the first expression and the second expression.
 193
 194     // Peek the the original source code and find out whether there is a newline between the first
 195     // expression and the second expression or the missing comment. We will preserve the original
 196     // layout whenever possible.
 197     let original_snippet = context.snippet(span);
 198     let prefer_same_line = if let Some(pos) = original_snippet.find('/') {
 199         !original_snippet[..pos].contains('\n')
 200     } else {
 201         !original_snippet.contains('\n')
 202     };
 203
 204     one_line_width -= first_sep.len();
 205     let first_sep = if prev_str.is_empty() || missing_comment.is_empty() {
 206         Cow::from("")
 207     } else {
 208         let one_line_width = last_line_width(prev_str) + first_line_width(&missing_comment) + 1;
 209         if prefer_same_line && one_line_width <= shape.width {
 210             Cow::from(" ")
 211         } else {
 212             indent.to_string_with_newline(config)
 213         }
 214     };
 215     result.push_str(&first_sep);
 216     result.push_str(&missing_comment);
 217
 218     let second_sep = if missing_comment.is_empty() || next_str.is_empty() {
 219         Cow::from("")
 220     } else if missing_comment.starts_with("//") {
 221         indent.to_string_with_newline(config)
 222     } else {
 223         one_line_width += missing_comment.len() + first_sep.len() + 1;
 224         allow_one_line &= !missing_comment.starts_with("//") && !missing_comment.contains('\n');
 225         if prefer_same_line && allow_one_line && one_line_width <= shape.width {
 226             Cow::from(" ")
 227         } else {
 228             indent.to_string_with_newline(config)
 229         }
 230     };
 231     result.push_str(&second_sep);
 232     result.push_str(next_str);
 233
 234     Some(result)
 235 }
 236
 237 pub fn rewrite_doc_comment(orig: &str, shape: Shape, config: &Config) -> Option<String> {
 238     identify_comment(orig, false, shape, config, true)
 239 }
 240
 241 pub fn rewrite_comment(
 242     orig: &str,
 243     block_style: bool,
 244     shape: Shape,
 245     config: &Config,
 246 ) -> Option<String> {
 247     identify_comment(orig, block_style, shape, config, false)
 248 }
 249
 250 fn identify_comment(
 251     orig: &str,
 252     block_style: bool,
 253     shape: Shape,
 254     config: &Config,
 255     is_doc_comment: bool,
 256 ) -> Option<String> {
 257     let style = comment_style(orig, false);
 258
 259     // Computes the len of line taking into account a newline if the line is part of a paragraph.
 260     fn compute_len(orig: &str, line: &str) -> usize {
 261         if orig.len() > line.len() {
 262             if orig.as_bytes()[line.len()] == b'\r' {
 263                 line.len() + 2
 264             } else {
 265                 line.len() + 1
 266             }
 267         } else {
 268             line.len()
 269         }
 270     }
 271
 272     // Get the first group of line comments having the same commenting style.
 273     //
 274     // Returns a tuple with:
 275     // - a boolean indicating if there is a blank line
 276     // - a number indicating the size of the first group of comments
 277     fn consume_same_line_comments(
 278         style: CommentStyle,
 279         orig: &str,
 280         line_start: &str,
 281     ) -> (bool, usize) {
 282         let mut first_group_ending = 0;
 283         let mut hbl = false;
 284
 285         for line in orig.lines() {
 286             let trimmed_line = line.trim_left();
 287             if trimmed_line.is_empty() {
 288                 hbl = true;
 289                 break;
 290             } else if trimmed_line.starts_with(line_start)
 291                 || comment_style(trimmed_line, false) == style
 292             {
 293                 first_group_ending += compute_len(&orig[first_group_ending..], line);
 294             } else {
 295                 break;
 296             }
 297         }
 298         (hbl, first_group_ending)
 299     }
 300
 301     let (has_bare_lines, first_group_ending) = match style {
 302         CommentStyle::DoubleSlash | CommentStyle::TripleSlash | CommentStyle::Doc => {
 303             let line_start = style.line_start().trim_left();
 304             consume_same_line_comments(style, orig, line_start)
 305         }
 306         CommentStyle::Custom(opener) => {
 307             let trimmed_opener = opener.trim_right();
 308             consume_same_line_comments(style, orig, trimmed_opener)
 309         }
 310         // for a block comment, search for the closing symbol
 311         CommentStyle::DoubleBullet | CommentStyle::SingleBullet | CommentStyle::Exclamation => {
 312             let closer = style.closer().trim_left();
 313             let mut closing_symbol_offset = 0;
 314             let mut hbl = false;
 315             for line in orig.lines() {
 316                 closing_symbol_offset += compute_len(&orig[closing_symbol_offset..], line);
 317                 let trimmed_line = line.trim_left();
 318                 if !trimmed_line.starts_with('*')
 319                     && !trimmed_line.starts_with("//")
 320                     && !trimmed_line.starts_with("/*")
 321                 {
 322                     hbl = true;
 323                 }
 324                 if trimmed_line.ends_with(closer) {
 325                     break;
 326                 }
 327             }
 328             (hbl, closing_symbol_offset)
 329         }
 330     };
 331
 332     let (first_group, rest) = orig.split_at(first_group_ending);
 333     let rewritten_first_group =
 334         if !config.normalize_comments() && has_bare_lines && style.is_block_comment() {
 335             trim_left_preserve_layout(first_group, &shape.indent, config)
 336         } else if !config.normalize_comments()
 337             && !config.wrap_comments()
 338             && !config.format_doc_comments()
 339         {
 340             light_rewrite_comment(first_group, shape.indent, config, is_doc_comment)
 341         } else {
 342             rewrite_comment_inner(
 343                 first_group,
 344                 block_style,
 345                 style,
 346                 shape,
 347                 config,
 348                 is_doc_comment || style.is_doc_comment(),
 349             )?
 350         };
 351     if rest.is_empty() {
 352         Some(rewritten_first_group)
 353     } else {
 354         identify_comment(rest.trim_left(), block_style, shape, config, is_doc_comment).map(
 355             |rest_str| {
 356                 format!(
 357                     "{}\n{}{}{}",
 358                     rewritten_first_group,
 359                     // insert back the blank line
 360                     if has_bare_lines && style.is_line_comment() {
 361                         "\n"
 362                     } else {
 363                         ""
 364                     },
 365                     shape.indent.to_string(config),
 366                     rest_str
 367                 )
 368             },
 369         )
 370     }
 371 }
 372
 373 /// Attributes for code blocks in rustdoc.
 374 /// See https://doc.rust-lang.org/rustdoc/print.html#attributes
 375 enum CodeBlockAttribute {
 376     Rust,
 377     Ignore,
 378     Text,
 379     ShouldPanic,
 380     NoRun,
 381     CompileFail,
 382 }
 383
 384 impl CodeBlockAttribute {
 385     fn new(attribute: &str) -> CodeBlockAttribute {
 386         match attribute {
 387             "rust" | "" => CodeBlockAttribute::Rust,
 388             "ignore" => CodeBlockAttribute::Ignore,
 389             "text" => CodeBlockAttribute::Text,
 390             "should_panic" => CodeBlockAttribute::ShouldPanic,
 391             "no_run" => CodeBlockAttribute::NoRun,
 392             "compile_fail" => CodeBlockAttribute::CompileFail,
 393             _ => CodeBlockAttribute::Text,
 394         }
 395     }
 396 }
 397
 398 /// Block that is formatted as an item.
 399 ///
 400 /// An item starts with either a star `*` or a dash `-`. Different level of indentation are
 401 /// handled.
 402 struct ItemizedBlock {
 403     /// the number of whitespaces up to the item sigil
 404     indent: usize,
 405     /// the string that marks the start of an item
 406     opener: String,
 407     /// sequence of whitespaces to prefix new lines that are part of the item
 408     line_start: String,
 409 }
 410
 411 impl ItemizedBlock {
 412     /// Returns true if the line is formatted as an item
 413     fn is_itemized_line(line: &str) -> bool {
 414         let trimmed = line.trim_left();
 415         trimmed.starts_with("* ") || trimmed.starts_with("- ")
 416     }
 417
 418     /// Creates a new ItemizedBlock described with the given line.
 419     /// The `is_itemized_line` needs to be called first.
 420     fn new(line: &str) -> ItemizedBlock {
 421         let space_to_sigil = line.chars().take_while(|c| c.is_whitespace()).count();
 422         let indent = space_to_sigil + 2;
 423         ItemizedBlock {
 424             indent,
 425             opener: line[..indent].to_string(),
 426             line_start: " ".repeat(indent),
 427         }
 428     }
 429
 430     /// Returns a `StringFormat` used for formatting the content of an item
 431     fn create_string_format<'a>(&'a self, fmt: &'a StringFormat) -> StringFormat<'a> {
 432         StringFormat {
 433             opener: "",
 434             closer: "",
 435             line_start: "",
 436             line_end: "",
 437             shape: Shape::legacy(fmt.shape.width.saturating_sub(self.indent), Indent::empty()),
 438             trim_end: true,
 439             config: fmt.config,
 440         }
 441     }
 442
 443     /// Returns true if the line is part of the current itemized block
 444     fn in_block(&self, line: &str) -> bool {
 445         !ItemizedBlock::is_itemized_line(line)
 446             && self.indent <= line.chars().take_while(|c| c.is_whitespace()).count()
 447     }
 448 }
 449
 450 struct CommentRewrite<'a> {
 451     result: String,
 452     code_block_buffer: String,
 453     is_prev_line_multi_line: bool,
 454     code_block_attr: Option<CodeBlockAttribute>,
 455     item_block_buffer: String,
 456     item_block: Option<ItemizedBlock>,
 457     comment_line_separator: String,
 458     indent_str: String,
 459     max_chars: usize,
 460     fmt_indent: Indent,
 461     fmt: StringFormat<'a>,
 462
 463     opener: String,
 464     closer: String,
 465     line_start: String,
 466 }
 467
 468 impl<'a> CommentRewrite<'a> {
 469     fn new(
 470         orig: &'a str,
 471         block_style: bool,
 472         shape: Shape,
 473         config: &'a Config,
 474     ) -> CommentRewrite<'a> {
 475         let (opener, closer, line_start) = if block_style {
 476             CommentStyle::SingleBullet.to_str_tuplet()
 477         } else {
 478             comment_style(orig, config.normalize_comments()).to_str_tuplet()
 479         };
 480
 481         let max_chars = shape
 482             .width
 483             .checked_sub(closer.len() + opener.len())
 484             .unwrap_or(1);
 485         let indent_str = shape.indent.to_string_with_newline(config).to_string();
 486         let fmt_indent = shape.indent + (opener.len() - line_start.len());
 487
 488         let mut cr = CommentRewrite {
 489             result: String::with_capacity(orig.len() * 2),
 490             code_block_buffer: String::with_capacity(128),
 491             is_prev_line_multi_line: false,
 492             code_block_attr: None,
 493             item_block_buffer: String::with_capacity(128),
 494             item_block: None,
 495             comment_line_separator: format!("{}{}", indent_str, line_start),
 496             max_chars,
 497             indent_str,
 498             fmt_indent,
 499
 500             fmt: StringFormat {
 501                 opener: "",
 502                 closer: "",
 503                 line_start,
 504                 line_end: "",
 505                 shape: Shape::legacy(max_chars, fmt_indent),
 506                 trim_end: true,
 507                 config,
 508             },
 509
 510             opener: opener.to_owned(),
 511             closer: closer.to_owned(),
 512             line_start: line_start.to_owned(),
 513         };
 514         cr.result.push_str(opener);
 515         cr
 516     }
 517
 518     fn join_block(s: &str, sep: &str) -> String {
 519         let mut result = String::with_capacity(s.len() + 128);
 520         let mut iter = s.lines().peekable();
 521         while let Some(line) = iter.next() {
 522             result.push_str(line);
 523             result.push_str(match iter.peek() {
 524                 Some(next_line) if next_line.is_empty() => sep.trim_right(),
 525                 Some(..) => &sep,
 526                 None => "",
 527             });
 528         }
 529         result
 530     }
 531
 532     fn finish(mut self) -> String {
 533         if !self.code_block_buffer.is_empty() {
 534             // There is a code block that is not properly enclosed by backticks.
 535             // We will leave them untouched.
 536             self.result.push_str(&self.comment_line_separator);
 537             self.result.push_str(&Self::join_block(
 538                 &trim_custom_comment_prefix(&self.code_block_buffer),
 539                 &self.comment_line_separator,
 540             ));
 541         }
 542
 543         if !self.item_block_buffer.is_empty() {
 544             // the last few lines are part of an itemized block
 545             self.fmt.shape = Shape::legacy(self.max_chars, self.fmt_indent);
 546             let mut ib = None;
 547             ::std::mem::swap(&mut ib, &mut self.item_block);
 548             let ib = ib.unwrap();
 549             let item_fmt = ib.create_string_format(&self.fmt);
 550             self.result.push_str(&self.comment_line_separator);
 551             self.result.push_str(&ib.opener);
 552             match rewrite_string(
 553                 &self.item_block_buffer.replace("\n", " "),
 554                 &item_fmt,
 555                 self.max_chars.saturating_sub(ib.indent),
 556             ) {
 557                 Some(s) => self.result.push_str(&Self::join_block(
 558                     &s,
 559                     &format!("{}{}", &self.comment_line_separator, ib.line_start),
 560                 )),
 561                 None => self.result.push_str(&Self::join_block(
 562                     &self.item_block_buffer,
 563                     &self.comment_line_separator,
 564                 )),
 565             };
 566         }
 567
 568         self.result.push_str(&self.closer);
 569         if self.result.ends_with(&self.opener) && self.opener.ends_with(' ') {
 570             // Trailing space.
 571             self.result.pop();
 572         }
 573
 574         self.result
 575     }
 576
 577     fn handle_line(
 578         &mut self,
 579         orig: &'a str,
 580         i: usize,
 581         line: &'a str,
 582         has_leading_whitespace: bool,
 583     ) -> bool {
 584         let is_last = i == count_newlines(orig);
 585
 586         if let Some(ref ib) = self.item_block {
 587             if ib.in_block(&line) {
 588                 self.item_block_buffer.push_str(&line);
 589                 self.item_block_buffer.push('\n');
 590                 return false;
 591             }
 592             self.is_prev_line_multi_line = false;
 593             self.fmt.shape = Shape::legacy(self.max_chars, self.fmt_indent);
 594             let item_fmt = ib.create_string_format(&self.fmt);
 595             self.result.push_str(&self.comment_line_separator);
 596             self.result.push_str(&ib.opener);
 597             match rewrite_string(
 598                 &self.item_block_buffer.replace("\n", " "),
 599                 &item_fmt,
 600                 self.max_chars.saturating_sub(ib.indent),
 601             ) {
 602                 Some(s) => self.result.push_str(&Self::join_block(
 603                     &s,
 604                     &format!("{}{}", &self.comment_line_separator, ib.line_start),
 605                 )),
 606                 None => self.result.push_str(&Self::join_block(
 607                     &self.item_block_buffer,
 608                     &self.comment_line_separator,
 609                 )),
 610             };
 611             self.item_block_buffer.clear();
 612         } else if self.code_block_attr.is_some() {
 613             if line.starts_with("```") {
 614                 let code_block = match self.code_block_attr.as_ref().unwrap() {
 615                     CodeBlockAttribute::Ignore | CodeBlockAttribute::Text => {
 616                         trim_custom_comment_prefix(&self.code_block_buffer)
 617                     }
 618                     _ if self.code_block_buffer.is_empty() => String::new(),
 619                     _ => {
 620                         let mut config = self.fmt.config.clone();
 621                         config.set().format_doc_comments(false);
 622                         match ::format_code_block(&self.code_block_buffer, &config) {
 623                             Some(ref s) => trim_custom_comment_prefix(&s.snippet),
 624                             None => trim_custom_comment_prefix(&self.code_block_buffer),
 625                         }
 626                     }
 627                 };
 628                 if !code_block.is_empty() {
 629                     self.result.push_str(&self.comment_line_separator);
 630                     self.result
 631                         .push_str(&Self::join_block(&code_block, &self.comment_line_separator));
 632                 }
 633                 self.code_block_buffer.clear();
 634                 self.result.push_str(&self.comment_line_separator);
 635                 self.result.push_str(line);
 636                 self.code_block_attr = None;
 637             } else {
 638                 self.code_block_buffer
 639                     .push_str(&hide_sharp_behind_comment(line));
 640                 self.code_block_buffer.push('\n');
 641             }
 642             return false;
 643         }
 644
 645         self.code_block_attr = None;
 646         self.item_block = None;
 647         if line.starts_with("```") {
 648             self.code_block_attr = Some(CodeBlockAttribute::new(&line[3..]))
 649         } else if self.fmt.config.wrap_comments() && ItemizedBlock::is_itemized_line(&line) {
 650             let ib = ItemizedBlock::new(&line);
 651             self.item_block_buffer.push_str(&line[ib.indent..]);
 652             self.item_block_buffer.push('\n');
 653             self.item_block = Some(ib);
 654             return false;
 655         }
 656
 657         if self.result == self.opener {
 658             let force_leading_whitespace = &self.opener == "/* " && count_newlines(orig) == 0;
 659             if !has_leading_whitespace && !force_leading_whitespace && self.result.ends_with(' ') {
 660                 self.result.pop();
 661             }
 662             if line.is_empty() {
 663                 return false;
 664             }
 665         } else if self.is_prev_line_multi_line && !line.is_empty() {
 666             self.result.push(' ')
 667         } else if is_last && line.is_empty() {
 668             // trailing blank lines are unwanted
 669             if !self.closer.is_empty() {
 670                 self.result.push_str(&self.indent_str);
 671             }
 672             return true;
 673         } else {
 674             self.result.push_str(&self.comment_line_separator);
 675             if !has_leading_whitespace && self.result.ends_with(' ') {
 676                 self.result.pop();
 677             }
 678         }
 679
 680         if self.fmt.config.wrap_comments() && line.len() > self.fmt.shape.width && !has_url(line) {
 681             match rewrite_string(line, &self.fmt, self.max_chars) {
 682                 Some(ref s) => {
 683                     self.is_prev_line_multi_line = s.contains('\n');
 684                     self.result.push_str(s);
 685                 }
 686                 None if self.is_prev_line_multi_line => {
 687                     // We failed to put the current `line` next to the previous `line`.
 688                     // Remove the trailing space, then start rewrite on the next line.
 689                     self.result.pop();
 690                     self.result.push_str(&self.comment_line_separator);
 691                     self.fmt.shape = Shape::legacy(self.max_chars, self.fmt_indent);
 692                     match rewrite_string(line, &self.fmt, self.max_chars) {
 693                         Some(ref s) => {
 694                             self.is_prev_line_multi_line = s.contains('\n');
 695                             self.result.push_str(s);
 696                         }
 697                         None => {
 698                             self.is_prev_line_multi_line = false;
 699                             self.result.push_str(line);
 700                         }
 701                     }
 702                 }
 703                 None => {
 704                     self.is_prev_line_multi_line = false;
 705                     self.result.push_str(line);
 706                 }
 707             }
 708
 709             self.fmt.shape = if self.is_prev_line_multi_line {
 710                 // 1 = " "
 711                 let offset = 1 + last_line_width(&self.result) - self.line_start.len();
 712                 Shape {
 713                     width: self.max_chars.saturating_sub(offset),
 714                     indent: self.fmt_indent,
 715                     offset: self.fmt.shape.offset + offset,
 716                 }
 717             } else {
 718                 Shape::legacy(self.max_chars, self.fmt_indent)
 719             };
 720         } else {
 721             if line.is_empty() && self.result.ends_with(' ') && !is_last {
 722                 // Remove space if this is an empty comment or a doc comment.
 723                 self.result.pop();
 724             }
 725             self.result.push_str(line);
 726             self.fmt.shape = Shape::legacy(self.max_chars, self.fmt_indent);
 727             self.is_prev_line_multi_line = false;
 728         }
 729
 730         false
 731     }
 732 }
 733
 734 fn rewrite_comment_inner(
 735     orig: &str,
 736     block_style: bool,
 737     style: CommentStyle,
 738     shape: Shape,
 739     config: &Config,
 740     is_doc_comment: bool,
 741 ) -> Option<String> {
 742     let mut rewriter = CommentRewrite::new(orig, block_style, shape, config);
 743
 744     let line_breaks = count_newlines(orig.trim_right());
 745     let lines = orig
 746         .lines()
 747         .enumerate()
 748         .map(|(i, mut line)| {
 749             line = trim_right_unless_two_whitespaces(line.trim_left(), is_doc_comment);
 750             // Drop old closer.
 751             if i == line_breaks && line.ends_with("*/") && !line.starts_with("//") {
 752                 line = line[..(line.len() - 2)].trim_right();
 753             }
 754
 755             line
 756         })
 757         .map(|s| left_trim_comment_line(s, &style))
 758         .map(|(line, has_leading_whitespace)| {
 759             if orig.starts_with("/*") && line_breaks == 0 {
 760                 (
 761                     line.trim_left(),
 762                     has_leading_whitespace || config.normalize_comments(),
 763                 )
 764             } else {
 765                 (line, has_leading_whitespace || config.normalize_comments())
 766             }
 767         });
 768
 769     for (i, (line, has_leading_whitespace)) in lines.enumerate() {
 770         if rewriter.handle_line(orig, i, line, has_leading_whitespace) {
 771             break;
 772         }
 773     }
 774
 775     Some(rewriter.finish())
 776 }
 777
 778 const RUSTFMT_CUSTOM_COMMENT_PREFIX: &str = "//#### ";
 779
 780 fn hide_sharp_behind_comment(s: &str) -> Cow<str> {
 781     if s.trim_left().starts_with("# ") {
 782         Cow::from(format!("{}{}", RUSTFMT_CUSTOM_COMMENT_PREFIX, s))
 783     } else {
 784         Cow::from(s)
 785     }
 786 }
 787
 788 fn trim_custom_comment_prefix(s: &str) -> String {
 789     s.lines()
 790         .map(|line| {
 791             let left_trimmed = line.trim_left();
 792             if left_trimmed.starts_with(RUSTFMT_CUSTOM_COMMENT_PREFIX) {
 793                 let orig = left_trimmed.trim_left_matches(RUSTFMT_CUSTOM_COMMENT_PREFIX);
 794                 // due to comment wrapping, a line that was originally behind `#` is split over
 795                 // multiple lines, which needs then to be prefixed with a `#`
 796                 if !orig.trim_left().starts_with("# ") {
 797                     Cow::from(format!("# {}", orig))
 798                 } else {
 799                     Cow::from(orig)
 800                 }
 801             } else {
 802                 Cow::from(line)
 803             }
 804         })
 805         .collect::<Vec<_>>()
 806         .join("\n")
 807 }
 808
 809 /// Returns true if the given string MAY include URLs or alike.
 810 fn has_url(s: &str) -> bool {
 811     // This function may return false positive, but should get its job done in most cases.
 812     s.contains("https://") || s.contains("http://") || s.contains("ftp://") || s.contains("file://")
 813 }
 814
 815 /// Given the span, rewrite the missing comment inside it if available.
 816 /// Note that the given span must only include comments (or leading/trailing whitespaces).
 817 pub fn rewrite_missing_comment(
 818     span: Span,
 819     shape: Shape,
 820     context: &RewriteContext,
 821 ) -> Option<String> {
 822     let missing_snippet = context.snippet(span);
 823     let trimmed_snippet = missing_snippet.trim();
 824     if !trimmed_snippet.is_empty() {
 825         rewrite_comment(trimmed_snippet, false, shape, context.config)
 826     } else {
 827         Some(String::new())
 828     }
 829 }
 830
 831 /// Recover the missing comments in the specified span, if available.
 832 /// The layout of the comments will be preserved as long as it does not break the code
 833 /// and its total width does not exceed the max width.
 834 pub fn recover_missing_comment_in_span(
 835     span: Span,
 836     shape: Shape,
 837     context: &RewriteContext,
 838     used_width: usize,
 839 ) -> Option<String> {
 840     let missing_comment = rewrite_missing_comment(span, shape, context)?;
 841     if missing_comment.is_empty() {
 842         Some(String::new())
 843     } else {
 844         let missing_snippet = context.snippet(span);
 845         let pos = missing_snippet.find('/').unwrap_or(0);
 846         // 1 = ` `
 847         let total_width = missing_comment.len() + used_width + 1;
 848         let force_new_line_before_comment =
 849             missing_snippet[..pos].contains('\n') || total_width > context.config.max_width();
 850         let sep = if force_new_line_before_comment {
 851             shape.indent.to_string_with_newline(context.config)
 852         } else {
 853             Cow::from(" ")
 854         };
 855         Some(format!("{}{}", sep, missing_comment))
 856     }
 857 }
 858
 859 /// Trim trailing whitespaces unless they consist of two or more whitespaces.
 860 fn trim_right_unless_two_whitespaces(s: &str, is_doc_comment: bool) -> &str {
 861     if is_doc_comment && s.ends_with("  ") {
 862         s
 863     } else {
 864         s.trim_right()
 865     }
 866 }
 867
 868 /// Trims whitespace and aligns to indent, but otherwise does not change comments.
 869 fn light_rewrite_comment(
 870     orig: &str,
 871     offset: Indent,
 872     config: &Config,
 873     is_doc_comment: bool,
 874 ) -> String {
 875     let lines: Vec<&str> = orig
 876         .lines()
 877         .map(|l| {
 878             // This is basically just l.trim(), but in the case that a line starts
 879             // with `*` we want to leave one space before it, so it aligns with the
 880             // `*` in `/*`.
 881             let first_non_whitespace = l.find(|c| !char::is_whitespace(c));
 882             let left_trimmed = if let Some(fnw) = first_non_whitespace {
 883                 if l.as_bytes()[fnw] == b'*' && fnw > 0 {
 884                     &l[fnw - 1..]
 885                 } else {
 886                     &l[fnw..]
 887                 }
 888             } else {
 889                 ""
 890             };
 891             // Preserve markdown's double-space line break syntax in doc comment.
 892             trim_right_unless_two_whitespaces(left_trimmed, is_doc_comment)
 893         })
 894         .collect();
 895     lines.join(&format!("\n{}", offset.to_string(config)))
 896 }
 897
 898 /// Trims comment characters and possibly a single space from the left of a string.
 899 /// Does not trim all whitespace. If a single space is trimmed from the left of the string,
 900 /// this function returns true.
 901 fn left_trim_comment_line<'a>(line: &'a str, style: &CommentStyle) -> (&'a str, bool) {
 902     if line.starts_with("//! ")
 903         || line.starts_with("/// ")
 904         || line.starts_with("/*! ")
 905         || line.starts_with("/** ")
 906     {
 907         (&line[4..], true)
 908     } else if let CommentStyle::Custom(opener) = *style {
 909         if line.starts_with(opener) {
 910             (&line[opener.len()..], true)
 911         } else {
 912             (&line[opener.trim_right().len()..], false)
 913         }
 914     } else if line.starts_with("/* ")
 915         || line.starts_with("// ")
 916         || line.starts_with("//!")
 917         || line.starts_with("///")
 918         || line.starts_with("** ")
 919         || line.starts_with("/*!")
 920         || (line.starts_with("/**") && !line.starts_with("/**/"))
 921     {
 922         (&line[3..], line.chars().nth(2).unwrap() == ' ')
 923     } else if line.starts_with("/*")
 924         || line.starts_with("* ")
 925         || line.starts_with("//")
 926         || line.starts_with("**")
 927     {
 928         (&line[2..], line.chars().nth(1).unwrap() == ' ')
 929     } else if line.starts_with('*') {
 930         (&line[1..], false)
 931     } else {
 932         (line, line.starts_with(' '))
 933     }
 934 }
 935
 936 pub trait FindUncommented {
 937     fn find_uncommented(&self, pat: &str) -> Option<usize>;
 938 }
 939
 940 impl FindUncommented for str {
 941     fn find_uncommented(&self, pat: &str) -> Option<usize> {
 942         let mut needle_iter = pat.chars();
 943         for (kind, (i, b)) in CharClasses::new(self.char_indices()) {
 944             match needle_iter.next() {
 945                 None => {
 946                     return Some(i - pat.len());
 947                 }
 948                 Some(c) => match kind {
 949                     FullCodeCharKind::Normal | FullCodeCharKind::InString if b == c => {}
 950                     _ => {
 951                         needle_iter = pat.chars();
 952                     }
 953                 },
 954             }
 955         }
 956
 957         // Handle case where the pattern is a suffix of the search string
 958         match needle_iter.next() {
 959             Some(_) => None,
 960             None => Some(self.len() - pat.len()),
 961         }
 962     }
 963 }
 964
 965 // Returns the first byte position after the first comment. The given string
 966 // is expected to be prefixed by a comment, including delimiters.
 967 // Good: "/* /* inner */ outer */ code();"
 968 // Bad:  "code(); // hello\n world!"
 969 pub fn find_comment_end(s: &str) -> Option<usize> {
 970     let mut iter = CharClasses::new(s.char_indices());
 971     for (kind, (i, _c)) in &mut iter {
 972         if kind == FullCodeCharKind::Normal || kind == FullCodeCharKind::InString {
 973             return Some(i);
 974         }
 975     }
 976
 977     // Handle case where the comment ends at the end of s.
 978     if iter.status == CharClassesStatus::Normal {
 979         Some(s.len())
 980     } else {
 981         None
 982     }
 983 }
 984
 985 /// Returns true if text contains any comment.
 986 pub fn contains_comment(text: &str) -> bool {
 987     CharClasses::new(text.chars()).any(|(kind, _)| kind.is_comment())
 988 }
 989
 990 pub struct CharClasses<T>
 991 where
 992     T: Iterator,
 993     T::Item: RichChar,
 994 {
 995     base: MultiPeek<T>,
 996     status: CharClassesStatus,
 997 }
 998
 999 pub trait RichChar {
1000     fn get_char(&self) -> char;
1001 }
1002
1003 impl RichChar for char {
1004     fn get_char(&self) -> char {
1005         *self
1006     }
1007 }
1008
1009 impl RichChar for (usize, char) {
1010     fn get_char(&self) -> char {
1011         self.1
1012     }
1013 }
1014
1015 #[derive(PartialEq, Eq, Debug, Clone, Copy)]
1016 enum CharClassesStatus {
1017     Normal,
1018     LitString,
1019     LitStringEscape,
1020     LitRawString(u32),
1021     RawStringPrefix(u32),
1022     RawStringSuffix(u32),
1023     LitChar,
1024     LitCharEscape,
1025     // The u32 is the nesting deepness of the comment
1026     BlockComment(u32),
1027     // Status when the '/' has been consumed, but not yet the '*', deepness is
1028     // the new deepness (after the comment opening).
1029     BlockCommentOpening(u32),
1030     // Status when the '*' has been consumed, but not yet the '/', deepness is
1031     // the new deepness (after the comment closing).
1032     BlockCommentClosing(u32),
1033     LineComment,
1034 }
1035
1036 /// Distinguish between functional part of code and comments
1037 #[derive(PartialEq, Eq, Debug, Clone, Copy)]
1038 pub enum CodeCharKind {
1039     Normal,
1040     Comment,
1041 }
1042
1043 /// Distinguish between functional part of code and comments,
1044 /// describing opening and closing of comments for ease when chunking
1045 /// code from tagged characters
1046 #[derive(PartialEq, Eq, Debug, Clone, Copy)]
1047 pub enum FullCodeCharKind {
1048     Normal,
1049     /// The first character of a comment, there is only one for a comment (always '/')
1050     StartComment,
1051     /// Any character inside a comment including the second character of comment
1052     /// marks ("//", "/*")
1053     InComment,
1054     /// Last character of a comment, '\n' for a line comment, '/' for a block comment.
1055     EndComment,
1056     /// Start of a mutlitine string
1057     StartString,
1058     /// End of a mutlitine string
1059     EndString,
1060     /// Inside a string.
1061     InString,
1062 }
1063
1064 impl FullCodeCharKind {
1065     pub fn is_comment(self) -> bool {
1066         match self {
1067             FullCodeCharKind::StartComment
1068             | FullCodeCharKind::InComment
1069             | FullCodeCharKind::EndComment => true,
1070             _ => false,
1071         }
1072     }
1073
1074     pub fn is_string(self) -> bool {
1075         self == FullCodeCharKind::InString || self == FullCodeCharKind::StartString
1076     }
1077
1078     fn to_codecharkind(self) -> CodeCharKind {
1079         if self.is_comment() {
1080             CodeCharKind::Comment
1081         } else {
1082             CodeCharKind::Normal
1083         }
1084     }
1085 }
1086
1087 impl<T> CharClasses<T>
1088 where
1089     T: Iterator,
1090     T::Item: RichChar,
1091 {
1092     pub fn new(base: T) -> CharClasses<T> {
1093         CharClasses {
1094             base: multipeek(base),
1095             status: CharClassesStatus::Normal,
1096         }
1097     }
1098 }
1099
1100 fn is_raw_string_suffix<T>(iter: &mut MultiPeek<T>, count: u32) -> bool
1101 where
1102     T: Iterator,
1103     T::Item: RichChar,
1104 {
1105     for _ in 0..count {
1106         match iter.peek() {
1107             Some(c) if c.get_char() == '#' => continue,
1108             _ => return false,
1109         }
1110     }
1111     true
1112 }
1113
1114 impl<T> Iterator for CharClasses<T>
1115 where
1116     T: Iterator,
1117     T::Item: RichChar,
1118 {
1119     type Item = (FullCodeCharKind, T::Item);
1120
1121     fn next(&mut self) -> Option<(FullCodeCharKind, T::Item)> {
1122         let item = self.base.next()?;
1123         let chr = item.get_char();
1124         let mut char_kind = FullCodeCharKind::Normal;
1125         self.status = match self.status {
1126             CharClassesStatus::LitRawString(sharps) => {
1127                 char_kind = FullCodeCharKind::InString;
1128                 match chr {
1129                     '"' => {
1130                         if sharps == 0 {
1131                             char_kind = FullCodeCharKind::Normal;
1132                             CharClassesStatus::Normal
1133                         } else if is_raw_string_suffix(&mut self.base, sharps) {
1134                             CharClassesStatus::RawStringSuffix(sharps)
1135                         } else {
1136                             CharClassesStatus::LitRawString(sharps)
1137                         }
1138                     }
1139                     _ => CharClassesStatus::LitRawString(sharps),
1140                 }
1141             }
1142             CharClassesStatus::RawStringPrefix(sharps) => {
1143                 char_kind = FullCodeCharKind::InString;
1144                 match chr {
1145                     '#' => CharClassesStatus::RawStringPrefix(sharps + 1),
1146                     '"' => CharClassesStatus::LitRawString(sharps),
1147                     _ => CharClassesStatus::Normal, // Unreachable.
1148                 }
1149             }
1150             CharClassesStatus::RawStringSuffix(sharps) => {
1151                 match chr {
1152                     '#' => {
1153                         if sharps == 1 {
1154                             CharClassesStatus::Normal
1155                         } else {
1156                             char_kind = FullCodeCharKind::InString;
1157                             CharClassesStatus::RawStringSuffix(sharps - 1)
1158                         }
1159                     }
1160                     _ => CharClassesStatus::Normal, // Unreachable
1161                 }
1162             }
1163             CharClassesStatus::LitString => {
1164                 char_kind = FullCodeCharKind::InString;
1165                 match chr {
1166                     '"' => CharClassesStatus::Normal,
1167                     '\\' => CharClassesStatus::LitStringEscape,
1168                     _ => CharClassesStatus::LitString,
1169                 }
1170             }
1171             CharClassesStatus::LitStringEscape => {
1172                 char_kind = FullCodeCharKind::InString;
1173                 CharClassesStatus::LitString
1174             }
1175             CharClassesStatus::LitChar => match chr {
1176                 '\\' => CharClassesStatus::LitCharEscape,
1177                 '\'' => CharClassesStatus::Normal,
1178                 _ => CharClassesStatus::LitChar,
1179             },
1180             CharClassesStatus::LitCharEscape => CharClassesStatus::LitChar,
1181             CharClassesStatus::Normal => match chr {
1182                 'r' => match self.base.peek().map(|c| c.get_char()) {
1183                     Some('#') | Some('"') => {
1184                         char_kind = FullCodeCharKind::InString;
1185                         CharClassesStatus::RawStringPrefix(0)
1186                     }
1187                     _ => CharClassesStatus::Normal,
1188                 },
1189                 '"' => {
1190                     char_kind = FullCodeCharKind::InString;
1191                     CharClassesStatus::LitString
1192                 }
1193                 '\'' => {
1194                     // HACK: Work around mut borrow.
1195                     match self.base.peek() {
1196                         Some(next) if next.get_char() == '\\' => {
1197                             self.status = CharClassesStatus::LitChar;
1198                             return Some((char_kind, item));
1199                         }
1200                         _ => (),
1201                     }
1202
1203                     match self.base.peek() {
1204                         Some(next) if next.get_char() == '\'' => CharClassesStatus::LitChar,
1205                         _ => CharClassesStatus::Normal,
1206                     }
1207                 }
1208                 '/' => match self.base.peek() {
1209                     Some(next) if next.get_char() == '*' => {
1210                         self.status = CharClassesStatus::BlockCommentOpening(1);
1211                         return Some((FullCodeCharKind::StartComment, item));
1212                     }
1213                     Some(next) if next.get_char() == '/' => {
1214                         self.status = CharClassesStatus::LineComment;
1215                         return Some((FullCodeCharKind::StartComment, item));
1216                     }
1217                     _ => CharClassesStatus::Normal,
1218                 },
1219                 _ => CharClassesStatus::Normal,
1220             },
1221             CharClassesStatus::BlockComment(deepness) => {
1222                 assert_ne!(deepness, 0);
1223                 self.status = match self.base.peek() {
1224                     Some(next) if next.get_char() == '/' && chr == '*' => {
1225                         CharClassesStatus::BlockCommentClosing(deepness - 1)
1226                     }
1227                     Some(next) if next.get_char() == '*' && chr == '/' => {
1228                         CharClassesStatus::BlockCommentOpening(deepness + 1)
1229                     }
1230                     _ => CharClassesStatus::BlockComment(deepness),
1231                 };
1232                 return Some((FullCodeCharKind::InComment, item));
1233             }
1234             CharClassesStatus::BlockCommentOpening(deepness) => {
1235                 assert_eq!(chr, '*');
1236                 self.status = CharClassesStatus::BlockComment(deepness);
1237                 return Some((FullCodeCharKind::InComment, item));
1238             }
1239             CharClassesStatus::BlockCommentClosing(deepness) => {
1240                 assert_eq!(chr, '/');
1241                 if deepness == 0 {
1242                     self.status = CharClassesStatus::Normal;
1243                     return Some((FullCodeCharKind::EndComment, item));
1244                 } else {
1245                     self.status = CharClassesStatus::BlockComment(deepness);
1246                     return Some((FullCodeCharKind::InComment, item));
1247                 }
1248             }
1249             CharClassesStatus::LineComment => match chr {
1250                 '\n' => {
1251                     self.status = CharClassesStatus::Normal;
1252                     return Some((FullCodeCharKind::EndComment, item));
1253                 }
1254                 _ => {
1255                     self.status = CharClassesStatus::LineComment;
1256                     return Some((FullCodeCharKind::InComment, item));
1257                 }
1258             },
1259         };
1260         Some((char_kind, item))
1261     }
1262 }
1263
1264 /// An iterator over the lines of a string, paired with the char kind at the
1265 /// end of the line.
1266 pub struct LineClasses<'a> {
1267     base: iter::Peekable<CharClasses<std::str::Chars<'a>>>,
1268     kind: FullCodeCharKind,
1269 }
1270
1271 impl<'a> LineClasses<'a> {
1272     pub fn new(s: &'a str) -> Self {
1273         LineClasses {
1274             base: CharClasses::new(s.chars()).peekable(),
1275             kind: FullCodeCharKind::Normal,
1276         }
1277     }
1278 }
1279
1280 impl<'a> Iterator for LineClasses<'a> {
1281     type Item = (FullCodeCharKind, String);
1282
1283     fn next(&mut self) -> Option<Self::Item> {
1284         self.base.peek()?;
1285
1286         let mut line = String::new();
1287
1288         let start_class = match self.base.peek() {
1289             Some((kind, _)) => *kind,
1290             None => FullCodeCharKind::Normal,
1291         };
1292
1293         while let Some((kind, c)) = self.base.next() {
1294             if c == '\n' {
1295                 self.kind = match (start_class, kind) {
1296                     (FullCodeCharKind::Normal, FullCodeCharKind::InString) => {
1297                         FullCodeCharKind::StartString
1298                     }
1299                     (FullCodeCharKind::InString, FullCodeCharKind::Normal) => {
1300                         FullCodeCharKind::EndString
1301                     }
1302                     _ => kind,
1303                 };
1304                 break;
1305             } else {
1306                 line.push(c);
1307             }
1308         }
1309
1310         Some((self.kind, line))
1311     }
1312 }
1313
1314 /// Iterator over functional and commented parts of a string. Any part of a string is either
1315 /// functional code, either *one* block comment, either *one* line comment. Whitespace between
1316 /// comments is functional code. Line comments contain their ending newlines.
1317 struct UngroupedCommentCodeSlices<'a> {
1318     slice: &'a str,
1319     iter: iter::Peekable<CharClasses<std::str::CharIndices<'a>>>,
1320 }
1321
1322 impl<'a> UngroupedCommentCodeSlices<'a> {
1323     fn new(code: &'a str) -> UngroupedCommentCodeSlices<'a> {
1324         UngroupedCommentCodeSlices {
1325             slice: code,
1326             iter: CharClasses::new(code.char_indices()).peekable(),
1327         }
1328     }
1329 }
1330
1331 impl<'a> Iterator for UngroupedCommentCodeSlices<'a> {
1332     type Item = (CodeCharKind, usize, &'a str);
1333
1334     fn next(&mut self) -> Option<Self::Item> {
1335         let (kind, (start_idx, _)) = self.iter.next()?;
1336         match kind {
1337             FullCodeCharKind::Normal | FullCodeCharKind::InString => {
1338                 // Consume all the Normal code
1339                 while let Some(&(char_kind, _)) = self.iter.peek() {
1340                     if char_kind.is_comment() {
1341                         break;
1342                     }
1343                     let _ = self.iter.next();
1344                 }
1345             }
1346             FullCodeCharKind::StartComment => {
1347                 // Consume the whole comment
1348                 while let Some((FullCodeCharKind::InComment, (_, _))) = self.iter.next() {}
1349             }
1350             _ => panic!(),
1351         }
1352         let slice = match self.iter.peek() {
1353             Some(&(_, (end_idx, _))) => &self.slice[start_idx..end_idx],
1354             None => &self.slice[start_idx..],
1355         };
1356         Some((
1357             if kind.is_comment() {
1358                 CodeCharKind::Comment
1359             } else {
1360                 CodeCharKind::Normal
1361             },
1362             start_idx,
1363             slice,
1364         ))
1365     }
1366 }
1367
1368 /// Iterator over an alternating sequence of functional and commented parts of
1369 /// a string. The first item is always a, possibly zero length, subslice of
1370 /// functional text. Line style comments contain their ending newlines.
1371 pub struct CommentCodeSlices<'a> {
1372     slice: &'a str,
1373     last_slice_kind: CodeCharKind,
1374     last_slice_end: usize,
1375 }
1376
1377 impl<'a> CommentCodeSlices<'a> {
1378     pub fn new(slice: &'a str) -> CommentCodeSlices<'a> {
1379         CommentCodeSlices {
1380             slice,
1381             last_slice_kind: CodeCharKind::Comment,
1382             last_slice_end: 0,
1383         }
1384     }
1385 }
1386
1387 impl<'a> Iterator for CommentCodeSlices<'a> {
1388     type Item = (CodeCharKind, usize, &'a str);
1389
1390     fn next(&mut self) -> Option<Self::Item> {
1391         if self.last_slice_end == self.slice.len() {
1392             return None;
1393         }
1394
1395         let mut sub_slice_end = self.last_slice_end;
1396         let mut first_whitespace = None;
1397         let subslice = &self.slice[self.last_slice_end..];
1398         let mut iter = CharClasses::new(subslice.char_indices());
1399
1400         for (kind, (i, c)) in &mut iter {
1401             let is_comment_connector = self.last_slice_kind == CodeCharKind::Normal
1402                 && &subslice[..2] == "//"
1403                 && [' ', '\t'].contains(&c);
1404
1405             if is_comment_connector && first_whitespace.is_none() {
1406                 first_whitespace = Some(i);
1407             }
1408
1409             if kind.to_codecharkind() == self.last_slice_kind && !is_comment_connector {
1410                 let last_index = match first_whitespace {
1411                     Some(j) => j,
1412                     None => i,
1413                 };
1414                 sub_slice_end = self.last_slice_end + last_index;
1415                 break;
1416             }
1417
1418             if !is_comment_connector {
1419                 first_whitespace = None;
1420             }
1421         }
1422
1423         if let (None, true) = (iter.next(), sub_slice_end == self.last_slice_end) {
1424             // This was the last subslice.
1425             sub_slice_end = match first_whitespace {
1426                 Some(i) => self.last_slice_end + i,
1427                 None => self.slice.len(),
1428             };
1429         }
1430
1431         let kind = match self.last_slice_kind {
1432             CodeCharKind::Comment => CodeCharKind::Normal,
1433             CodeCharKind::Normal => CodeCharKind::Comment,
1434         };
1435         let res = (
1436             kind,
1437             self.last_slice_end,
1438             &self.slice[self.last_slice_end..sub_slice_end],
1439         );
1440         self.last_slice_end = sub_slice_end;
1441         self.last_slice_kind = kind;
1442
1443         Some(res)
1444     }
1445 }
1446
1447 /// Checks is `new` didn't miss any comment from `span`, if it removed any, return previous text
1448 /// (if it fits in the width/offset, else return None), else return `new`
1449 pub fn recover_comment_removed(
1450     new: String,
1451     span: Span,
1452     context: &RewriteContext,
1453 ) -> Option<String> {
1454     let snippet = context.snippet(span);
1455     if snippet != new && changed_comment_content(snippet, &new) {
1456         // We missed some comments. Warn and keep the original text.
1457         if context.config.error_on_unformatted() {
1458             context.report.append(
1459                 context.source_map.span_to_filename(span).into(),
1460                 vec![FormattingError::from_span(
1461                     span,
1462                     &context.source_map,
1463                     ErrorKind::LostComment,
1464                 )],
1465             );
1466         }
1467         Some(snippet.to_owned())
1468     } else {
1469         Some(new)
1470     }
1471 }
1472
1473 pub fn filter_normal_code(code: &str) -> String {
1474     let mut buffer = String::with_capacity(code.len());
1475     LineClasses::new(code).for_each(|(kind, line)| match kind {
1476         FullCodeCharKind::Normal
1477         | FullCodeCharKind::StartString
1478         | FullCodeCharKind::InString
1479         | FullCodeCharKind::EndString => {
1480             buffer.push_str(&line);
1481             buffer.push('\n');
1482         }
1483         _ => (),
1484     });
1485     if !code.ends_with('\n') && buffer.ends_with('\n') {
1486         buffer.pop();
1487     }
1488     buffer
1489 }
1490
1491 /// Return true if the two strings of code have the same payload of comments.
1492 /// The payload of comments is everything in the string except:
1493 ///     - actual code (not comments)
1494 ///     - comment start/end marks
1495 ///     - whitespace
1496 ///     - '*' at the beginning of lines in block comments
1497 fn changed_comment_content(orig: &str, new: &str) -> bool {
1498     // Cannot write this as a fn since we cannot return types containing closures
1499     let code_comment_content = |code| {
1500         let slices = UngroupedCommentCodeSlices::new(code);
1501         slices
1502             .filter(|&(ref kind, _, _)| *kind == CodeCharKind::Comment)
1503             .flat_map(|(_, _, s)| CommentReducer::new(s))
1504     };
1505     let res = code_comment_content(orig).ne(code_comment_content(new));
1506     debug!(
1507         "comment::changed_comment_content: {}\norig: '{}'\nnew: '{}'\nraw_old: {}\nraw_new: {}",
1508         res,
1509         orig,
1510         new,
1511         code_comment_content(orig).collect::<String>(),
1512         code_comment_content(new).collect::<String>()
1513     );
1514     res
1515 }
1516
1517 /// Iterator over the 'payload' characters of a comment.
1518 /// It skips whitespace, comment start/end marks, and '*' at the beginning of lines.
1519 /// The comment must be one comment, ie not more than one start mark (no multiple line comments,
1520 /// for example).
1521 struct CommentReducer<'a> {
1522     is_block: bool,
1523     at_start_line: bool,
1524     iter: std::str::Chars<'a>,
1525 }
1526
1527 impl<'a> CommentReducer<'a> {
1528     fn new(comment: &'a str) -> CommentReducer<'a> {
1529         let is_block = comment.starts_with("/*");
1530         let comment = remove_comment_header(comment);
1531         CommentReducer {
1532             is_block,
1533             at_start_line: false, // There are no supplementary '*' on the first line
1534             iter: comment.chars(),
1535         }
1536     }
1537 }
1538
1539 impl<'a> Iterator for CommentReducer<'a> {
1540     type Item = char;
1541
1542     fn next(&mut self) -> Option<Self::Item> {
1543         loop {
1544             let mut c = self.iter.next()?;
1545             if self.is_block && self.at_start_line {
1546                 while c.is_whitespace() {
1547                     c = self.iter.next()?;
1548                 }
1549                 // Ignore leading '*'
1550                 if c == '*' {
1551                     c = self.iter.next()?;
1552                 }
1553             } else if c == '\n' {
1554                 self.at_start_line = true;
1555             }
1556             if !c.is_whitespace() {
1557                 return Some(c);
1558             }
1559         }
1560     }
1561 }
1562
1563 fn remove_comment_header(comment: &str) -> &str {
1564     if comment.starts_with("///") || comment.starts_with("//!") {
1565         &comment[3..]
1566     } else if comment.starts_with("//") {
1567         &comment[2..]
1568     } else if (comment.starts_with("/**") && !comment.starts_with("/**/"))
1569         || comment.starts_with("/*!")
1570     {
1571         &comment[3..comment.len() - 2]
1572     } else {
1573         assert!(
1574             comment.starts_with("/*"),
1575             format!("string '{}' is not a comment", comment)
1576         );
1577         &comment[2..comment.len() - 2]
1578     }
1579 }
1580
1581 #[cfg(test)]
1582 mod test {
1583     use super::*;
1584     use shape::{Indent, Shape};
1585
1586     #[test]
1587     fn char_classes() {
1588         let mut iter = CharClasses::new("//\n\n".chars());
1589
1590         assert_eq!((FullCodeCharKind::StartComment, '/'), iter.next().unwrap());
1591         assert_eq!((FullCodeCharKind::InComment, '/'), iter.next().unwrap());
1592         assert_eq!((FullCodeCharKind::EndComment, '\n'), iter.next().unwrap());
1593         assert_eq!((FullCodeCharKind::Normal, '\n'), iter.next().unwrap());
1594         assert_eq!(None, iter.next());
1595     }
1596
1597     #[test]
1598     fn comment_code_slices() {
1599         let input = "code(); /* test */ 1 + 1";
1600         let mut iter = CommentCodeSlices::new(input);
1601
1602         assert_eq!((CodeCharKind::Normal, 0, "code(); "), iter.next().unwrap());
1603         assert_eq!(
1604             (CodeCharKind::Comment, 8, "/* test */"),
1605             iter.next().unwrap()
1606         );
1607         assert_eq!((CodeCharKind::Normal, 18, " 1 + 1"), iter.next().unwrap());
1608         assert_eq!(None, iter.next());
1609     }
1610
1611     #[test]
1612     fn comment_code_slices_two() {
1613         let input = "// comment\n    test();";
1614         let mut iter = CommentCodeSlices::new(input);
1615
1616         assert_eq!((CodeCharKind::Normal, 0, ""), iter.next().unwrap());
1617         assert_eq!(
1618             (CodeCharKind::Comment, 0, "// comment\n"),
1619             iter.next().unwrap()
1620         );
1621         assert_eq!(
1622             (CodeCharKind::Normal, 11, "    test();"),
1623             iter.next().unwrap()
1624         );
1625         assert_eq!(None, iter.next());
1626     }
1627
1628     #[test]
1629     fn comment_code_slices_three() {
1630         let input = "1 // comment\n    // comment2\n\n";
1631         let mut iter = CommentCodeSlices::new(input);
1632
1633         assert_eq!((CodeCharKind::Normal, 0, "1 "), iter.next().unwrap());
1634         assert_eq!(
1635             (CodeCharKind::Comment, 2, "// comment\n    // comment2\n"),
1636             iter.next().unwrap()
1637         );
1638         assert_eq!((CodeCharKind::Normal, 29, "\n"), iter.next().unwrap());
1639         assert_eq!(None, iter.next());
1640     }
1641
1642     #[test]
1643     #[rustfmt::skip]
1644     fn format_doc_comments() {
1645         let mut wrap_normalize_config: ::config::Config = Default::default();
1646         wrap_normalize_config.set().wrap_comments(true);
1647         wrap_normalize_config.set().normalize_comments(true);
1648
1649         let mut wrap_config: ::config::Config = Default::default();
1650         wrap_config.set().wrap_comments(true);
1651
1652         let comment = rewrite_comment(" //test",
1653                                       true,
1654                                       Shape::legacy(100, Indent::new(0, 100)),
1655                                       &wrap_normalize_config).unwrap();
1656         assert_eq!("/* test */", comment);
1657
1658         let comment = rewrite_comment("// comment on a",
1659                                       false,
1660                                       Shape::legacy(10, Indent::empty()),
1661                                       &wrap_normalize_config).unwrap();
1662         assert_eq!("// comment\n// on a", comment);
1663
1664         let comment = rewrite_comment("//  A multi line comment\n             // between args.",
1665                                       false,
1666                                       Shape::legacy(60, Indent::new(0, 12)),
1667                                       &wrap_normalize_config).unwrap();
1668         assert_eq!("//  A multi line comment\n            // between args.", comment);
1669
1670         let input = "// comment";
1671         let expected =
1672             "/* comment */";
1673         let comment = rewrite_comment(input,
1674                                       true,
1675                                       Shape::legacy(9, Indent::new(0, 69)),
1676                                       &wrap_normalize_config).unwrap();
1677         assert_eq!(expected, comment);
1678
1679         let comment = rewrite_comment("/*   trimmed    */",
1680                                       true,
1681                                       Shape::legacy(100, Indent::new(0, 100)),
1682                                       &wrap_normalize_config).unwrap();
1683         assert_eq!("/* trimmed */", comment);
1684
1685         // check that different comment style are properly recognised
1686         let comment = rewrite_comment(r#"/// test1
1687                                          /// test2
1688                                          /*
1689                                           * test3
1690                                           */"#,
1691                                       false,
1692                                       Shape::legacy(100, Indent::new(0, 0)),
1693                                       &wrap_normalize_config).unwrap();
1694         assert_eq!("/// test1\n/// test2\n// test3", comment);
1695
1696         // check that the blank line marks the end of a commented paragraph
1697         let comment = rewrite_comment(r#"// test1
1698
1699                                          // test2"#,
1700                                       false,
1701                                       Shape::legacy(100, Indent::new(0, 0)),
1702                                       &wrap_normalize_config).unwrap();
1703         assert_eq!("// test1\n\n// test2", comment);
1704
1705         // check that the blank line marks the end of a custom-commented paragraph
1706         let comment = rewrite_comment(r#"//@ test1
1707
1708                                          //@ test2"#,
1709                                       false,
1710                                       Shape::legacy(100, Indent::new(0, 0)),
1711                                       &wrap_normalize_config).unwrap();
1712         assert_eq!("//@ test1\n\n//@ test2", comment);
1713
1714         // check that bare lines are just indented but left unchanged otherwise
1715         let comment = rewrite_comment(r#"// test1
1716                                          /*
1717                                            a bare line!
1718
1719                                                 another bare line!
1720                                           */"#,
1721                                       false,
1722                                       Shape::legacy(100, Indent::new(0, 0)),
1723                                       &wrap_config).unwrap();
1724         assert_eq!("// test1\n/*\n a bare line!\n\n      another bare line!\n*/", comment);
1725     }
1726
1727     // This is probably intended to be a non-test fn, but it is not used. I'm
1728     // keeping it around unless it helps us test stuff.
1729     fn uncommented(text: &str) -> String {
1730         CharClasses::new(text.chars())
1731             .filter_map(|(s, c)| match s {
1732                 FullCodeCharKind::Normal | FullCodeCharKind::InString => Some(c),
1733                 _ => None,
1734             })
1735             .collect()
1736     }
1737
1738     #[test]
1739     fn test_uncommented() {
1740         assert_eq!(&uncommented("abc/*...*/"), "abc");
1741         assert_eq!(
1742             &uncommented("// .... /* \n../* /* *** / */ */a/* // */c\n"),
1743             "..ac\n"
1744         );
1745         assert_eq!(&uncommented("abc \" /* */\" qsdf"), "abc \" /* */\" qsdf");
1746     }
1747
1748     #[test]
1749     fn test_contains_comment() {
1750         assert_eq!(contains_comment("abc"), false);
1751         assert_eq!(contains_comment("abc // qsdf"), true);
1752         assert_eq!(contains_comment("abc /* kqsdf"), true);
1753         assert_eq!(contains_comment("abc \" /* */\" qsdf"), false);
1754     }
1755
1756     #[test]
1757     fn test_find_uncommented() {
1758         fn check(haystack: &str, needle: &str, expected: Option<usize>) {
1759             assert_eq!(expected, haystack.find_uncommented(needle));
1760         }
1761
1762         check("/*/ */test", "test", Some(6));
1763         check("//test\ntest", "test", Some(7));
1764         check("/* comment only */", "whatever", None);
1765         check(
1766             "/* comment */ some text /* more commentary */ result",
1767             "result",
1768             Some(46),
1769         );
1770         check("sup // sup", "p", Some(2));
1771         check("sup", "x", None);
1772         check(r#"π? /**/ π is nice!"#, r#"π is nice"#, Some(9));
1773         check("/*sup yo? \n sup*/ sup", "p", Some(20));
1774         check("hel/*lohello*/lo", "hello", None);
1775         check("acb", "ab", None);
1776         check(",/*A*/ ", ",", Some(0));
1777         check("abc", "abc", Some(0));
1778         check("/* abc */", "abc", None);
1779         check("/**/abc/* */", "abc", Some(4));
1780         check("\"/* abc */\"", "abc", Some(4));
1781         check("\"/* abc", "abc", Some(4));
1782     }
1783
1784     #[test]
1785     fn test_filter_normal_code() {
1786         let s = r#"
1787 fn main() {
1788     println!("hello, world");
1789 }
1790 "#;
1791         assert_eq!(s, filter_normal_code(s));
1792         let s_with_comment = r#"
1793 fn main() {
1794     // hello, world
1795     println!("hello, world");
1796 }
1797 "#;
1798         assert_eq!(s, filter_normal_code(s_with_comment));
1799     }
1800 }