src/comment.rs

   1 // Copyright 2015 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 // Formatting and tools for comments.
  12
  13 use std::{self, borrow::Cow, iter};
  14
  15 use itertools::{multipeek, MultiPeek};
  16 use syntax::source_map::Span;
  17
  18 use config::Config;
  19 use rewrite::RewriteContext;
  20 use shape::{Indent, Shape};
  21 use string::{rewrite_string, StringFormat};
  22 use utils::{
  23     count_newlines, first_line_width, last_line_width, trim_left_preserve_layout, unicode_str_width,
  24 };
  25 use {ErrorKind, FormattingError};
  26
  27 fn is_custom_comment(comment: &str) -> bool {
  28     if !comment.starts_with("//") {
  29         false
  30     } else if let Some(c) = comment.chars().nth(2) {
  31         !c.is_alphanumeric() && !c.is_whitespace()
  32     } else {
  33         false
  34     }
  35 }
  36
  37 #[derive(Copy, Clone, PartialEq, Eq)]
  38 pub enum CommentStyle<'a> {
  39     DoubleSlash,
  40     TripleSlash,
  41     Doc,
  42     SingleBullet,
  43     DoubleBullet,
  44     Exclamation,
  45     Custom(&'a str),
  46 }
  47
  48 fn custom_opener(s: &str) -> &str {
  49     s.lines().next().map_or("", |first_line| {
  50         first_line
  51             .find(' ')
  52             .map_or(first_line, |space_index| &first_line[0..=space_index])
  53     })
  54 }
  55
  56 impl<'a> CommentStyle<'a> {
  57     /// Returns true if the commenting style covers a line only.
  58     pub fn is_line_comment(&self) -> bool {
  59         match *self {
  60             CommentStyle::DoubleSlash
  61             | CommentStyle::TripleSlash
  62             | CommentStyle::Doc
  63             | CommentStyle::Custom(_) => true,
  64             _ => false,
  65         }
  66     }
  67
  68     /// Returns true if the commenting style can span over multiple lines.
  69     pub fn is_block_comment(&self) -> bool {
  70         match *self {
  71             CommentStyle::SingleBullet | CommentStyle::DoubleBullet | CommentStyle::Exclamation => {
  72                 true
  73             }
  74             _ => false,
  75         }
  76     }
  77
  78     /// Returns true if the commenting style is for documentation.
  79     pub fn is_doc_comment(&self) -> bool {
  80         match *self {
  81             CommentStyle::TripleSlash | CommentStyle::Doc => true,
  82             _ => false,
  83         }
  84     }
  85
  86     pub fn opener(&self) -> &'a str {
  87         match *self {
  88             CommentStyle::DoubleSlash => "// ",
  89             CommentStyle::TripleSlash => "/// ",
  90             CommentStyle::Doc => "//! ",
  91             CommentStyle::SingleBullet => "/* ",
  92             CommentStyle::DoubleBullet => "/** ",
  93             CommentStyle::Exclamation => "/*! ",
  94             CommentStyle::Custom(opener) => opener,
  95         }
  96     }
  97
  98     pub fn closer(&self) -> &'a str {
  99         match *self {
 100             CommentStyle::DoubleSlash
 101             | CommentStyle::TripleSlash
 102             | CommentStyle::Custom(..)
 103             | CommentStyle::Doc => "",
 104             CommentStyle::DoubleBullet => " **/",
 105             CommentStyle::SingleBullet | CommentStyle::Exclamation => " */",
 106         }
 107     }
 108
 109     pub fn line_start(&self) -> &'a str {
 110         match *self {
 111             CommentStyle::DoubleSlash => "// ",
 112             CommentStyle::TripleSlash => "/// ",
 113             CommentStyle::Doc => "//! ",
 114             CommentStyle::SingleBullet | CommentStyle::Exclamation => " * ",
 115             CommentStyle::DoubleBullet => " ** ",
 116             CommentStyle::Custom(opener) => opener,
 117         }
 118     }
 119
 120     pub fn to_str_tuplet(&self) -> (&'a str, &'a str, &'a str) {
 121         (self.opener(), self.closer(), self.line_start())
 122     }
 123 }
 124
 125 fn comment_style(orig: &str, normalize_comments: bool) -> CommentStyle {
 126     if !normalize_comments {
 127         if orig.starts_with("/**") && !orig.starts_with("/**/") {
 128             CommentStyle::DoubleBullet
 129         } else if orig.starts_with("/*!") {
 130             CommentStyle::Exclamation
 131         } else if orig.starts_with("/*") {
 132             CommentStyle::SingleBullet
 133         } else if orig.starts_with("///") && orig.chars().nth(3).map_or(true, |c| c != '/') {
 134             CommentStyle::TripleSlash
 135         } else if orig.starts_with("//!") {
 136             CommentStyle::Doc
 137         } else if is_custom_comment(orig) {
 138             CommentStyle::Custom(custom_opener(orig))
 139         } else {
 140             CommentStyle::DoubleSlash
 141         }
 142     } else if (orig.starts_with("///") && orig.chars().nth(3).map_or(true, |c| c != '/'))
 143         || (orig.starts_with("/**") && !orig.starts_with("/**/"))
 144     {
 145         CommentStyle::TripleSlash
 146     } else if orig.starts_with("//!") || orig.starts_with("/*!") {
 147         CommentStyle::Doc
 148     } else if is_custom_comment(orig) {
 149         CommentStyle::Custom(custom_opener(orig))
 150     } else {
 151         CommentStyle::DoubleSlash
 152     }
 153 }
 154
 155 /// Combine `prev_str` and `next_str` into a single `String`. `span` may contain
 156 /// comments between two strings. If there are such comments, then that will be
 157 /// recovered. If `allow_extend` is true and there is no comment between the two
 158 /// strings, then they will be put on a single line as long as doing so does not
 159 /// exceed max width.
 160 pub fn combine_strs_with_missing_comments(
 161     context: &RewriteContext,
 162     prev_str: &str,
 163     next_str: &str,
 164     span: Span,
 165     shape: Shape,
 166     allow_extend: bool,
 167 ) -> Option<String> {
 168     trace!(
 169         "combine_strs_with_missing_comments `{}` `{}` {:?} {:?}",
 170         prev_str,
 171         next_str,
 172         span,
 173         shape
 174     );
 175
 176     let mut result =
 177         String::with_capacity(prev_str.len() + next_str.len() + shape.indent.width() + 128);
 178     result.push_str(prev_str);
 179     let mut allow_one_line = !prev_str.contains('\n') && !next_str.contains('\n');
 180     let first_sep = if prev_str.is_empty() || next_str.is_empty() {
 181         ""
 182     } else {
 183         " "
 184     };
 185     let mut one_line_width =
 186         last_line_width(prev_str) + first_line_width(next_str) + first_sep.len();
 187
 188     let config = context.config;
 189     let indent = shape.indent;
 190     let missing_comment = rewrite_missing_comment(span, shape, context)?;
 191
 192     if missing_comment.is_empty() {
 193         if allow_extend && prev_str.len() + first_sep.len() + next_str.len() <= shape.width {
 194             result.push_str(first_sep);
 195         } else if !prev_str.is_empty() {
 196             result.push_str(&indent.to_string_with_newline(config))
 197         }
 198         result.push_str(next_str);
 199         return Some(result);
 200     }
 201
 202     // We have a missing comment between the first expression and the second expression.
 203
 204     // Peek the the original source code and find out whether there is a newline between the first
 205     // expression and the second expression or the missing comment. We will preserve the original
 206     // layout whenever possible.
 207     let original_snippet = context.snippet(span);
 208     let prefer_same_line = if let Some(pos) = original_snippet.find('/') {
 209         !original_snippet[..pos].contains('\n')
 210     } else {
 211         !original_snippet.contains('\n')
 212     };
 213
 214     one_line_width -= first_sep.len();
 215     let first_sep = if prev_str.is_empty() || missing_comment.is_empty() {
 216         Cow::from("")
 217     } else {
 218         let one_line_width = last_line_width(prev_str) + first_line_width(&missing_comment) + 1;
 219         if prefer_same_line && one_line_width <= shape.width {
 220             Cow::from(" ")
 221         } else {
 222             indent.to_string_with_newline(config)
 223         }
 224     };
 225     result.push_str(&first_sep);
 226     result.push_str(&missing_comment);
 227
 228     let second_sep = if missing_comment.is_empty() || next_str.is_empty() {
 229         Cow::from("")
 230     } else if missing_comment.starts_with("//") {
 231         indent.to_string_with_newline(config)
 232     } else {
 233         one_line_width += missing_comment.len() + first_sep.len() + 1;
 234         allow_one_line &= !missing_comment.starts_with("//") && !missing_comment.contains('\n');
 235         if prefer_same_line && allow_one_line && one_line_width <= shape.width {
 236             Cow::from(" ")
 237         } else {
 238             indent.to_string_with_newline(config)
 239         }
 240     };
 241     result.push_str(&second_sep);
 242     result.push_str(next_str);
 243
 244     Some(result)
 245 }
 246
 247 pub fn rewrite_doc_comment(orig: &str, shape: Shape, config: &Config) -> Option<String> {
 248     identify_comment(orig, false, shape, config, true)
 249 }
 250
 251 pub fn rewrite_comment(
 252     orig: &str,
 253     block_style: bool,
 254     shape: Shape,
 255     config: &Config,
 256 ) -> Option<String> {
 257     identify_comment(orig, block_style, shape, config, false)
 258 }
 259
 260 fn identify_comment(
 261     orig: &str,
 262     block_style: bool,
 263     shape: Shape,
 264     config: &Config,
 265     is_doc_comment: bool,
 266 ) -> Option<String> {
 267     let style = comment_style(orig, false);
 268
 269     // Computes the byte length of line taking into account a newline if the line is part of a
 270     // paragraph.
 271     fn compute_len(orig: &str, line: &str) -> usize {
 272         if orig.len() > line.len() {
 273             if orig.as_bytes()[line.len()] == b'\r' {
 274                 line.len() + 2
 275             } else {
 276                 line.len() + 1
 277             }
 278         } else {
 279             line.len()
 280         }
 281     }
 282
 283     // Get the first group of line comments having the same commenting style.
 284     //
 285     // Returns a tuple with:
 286     // - a boolean indicating if there is a blank line
 287     // - a number indicating the size of the first group of comments
 288     fn consume_same_line_comments(
 289         style: CommentStyle,
 290         orig: &str,
 291         line_start: &str,
 292     ) -> (bool, usize) {
 293         let mut first_group_ending = 0;
 294         let mut hbl = false;
 295
 296         for line in orig.lines() {
 297             let trimmed_line = line.trim_start();
 298             if trimmed_line.is_empty() {
 299                 hbl = true;
 300                 break;
 301             } else if trimmed_line.starts_with(line_start)
 302                 || comment_style(trimmed_line, false) == style
 303             {
 304                 first_group_ending += compute_len(&orig[first_group_ending..], line);
 305             } else {
 306                 break;
 307             }
 308         }
 309         (hbl, first_group_ending)
 310     }
 311
 312     let (has_bare_lines, first_group_ending) = match style {
 313         CommentStyle::DoubleSlash | CommentStyle::TripleSlash | CommentStyle::Doc => {
 314             let line_start = style.line_start().trim_start();
 315             consume_same_line_comments(style, orig, line_start)
 316         }
 317         CommentStyle::Custom(opener) => {
 318             let trimmed_opener = opener.trim_end();
 319             consume_same_line_comments(style, orig, trimmed_opener)
 320         }
 321         // for a block comment, search for the closing symbol
 322         CommentStyle::DoubleBullet | CommentStyle::SingleBullet | CommentStyle::Exclamation => {
 323             let closer = style.closer().trim_start();
 324             let mut closing_symbol_offset = 0;
 325             let mut hbl = false;
 326             let mut first = true;
 327             for line in orig.lines() {
 328                 closing_symbol_offset += compute_len(&orig[closing_symbol_offset..], line);
 329                 let mut trimmed_line = line.trim_start();
 330                 if !trimmed_line.starts_with('*')
 331                     && !trimmed_line.starts_with("//")
 332                     && !trimmed_line.starts_with("/*")
 333                 {
 334                     hbl = true;
 335                 }
 336
 337                 // Remove opener from consideration when searching for closer
 338                 if first {
 339                     let opener = style.opener().trim_end();
 340                     trimmed_line = &trimmed_line[opener.len()..];
 341                     first = false;
 342                 }
 343                 if trimmed_line.ends_with(closer) {
 344                     break;
 345                 }
 346             }
 347             (hbl, closing_symbol_offset)
 348         }
 349     };
 350
 351     let (first_group, rest) = orig.split_at(first_group_ending);
 352     let rewritten_first_group =
 353         if !config.normalize_comments() && has_bare_lines && style.is_block_comment() {
 354             trim_left_preserve_layout(first_group, shape.indent, config)?
 355         } else if !config.normalize_comments()
 356             && !config.wrap_comments()
 357             && !config.format_doc_comments()
 358         {
 359             light_rewrite_comment(first_group, shape.indent, config, is_doc_comment)
 360         } else {
 361             rewrite_comment_inner(
 362                 first_group,
 363                 block_style,
 364                 style,
 365                 shape,
 366                 config,
 367                 is_doc_comment || style.is_doc_comment(),
 368             )?
 369         };
 370     if rest.is_empty() {
 371         Some(rewritten_first_group)
 372     } else {
 373         identify_comment(
 374             rest.trim_start(),
 375             block_style,
 376             shape,
 377             config,
 378             is_doc_comment,
 379         )
 380         .map(|rest_str| {
 381             format!(
 382                 "{}\n{}{}{}",
 383                 rewritten_first_group,
 384                 // insert back the blank line
 385                 if has_bare_lines && style.is_line_comment() {
 386                     "\n"
 387                 } else {
 388                     ""
 389                 },
 390                 shape.indent.to_string(config),
 391                 rest_str
 392             )
 393         })
 394     }
 395 }
 396
 397 /// Attributes for code blocks in rustdoc.
 398 /// See https://doc.rust-lang.org/rustdoc/print.html#attributes
 399 enum CodeBlockAttribute {
 400     Rust,
 401     Ignore,
 402     Text,
 403     ShouldPanic,
 404     NoRun,
 405     CompileFail,
 406 }
 407
 408 impl CodeBlockAttribute {
 409     fn new(attribute: &str) -> CodeBlockAttribute {
 410         match attribute {
 411             "rust" | "" => CodeBlockAttribute::Rust,
 412             "ignore" => CodeBlockAttribute::Ignore,
 413             "text" => CodeBlockAttribute::Text,
 414             "should_panic" => CodeBlockAttribute::ShouldPanic,
 415             "no_run" => CodeBlockAttribute::NoRun,
 416             "compile_fail" => CodeBlockAttribute::CompileFail,
 417             _ => CodeBlockAttribute::Text,
 418         }
 419     }
 420 }
 421
 422 /// Block that is formatted as an item.
 423 ///
 424 /// An item starts with either a star `*` or a dash `-`. Different level of indentation are
 425 /// handled by shrinking the shape accordingly.
 426 struct ItemizedBlock {
 427     /// the lines that are identified as part of an itemized block
 428     lines: Vec<String>,
 429     /// the number of whitespaces up to the item sigil
 430     indent: usize,
 431     /// the string that marks the start of an item
 432     opener: String,
 433     /// sequence of whitespaces to prefix new lines that are part of the item
 434     line_start: String,
 435 }
 436
 437 impl ItemizedBlock {
 438     /// Returns true if the line is formatted as an item
 439     fn is_itemized_line(line: &str) -> bool {
 440         let trimmed = line.trim_start();
 441         trimmed.starts_with("* ") || trimmed.starts_with("- ")
 442     }
 443
 444     /// Creates a new ItemizedBlock described with the given line.
 445     /// The `is_itemized_line` needs to be called first.
 446     fn new(line: &str) -> ItemizedBlock {
 447         let space_to_sigil = line.chars().take_while(|c| c.is_whitespace()).count();
 448         let indent = space_to_sigil + 2;
 449         ItemizedBlock {
 450             lines: vec![line[indent..].to_string()],
 451             indent,
 452             opener: line[..indent].to_string(),
 453             line_start: " ".repeat(indent),
 454         }
 455     }
 456
 457     /// Returns a `StringFormat` used for formatting the content of an item
 458     fn create_string_format<'a>(&'a self, fmt: &'a StringFormat) -> StringFormat<'a> {
 459         StringFormat {
 460             opener: "",
 461             closer: "",
 462             line_start: "",
 463             line_end: "",
 464             shape: Shape::legacy(fmt.shape.width.saturating_sub(self.indent), Indent::empty()),
 465             trim_end: true,
 466             config: fmt.config,
 467         }
 468     }
 469
 470     /// Returns true if the line is part of the current itemized block.
 471     /// If it is, then it is added to the internal lines vec.
 472     fn add_line(&mut self, line: &str) -> bool {
 473         if !ItemizedBlock::is_itemized_line(line)
 474             && self.indent <= line.chars().take_while(|c| c.is_whitespace()).count()
 475         {
 476             self.lines.push(line.to_string());
 477             return true;
 478         }
 479         false
 480     }
 481
 482     /// Returns the block as a string, with each line trimmed at the start.
 483     fn trimmed_block_as_string(&self) -> String {
 484         self.lines
 485             .iter()
 486             .map(|line| format!("{} ", line.trim_start()))
 487             .collect::<String>()
 488     }
 489
 490     /// Returns the block as a string under its original form
 491     fn original_block_as_string(&self) -> String {
 492         self.lines.join("\n")
 493     }
 494 }
 495
 496 struct CommentRewrite<'a> {
 497     result: String,
 498     code_block_buffer: String,
 499     is_prev_line_multi_line: bool,
 500     code_block_attr: Option<CodeBlockAttribute>,
 501     item_block: Option<ItemizedBlock>,
 502     comment_line_separator: String,
 503     indent_str: String,
 504     max_width: usize,
 505     fmt_indent: Indent,
 506     fmt: StringFormat<'a>,
 507
 508     opener: String,
 509     closer: String,
 510     line_start: String,
 511 }
 512
 513 impl<'a> CommentRewrite<'a> {
 514     fn new(
 515         orig: &'a str,
 516         block_style: bool,
 517         shape: Shape,
 518         config: &'a Config,
 519     ) -> CommentRewrite<'a> {
 520         let (opener, closer, line_start) = if block_style {
 521             CommentStyle::SingleBullet.to_str_tuplet()
 522         } else {
 523             comment_style(orig, config.normalize_comments()).to_str_tuplet()
 524         };
 525
 526         let max_width = shape
 527             .width
 528             .checked_sub(closer.len() + opener.len())
 529             .unwrap_or(1);
 530         let indent_str = shape.indent.to_string_with_newline(config).to_string();
 531         let fmt_indent = shape.indent + (opener.len() - line_start.len());
 532
 533         let mut cr = CommentRewrite {
 534             result: String::with_capacity(orig.len() * 2),
 535             code_block_buffer: String::with_capacity(128),
 536             is_prev_line_multi_line: false,
 537             code_block_attr: None,
 538             item_block: None,
 539             comment_line_separator: format!("{}{}", indent_str, line_start),
 540             max_width,
 541             indent_str,
 542             fmt_indent,
 543
 544             fmt: StringFormat {
 545                 opener: "",
 546                 closer: "",
 547                 line_start,
 548                 line_end: "",
 549                 shape: Shape::legacy(max_width, fmt_indent),
 550                 trim_end: true,
 551                 config,
 552             },
 553
 554             opener: opener.to_owned(),
 555             closer: closer.to_owned(),
 556             line_start: line_start.to_owned(),
 557         };
 558         cr.result.push_str(opener);
 559         cr
 560     }
 561
 562     fn join_block(s: &str, sep: &str) -> String {
 563         let mut result = String::with_capacity(s.len() + 128);
 564         let mut iter = s.lines().peekable();
 565         while let Some(line) = iter.next() {
 566             result.push_str(line);
 567             result.push_str(match iter.peek() {
 568                 Some(next_line) if next_line.is_empty() => sep.trim_end(),
 569                 Some(..) => &sep,
 570                 None => "",
 571             });
 572         }
 573         result
 574     }
 575
 576     fn finish(mut self) -> String {
 577         if !self.code_block_buffer.is_empty() {
 578             // There is a code block that is not properly enclosed by backticks.
 579             // We will leave them untouched.
 580             self.result.push_str(&self.comment_line_separator);
 581             self.result.push_str(&Self::join_block(
 582                 &trim_custom_comment_prefix(&self.code_block_buffer),
 583                 &self.comment_line_separator,
 584             ));
 585         }
 586
 587         if let Some(ref ib) = self.item_block {
 588             // the last few lines are part of an itemized block
 589             self.fmt.shape = Shape::legacy(self.max_width, self.fmt_indent);
 590             let item_fmt = ib.create_string_format(&self.fmt);
 591             self.result.push_str(&self.comment_line_separator);
 592             self.result.push_str(&ib.opener);
 593             match rewrite_string(
 594                 &ib.trimmed_block_as_string(),
 595                 &item_fmt,
 596                 self.max_width.saturating_sub(ib.indent),
 597             ) {
 598                 Some(s) => self.result.push_str(&Self::join_block(
 599                     &s,
 600                     &format!("{}{}", &self.comment_line_separator, ib.line_start),
 601                 )),
 602                 None => self.result.push_str(&Self::join_block(
 603                     &ib.original_block_as_string(),
 604                     &self.comment_line_separator,
 605                 )),
 606             };
 607         }
 608
 609         self.result.push_str(&self.closer);
 610         if self.result.ends_with(&self.opener) && self.opener.ends_with(' ') {
 611             // Trailing space.
 612             self.result.pop();
 613         }
 614
 615         self.result
 616     }
 617
 618     fn handle_line(
 619         &mut self,
 620         orig: &'a str,
 621         i: usize,
 622         line: &'a str,
 623         has_leading_whitespace: bool,
 624     ) -> bool {
 625         let is_last = i == count_newlines(orig);
 626
 627         if let Some(ref mut ib) = self.item_block {
 628             if ib.add_line(&line) {
 629                 return false;
 630             }
 631             self.is_prev_line_multi_line = false;
 632             self.fmt.shape = Shape::legacy(self.max_width, self.fmt_indent);
 633             let item_fmt = ib.create_string_format(&self.fmt);
 634             self.result.push_str(&self.comment_line_separator);
 635             self.result.push_str(&ib.opener);
 636             match rewrite_string(
 637                 &ib.trimmed_block_as_string(),
 638                 &item_fmt,
 639                 self.max_width.saturating_sub(ib.indent),
 640             ) {
 641                 Some(s) => self.result.push_str(&Self::join_block(
 642                     &s,
 643                     &format!("{}{}", &self.comment_line_separator, ib.line_start),
 644                 )),
 645                 None => self.result.push_str(&Self::join_block(
 646                     &ib.original_block_as_string(),
 647                     &self.comment_line_separator,
 648                 )),
 649             };
 650         } else if self.code_block_attr.is_some() {
 651             if line.starts_with("```") {
 652                 let code_block = match self.code_block_attr.as_ref().unwrap() {
 653                     CodeBlockAttribute::Ignore | CodeBlockAttribute::Text => {
 654                         trim_custom_comment_prefix(&self.code_block_buffer)
 655                     }
 656                     _ if self.code_block_buffer.is_empty() => String::new(),
 657                     _ => {
 658                         let mut config = self.fmt.config.clone();
 659                         config.set().wrap_comments(false);
 660                         match ::format_code_block(&self.code_block_buffer, &config) {
 661                             Some(ref s) => trim_custom_comment_prefix(&s.snippet),
 662                             None => trim_custom_comment_prefix(&self.code_block_buffer),
 663                         }
 664                     }
 665                 };
 666                 if !code_block.is_empty() {
 667                     self.result.push_str(&self.comment_line_separator);
 668                     self.result
 669                         .push_str(&Self::join_block(&code_block, &self.comment_line_separator));
 670                 }
 671                 self.code_block_buffer.clear();
 672                 self.result.push_str(&self.comment_line_separator);
 673                 self.result.push_str(line);
 674                 self.code_block_attr = None;
 675             } else {
 676                 self.code_block_buffer
 677                     .push_str(&hide_sharp_behind_comment(line));
 678                 self.code_block_buffer.push('\n');
 679             }
 680             return false;
 681         }
 682
 683         self.code_block_attr = None;
 684         self.item_block = None;
 685         if line.starts_with("```") {
 686             self.code_block_attr = Some(CodeBlockAttribute::new(&line[3..]))
 687         } else if self.fmt.config.wrap_comments() && ItemizedBlock::is_itemized_line(&line) {
 688             let ib = ItemizedBlock::new(&line);
 689             self.item_block = Some(ib);
 690             return false;
 691         }
 692
 693         if self.result == self.opener {
 694             let force_leading_whitespace = &self.opener == "/* " && count_newlines(orig) == 0;
 695             if !has_leading_whitespace && !force_leading_whitespace && self.result.ends_with(' ') {
 696                 self.result.pop();
 697             }
 698             if line.is_empty() {
 699                 return false;
 700             }
 701         } else if self.is_prev_line_multi_line && !line.is_empty() {
 702             self.result.push(' ')
 703         } else if is_last && line.is_empty() {
 704             // trailing blank lines are unwanted
 705             if !self.closer.is_empty() {
 706                 self.result.push_str(&self.indent_str);
 707             }
 708             return true;
 709         } else {
 710             self.result.push_str(&self.comment_line_separator);
 711             if !has_leading_whitespace && self.result.ends_with(' ') {
 712                 self.result.pop();
 713             }
 714         }
 715
 716         if self.fmt.config.wrap_comments()
 717             && unicode_str_width(line) > self.fmt.shape.width
 718             && !has_url(line)
 719         {
 720             match rewrite_string(line, &self.fmt, self.max_width) {
 721                 Some(ref s) => {
 722                     self.is_prev_line_multi_line = s.contains('\n');
 723                     self.result.push_str(s);
 724                 }
 725                 None if self.is_prev_line_multi_line => {
 726                     // We failed to put the current `line` next to the previous `line`.
 727                     // Remove the trailing space, then start rewrite on the next line.
 728                     self.result.pop();
 729                     self.result.push_str(&self.comment_line_separator);
 730                     self.fmt.shape = Shape::legacy(self.max_width, self.fmt_indent);
 731                     match rewrite_string(line, &self.fmt, self.max_width) {
 732                         Some(ref s) => {
 733                             self.is_prev_line_multi_line = s.contains('\n');
 734                             self.result.push_str(s);
 735                         }
 736                         None => {
 737                             self.is_prev_line_multi_line = false;
 738                             self.result.push_str(line);
 739                         }
 740                     }
 741                 }
 742                 None => {
 743                     self.is_prev_line_multi_line = false;
 744                     self.result.push_str(line);
 745                 }
 746             }
 747
 748             self.fmt.shape = if self.is_prev_line_multi_line {
 749                 // 1 = " "
 750                 let offset = 1 + last_line_width(&self.result) - self.line_start.len();
 751                 Shape {
 752                     width: self.max_width.saturating_sub(offset),
 753                     indent: self.fmt_indent,
 754                     offset: self.fmt.shape.offset + offset,
 755                 }
 756             } else {
 757                 Shape::legacy(self.max_width, self.fmt_indent)
 758             };
 759         } else {
 760             if line.is_empty() && self.result.ends_with(' ') && !is_last {
 761                 // Remove space if this is an empty comment or a doc comment.
 762                 self.result.pop();
 763             }
 764             self.result.push_str(line);
 765             self.fmt.shape = Shape::legacy(self.max_width, self.fmt_indent);
 766             self.is_prev_line_multi_line = false;
 767         }
 768
 769         false
 770     }
 771 }
 772
 773 fn rewrite_comment_inner(
 774     orig: &str,
 775     block_style: bool,
 776     style: CommentStyle,
 777     shape: Shape,
 778     config: &Config,
 779     is_doc_comment: bool,
 780 ) -> Option<String> {
 781     let mut rewriter = CommentRewrite::new(orig, block_style, shape, config);
 782
 783     let line_breaks = count_newlines(orig.trim_end());
 784     let lines = orig
 785         .lines()
 786         .enumerate()
 787         .map(|(i, mut line)| {
 788             line = trim_end_unless_two_whitespaces(line.trim_start(), is_doc_comment);
 789             // Drop old closer.
 790             if i == line_breaks && line.ends_with("*/") && !line.starts_with("//") {
 791                 line = line[..(line.len() - 2)].trim_end();
 792             }
 793
 794             line
 795         })
 796         .map(|s| left_trim_comment_line(s, &style))
 797         .map(|(line, has_leading_whitespace)| {
 798             if orig.starts_with("/*") && line_breaks == 0 {
 799                 (
 800                     line.trim_start(),
 801                     has_leading_whitespace || config.normalize_comments(),
 802                 )
 803             } else {
 804                 (line, has_leading_whitespace || config.normalize_comments())
 805             }
 806         });
 807
 808     for (i, (line, has_leading_whitespace)) in lines.enumerate() {
 809         if rewriter.handle_line(orig, i, line, has_leading_whitespace) {
 810             break;
 811         }
 812     }
 813
 814     Some(rewriter.finish())
 815 }
 816
 817 const RUSTFMT_CUSTOM_COMMENT_PREFIX: &str = "//#### ";
 818
 819 fn hide_sharp_behind_comment(s: &str) -> Cow<str> {
 820     if s.trim_start().starts_with("# ") {
 821         Cow::from(format!("{}{}", RUSTFMT_CUSTOM_COMMENT_PREFIX, s))
 822     } else {
 823         Cow::from(s)
 824     }
 825 }
 826
 827 fn trim_custom_comment_prefix(s: &str) -> String {
 828     s.lines()
 829         .map(|line| {
 830             let left_trimmed = line.trim_start();
 831             if left_trimmed.starts_with(RUSTFMT_CUSTOM_COMMENT_PREFIX) {
 832                 left_trimmed.trim_start_matches(RUSTFMT_CUSTOM_COMMENT_PREFIX)
 833             } else {
 834                 line
 835             }
 836         })
 837         .collect::<Vec<_>>()
 838         .join("\n")
 839 }
 840
 841 /// Returns true if the given string MAY include URLs or alike.
 842 fn has_url(s: &str) -> bool {
 843     // This function may return false positive, but should get its job done in most cases.
 844     s.contains("https://") || s.contains("http://") || s.contains("ftp://") || s.contains("file://")
 845 }
 846
 847 /// Given the span, rewrite the missing comment inside it if available.
 848 /// Note that the given span must only include comments (or leading/trailing whitespaces).
 849 pub fn rewrite_missing_comment(
 850     span: Span,
 851     shape: Shape,
 852     context: &RewriteContext,
 853 ) -> Option<String> {
 854     let missing_snippet = context.snippet(span);
 855     let trimmed_snippet = missing_snippet.trim();
 856     if !trimmed_snippet.is_empty() {
 857         rewrite_comment(trimmed_snippet, false, shape, context.config)
 858     } else {
 859         Some(String::new())
 860     }
 861 }
 862
 863 /// Recover the missing comments in the specified span, if available.
 864 /// The layout of the comments will be preserved as long as it does not break the code
 865 /// and its total width does not exceed the max width.
 866 pub fn recover_missing_comment_in_span(
 867     span: Span,
 868     shape: Shape,
 869     context: &RewriteContext,
 870     used_width: usize,
 871 ) -> Option<String> {
 872     let missing_comment = rewrite_missing_comment(span, shape, context)?;
 873     if missing_comment.is_empty() {
 874         Some(String::new())
 875     } else {
 876         let missing_snippet = context.snippet(span);
 877         let pos = missing_snippet.find('/').unwrap_or(0);
 878         // 1 = ` `
 879         let total_width = missing_comment.len() + used_width + 1;
 880         let force_new_line_before_comment =
 881             missing_snippet[..pos].contains('\n') || total_width > context.config.max_width();
 882         let sep = if force_new_line_before_comment {
 883             shape.indent.to_string_with_newline(context.config)
 884         } else {
 885             Cow::from(" ")
 886         };
 887         Some(format!("{}{}", sep, missing_comment))
 888     }
 889 }
 890
 891 /// Trim trailing whitespaces unless they consist of two or more whitespaces.
 892 fn trim_end_unless_two_whitespaces(s: &str, is_doc_comment: bool) -> &str {
 893     if is_doc_comment && s.ends_with("  ") {
 894         s
 895     } else {
 896         s.trim_end()
 897     }
 898 }
 899
 900 /// Trims whitespace and aligns to indent, but otherwise does not change comments.
 901 fn light_rewrite_comment(
 902     orig: &str,
 903     offset: Indent,
 904     config: &Config,
 905     is_doc_comment: bool,
 906 ) -> String {
 907     let lines: Vec<&str> = orig
 908         .lines()
 909         .map(|l| {
 910             // This is basically just l.trim(), but in the case that a line starts
 911             // with `*` we want to leave one space before it, so it aligns with the
 912             // `*` in `/*`.
 913             let first_non_whitespace = l.find(|c| !char::is_whitespace(c));
 914             let left_trimmed = if let Some(fnw) = first_non_whitespace {
 915                 if l.as_bytes()[fnw] == b'*' && fnw > 0 {
 916                     &l[fnw - 1..]
 917                 } else {
 918                     &l[fnw..]
 919                 }
 920             } else {
 921                 ""
 922             };
 923             // Preserve markdown's double-space line break syntax in doc comment.
 924             trim_end_unless_two_whitespaces(left_trimmed, is_doc_comment)
 925         })
 926         .collect();
 927     lines.join(&format!("\n{}", offset.to_string(config)))
 928 }
 929
 930 /// Trims comment characters and possibly a single space from the left of a string.
 931 /// Does not trim all whitespace. If a single space is trimmed from the left of the string,
 932 /// this function returns true.
 933 fn left_trim_comment_line<'a>(line: &'a str, style: &CommentStyle) -> (&'a str, bool) {
 934     if line.starts_with("//! ")
 935         || line.starts_with("/// ")
 936         || line.starts_with("/*! ")
 937         || line.starts_with("/** ")
 938     {
 939         (&line[4..], true)
 940     } else if let CommentStyle::Custom(opener) = *style {
 941         if line.starts_with(opener) {
 942             (&line[opener.len()..], true)
 943         } else {
 944             (&line[opener.trim_end().len()..], false)
 945         }
 946     } else if line.starts_with("/* ")
 947         || line.starts_with("// ")
 948         || line.starts_with("//!")
 949         || line.starts_with("///")
 950         || line.starts_with("** ")
 951         || line.starts_with("/*!")
 952         || (line.starts_with("/**") && !line.starts_with("/**/"))
 953     {
 954         (&line[3..], line.chars().nth(2).unwrap() == ' ')
 955     } else if line.starts_with("/*")
 956         || line.starts_with("* ")
 957         || line.starts_with("//")
 958         || line.starts_with("**")
 959     {
 960         (&line[2..], line.chars().nth(1).unwrap() == ' ')
 961     } else if line.starts_with('*') {
 962         (&line[1..], false)
 963     } else {
 964         (line, line.starts_with(' '))
 965     }
 966 }
 967
 968 pub trait FindUncommented {
 969     fn find_uncommented(&self, pat: &str) -> Option<usize>;
 970 }
 971
 972 impl FindUncommented for str {
 973     fn find_uncommented(&self, pat: &str) -> Option<usize> {
 974         let mut needle_iter = pat.chars();
 975         for (kind, (i, b)) in CharClasses::new(self.char_indices()) {
 976             match needle_iter.next() {
 977                 None => {
 978                     return Some(i - pat.len());
 979                 }
 980                 Some(c) => match kind {
 981                     FullCodeCharKind::Normal | FullCodeCharKind::InString if b == c => {}
 982                     _ => {
 983                         needle_iter = pat.chars();
 984                     }
 985                 },
 986             }
 987         }
 988
 989         // Handle case where the pattern is a suffix of the search string
 990         match needle_iter.next() {
 991             Some(_) => None,
 992             None => Some(self.len() - pat.len()),
 993         }
 994     }
 995 }
 996
 997 // Returns the first byte position after the first comment. The given string
 998 // is expected to be prefixed by a comment, including delimiters.
 999 // Good: "/* /* inner */ outer */ code();"
1000 // Bad:  "code(); // hello\n world!"
1001 pub fn find_comment_end(s: &str) -> Option<usize> {
1002     let mut iter = CharClasses::new(s.char_indices());
1003     for (kind, (i, _c)) in &mut iter {
1004         if kind == FullCodeCharKind::Normal || kind == FullCodeCharKind::InString {
1005             return Some(i);
1006         }
1007     }
1008
1009     // Handle case where the comment ends at the end of s.
1010     if iter.status == CharClassesStatus::Normal {
1011         Some(s.len())
1012     } else {
1013         None
1014     }
1015 }
1016
1017 /// Returns true if text contains any comment.
1018 pub fn contains_comment(text: &str) -> bool {
1019     CharClasses::new(text.chars()).any(|(kind, _)| kind.is_comment())
1020 }
1021
1022 pub struct CharClasses<T>
1023 where
1024     T: Iterator,
1025     T::Item: RichChar,
1026 {
1027     base: MultiPeek<T>,
1028     status: CharClassesStatus,
1029 }
1030
1031 pub trait RichChar {
1032     fn get_char(&self) -> char;
1033 }
1034
1035 impl RichChar for char {
1036     fn get_char(&self) -> char {
1037         *self
1038     }
1039 }
1040
1041 impl RichChar for (usize, char) {
1042     fn get_char(&self) -> char {
1043         self.1
1044     }
1045 }
1046
1047 #[derive(PartialEq, Eq, Debug, Clone, Copy)]
1048 enum CharClassesStatus {
1049     Normal,
1050     /// Character is within a string
1051     LitString,
1052     LitStringEscape,
1053     /// Character is within a raw string
1054     LitRawString(u32),
1055     RawStringPrefix(u32),
1056     RawStringSuffix(u32),
1057     LitChar,
1058     LitCharEscape,
1059     /// Character inside a block comment, with the integer indicating the nesting deepness of the
1060     /// comment
1061     BlockComment(u32),
1062     /// Character inside a block-commented string, with the integer indicating the nesting deepness
1063     /// of the comment
1064     StringInBlockComment(u32),
1065     /// Status when the '/' has been consumed, but not yet the '*', deepness is
1066     /// the new deepness (after the comment opening).
1067     BlockCommentOpening(u32),
1068     /// Status when the '*' has been consumed, but not yet the '/', deepness is
1069     /// the new deepness (after the comment closing).
1070     BlockCommentClosing(u32),
1071     /// Character is within a line comment
1072     LineComment,
1073 }
1074
1075 /// Distinguish between functional part of code and comments
1076 #[derive(PartialEq, Eq, Debug, Clone, Copy)]
1077 pub enum CodeCharKind {
1078     Normal,
1079     Comment,
1080 }
1081
1082 /// Distinguish between functional part of code and comments,
1083 /// describing opening and closing of comments for ease when chunking
1084 /// code from tagged characters
1085 #[derive(PartialEq, Eq, Debug, Clone, Copy)]
1086 pub enum FullCodeCharKind {
1087     Normal,
1088     /// The first character of a comment, there is only one for a comment (always '/')
1089     StartComment,
1090     /// Any character inside a comment including the second character of comment
1091     /// marks ("//", "/*")
1092     InComment,
1093     /// Last character of a comment, '\n' for a line comment, '/' for a block comment.
1094     EndComment,
1095     /// Start of a mutlitine string inside a comment
1096     StartStringCommented,
1097     /// End of a mutlitine string inside a comment
1098     EndStringCommented,
1099     /// Inside a commented string
1100     InStringCommented,
1101     /// Start of a mutlitine string
1102     StartString,
1103     /// End of a mutlitine string
1104     EndString,
1105     /// Inside a string.
1106     InString,
1107 }
1108
1109 impl FullCodeCharKind {
1110     pub fn is_comment(self) -> bool {
1111         match self {
1112             FullCodeCharKind::StartComment
1113             | FullCodeCharKind::InComment
1114             | FullCodeCharKind::EndComment
1115             | FullCodeCharKind::StartStringCommented
1116             | FullCodeCharKind::InStringCommented
1117             | FullCodeCharKind::EndStringCommented => true,
1118             _ => false,
1119         }
1120     }
1121
1122     /// Returns true if the character is inside a comment
1123     pub fn inside_comment(self) -> bool {
1124         match self {
1125             FullCodeCharKind::InComment
1126             | FullCodeCharKind::StartStringCommented
1127             | FullCodeCharKind::InStringCommented
1128             | FullCodeCharKind::EndStringCommented => true,
1129             _ => false,
1130         }
1131     }
1132
1133     pub fn is_string(self) -> bool {
1134         self == FullCodeCharKind::InString || self == FullCodeCharKind::StartString
1135     }
1136
1137     /// Returns true if the character is within a commented string
1138     pub fn is_commented_string(self) -> bool {
1139         self == FullCodeCharKind::InStringCommented
1140             || self == FullCodeCharKind::StartStringCommented
1141     }
1142
1143     fn to_codecharkind(self) -> CodeCharKind {
1144         if self.is_comment() {
1145             CodeCharKind::Comment
1146         } else {
1147             CodeCharKind::Normal
1148         }
1149     }
1150 }
1151
1152 impl<T> CharClasses<T>
1153 where
1154     T: Iterator,
1155     T::Item: RichChar,
1156 {
1157     pub fn new(base: T) -> CharClasses<T> {
1158         CharClasses {
1159             base: multipeek(base),
1160             status: CharClassesStatus::Normal,
1161         }
1162     }
1163 }
1164
1165 fn is_raw_string_suffix<T>(iter: &mut MultiPeek<T>, count: u32) -> bool
1166 where
1167     T: Iterator,
1168     T::Item: RichChar,
1169 {
1170     for _ in 0..count {
1171         match iter.peek() {
1172             Some(c) if c.get_char() == '#' => continue,
1173             _ => return false,
1174         }
1175     }
1176     true
1177 }
1178
1179 impl<T> Iterator for CharClasses<T>
1180 where
1181     T: Iterator,
1182     T::Item: RichChar,
1183 {
1184     type Item = (FullCodeCharKind, T::Item);
1185
1186     fn next(&mut self) -> Option<(FullCodeCharKind, T::Item)> {
1187         let item = self.base.next()?;
1188         let chr = item.get_char();
1189         let mut char_kind = FullCodeCharKind::Normal;
1190         self.status = match self.status {
1191             CharClassesStatus::LitRawString(sharps) => {
1192                 char_kind = FullCodeCharKind::InString;
1193                 match chr {
1194                     '"' => {
1195                         if sharps == 0 {
1196                             char_kind = FullCodeCharKind::Normal;
1197                             CharClassesStatus::Normal
1198                         } else if is_raw_string_suffix(&mut self.base, sharps) {
1199                             CharClassesStatus::RawStringSuffix(sharps)
1200                         } else {
1201                             CharClassesStatus::LitRawString(sharps)
1202                         }
1203                     }
1204                     _ => CharClassesStatus::LitRawString(sharps),
1205                 }
1206             }
1207             CharClassesStatus::RawStringPrefix(sharps) => {
1208                 char_kind = FullCodeCharKind::InString;
1209                 match chr {
1210                     '#' => CharClassesStatus::RawStringPrefix(sharps + 1),
1211                     '"' => CharClassesStatus::LitRawString(sharps),
1212                     _ => CharClassesStatus::Normal, // Unreachable.
1213                 }
1214             }
1215             CharClassesStatus::RawStringSuffix(sharps) => {
1216                 match chr {
1217                     '#' => {
1218                         if sharps == 1 {
1219                             CharClassesStatus::Normal
1220                         } else {
1221                             char_kind = FullCodeCharKind::InString;
1222                             CharClassesStatus::RawStringSuffix(sharps - 1)
1223                         }
1224                     }
1225                     _ => CharClassesStatus::Normal, // Unreachable
1226                 }
1227             }
1228             CharClassesStatus::LitString => {
1229                 char_kind = FullCodeCharKind::InString;
1230                 match chr {
1231                     '"' => CharClassesStatus::Normal,
1232                     '\\' => CharClassesStatus::LitStringEscape,
1233                     _ => CharClassesStatus::LitString,
1234                 }
1235             }
1236             CharClassesStatus::LitStringEscape => {
1237                 char_kind = FullCodeCharKind::InString;
1238                 CharClassesStatus::LitString
1239             }
1240             CharClassesStatus::LitChar => match chr {
1241                 '\\' => CharClassesStatus::LitCharEscape,
1242                 '\'' => CharClassesStatus::Normal,
1243                 _ => CharClassesStatus::LitChar,
1244             },
1245             CharClassesStatus::LitCharEscape => CharClassesStatus::LitChar,
1246             CharClassesStatus::Normal => match chr {
1247                 'r' => match self.base.peek().map(|c| c.get_char()) {
1248                     Some('#') | Some('"') => {
1249                         char_kind = FullCodeCharKind::InString;
1250                         CharClassesStatus::RawStringPrefix(0)
1251                     }
1252                     _ => CharClassesStatus::Normal,
1253                 },
1254                 '"' => {
1255                     char_kind = FullCodeCharKind::InString;
1256                     CharClassesStatus::LitString
1257                 }
1258                 '\'' => {
1259                     // HACK: Work around mut borrow.
1260                     match self.base.peek() {
1261                         Some(next) if next.get_char() == '\\' => {
1262                             self.status = CharClassesStatus::LitChar;
1263                             return Some((char_kind, item));
1264                         }
1265                         _ => (),
1266                     }
1267
1268                     match self.base.peek() {
1269                         Some(next) if next.get_char() == '\'' => CharClassesStatus::LitChar,
1270                         _ => CharClassesStatus::Normal,
1271                     }
1272                 }
1273                 '/' => match self.base.peek() {
1274                     Some(next) if next.get_char() == '*' => {
1275                         self.status = CharClassesStatus::BlockCommentOpening(1);
1276                         return Some((FullCodeCharKind::StartComment, item));
1277                     }
1278                     Some(next) if next.get_char() == '/' => {
1279                         self.status = CharClassesStatus::LineComment;
1280                         return Some((FullCodeCharKind::StartComment, item));
1281                     }
1282                     _ => CharClassesStatus::Normal,
1283                 },
1284                 _ => CharClassesStatus::Normal,
1285             },
1286             CharClassesStatus::StringInBlockComment(deepness) => {
1287                 char_kind = FullCodeCharKind::InStringCommented;
1288                 if chr == '"' {
1289                     CharClassesStatus::BlockComment(deepness)
1290                 } else {
1291                     CharClassesStatus::StringInBlockComment(deepness)
1292                 }
1293             }
1294             CharClassesStatus::BlockComment(deepness) => {
1295                 assert_ne!(deepness, 0);
1296                 char_kind = FullCodeCharKind::InComment;
1297                 match self.base.peek() {
1298                     Some(next) if next.get_char() == '/' && chr == '*' => {
1299                         CharClassesStatus::BlockCommentClosing(deepness - 1)
1300                     }
1301                     Some(next) if next.get_char() == '*' && chr == '/' => {
1302                         CharClassesStatus::BlockCommentOpening(deepness + 1)
1303                     }
1304                     _ if chr == '"' => CharClassesStatus::StringInBlockComment(deepness),
1305                     _ => self.status,
1306                 }
1307             }
1308             CharClassesStatus::BlockCommentOpening(deepness) => {
1309                 assert_eq!(chr, '*');
1310                 self.status = CharClassesStatus::BlockComment(deepness);
1311                 return Some((FullCodeCharKind::InComment, item));
1312             }
1313             CharClassesStatus::BlockCommentClosing(deepness) => {
1314                 assert_eq!(chr, '/');
1315                 if deepness == 0 {
1316                     self.status = CharClassesStatus::Normal;
1317                     return Some((FullCodeCharKind::EndComment, item));
1318                 } else {
1319                     self.status = CharClassesStatus::BlockComment(deepness);
1320                     return Some((FullCodeCharKind::InComment, item));
1321                 }
1322             }
1323             CharClassesStatus::LineComment => match chr {
1324                 '\n' => {
1325                     self.status = CharClassesStatus::Normal;
1326                     return Some((FullCodeCharKind::EndComment, item));
1327                 }
1328                 _ => {
1329                     self.status = CharClassesStatus::LineComment;
1330                     return Some((FullCodeCharKind::InComment, item));
1331                 }
1332             },
1333         };
1334         Some((char_kind, item))
1335     }
1336 }
1337
1338 /// An iterator over the lines of a string, paired with the char kind at the
1339 /// end of the line.
1340 pub struct LineClasses<'a> {
1341     base: iter::Peekable<CharClasses<std::str::Chars<'a>>>,
1342     kind: FullCodeCharKind,
1343 }
1344
1345 impl<'a> LineClasses<'a> {
1346     pub fn new(s: &'a str) -> Self {
1347         LineClasses {
1348             base: CharClasses::new(s.chars()).peekable(),
1349             kind: FullCodeCharKind::Normal,
1350         }
1351     }
1352 }
1353
1354 impl<'a> Iterator for LineClasses<'a> {
1355     type Item = (FullCodeCharKind, String);
1356
1357     fn next(&mut self) -> Option<Self::Item> {
1358         self.base.peek()?;
1359
1360         let mut line = String::new();
1361
1362         let start_kind = match self.base.peek() {
1363             Some((kind, _)) => *kind,
1364             None => unreachable!(),
1365         };
1366
1367         while let Some((kind, c)) = self.base.next() {
1368             // needed to set the kind of the ending character on the last line
1369             self.kind = kind;
1370             if c == '\n' {
1371                 self.kind = match (start_kind, kind) {
1372                     (FullCodeCharKind::Normal, FullCodeCharKind::InString) => {
1373                         FullCodeCharKind::StartString
1374                     }
1375                     (FullCodeCharKind::InString, FullCodeCharKind::Normal) => {
1376                         FullCodeCharKind::EndString
1377                     }
1378                     (FullCodeCharKind::InComment, FullCodeCharKind::InStringCommented) => {
1379                         FullCodeCharKind::StartStringCommented
1380                     }
1381                     (FullCodeCharKind::InStringCommented, FullCodeCharKind::InComment) => {
1382                         FullCodeCharKind::EndStringCommented
1383                     }
1384                     _ => kind,
1385                 };
1386                 break;
1387             }
1388             line.push(c);
1389         }
1390
1391         // Workaround for CRLF newline.
1392         if line.ends_with('\r') {
1393             line.pop();
1394         }
1395
1396         Some((self.kind, line))
1397     }
1398 }
1399
1400 /// Iterator over functional and commented parts of a string. Any part of a string is either
1401 /// functional code, either *one* block comment, either *one* line comment. Whitespace between
1402 /// comments is functional code. Line comments contain their ending newlines.
1403 struct UngroupedCommentCodeSlices<'a> {
1404     slice: &'a str,
1405     iter: iter::Peekable<CharClasses<std::str::CharIndices<'a>>>,
1406 }
1407
1408 impl<'a> UngroupedCommentCodeSlices<'a> {
1409     fn new(code: &'a str) -> UngroupedCommentCodeSlices<'a> {
1410         UngroupedCommentCodeSlices {
1411             slice: code,
1412             iter: CharClasses::new(code.char_indices()).peekable(),
1413         }
1414     }
1415 }
1416
1417 impl<'a> Iterator for UngroupedCommentCodeSlices<'a> {
1418     type Item = (CodeCharKind, usize, &'a str);
1419
1420     fn next(&mut self) -> Option<Self::Item> {
1421         let (kind, (start_idx, _)) = self.iter.next()?;
1422         match kind {
1423             FullCodeCharKind::Normal | FullCodeCharKind::InString => {
1424                 // Consume all the Normal code
1425                 while let Some(&(char_kind, _)) = self.iter.peek() {
1426                     if char_kind.is_comment() {
1427                         break;
1428                     }
1429                     let _ = self.iter.next();
1430                 }
1431             }
1432             FullCodeCharKind::StartComment => {
1433                 // Consume the whole comment
1434                 loop {
1435                     match self.iter.next() {
1436                         Some((kind, ..)) if kind.inside_comment() => continue,
1437                         _ => break,
1438                     }
1439                 }
1440             }
1441             _ => panic!(),
1442         }
1443         let slice = match self.iter.peek() {
1444             Some(&(_, (end_idx, _))) => &self.slice[start_idx..end_idx],
1445             None => &self.slice[start_idx..],
1446         };
1447         Some((
1448             if kind.is_comment() {
1449                 CodeCharKind::Comment
1450             } else {
1451                 CodeCharKind::Normal
1452             },
1453             start_idx,
1454             slice,
1455         ))
1456     }
1457 }
1458
1459 /// Iterator over an alternating sequence of functional and commented parts of
1460 /// a string. The first item is always a, possibly zero length, subslice of
1461 /// functional text. Line style comments contain their ending newlines.
1462 pub struct CommentCodeSlices<'a> {
1463     slice: &'a str,
1464     last_slice_kind: CodeCharKind,
1465     last_slice_end: usize,
1466 }
1467
1468 impl<'a> CommentCodeSlices<'a> {
1469     pub fn new(slice: &'a str) -> CommentCodeSlices<'a> {
1470         CommentCodeSlices {
1471             slice,
1472             last_slice_kind: CodeCharKind::Comment,
1473             last_slice_end: 0,
1474         }
1475     }
1476 }
1477
1478 impl<'a> Iterator for CommentCodeSlices<'a> {
1479     type Item = (CodeCharKind, usize, &'a str);
1480
1481     fn next(&mut self) -> Option<Self::Item> {
1482         if self.last_slice_end == self.slice.len() {
1483             return None;
1484         }
1485
1486         let mut sub_slice_end = self.last_slice_end;
1487         let mut first_whitespace = None;
1488         let subslice = &self.slice[self.last_slice_end..];
1489         let mut iter = CharClasses::new(subslice.char_indices());
1490
1491         for (kind, (i, c)) in &mut iter {
1492             let is_comment_connector = self.last_slice_kind == CodeCharKind::Normal
1493                 && &subslice[..2] == "//"
1494                 && [' ', '\t'].contains(&c);
1495
1496             if is_comment_connector && first_whitespace.is_none() {
1497                 first_whitespace = Some(i);
1498             }
1499
1500             if kind.to_codecharkind() == self.last_slice_kind && !is_comment_connector {
1501                 let last_index = match first_whitespace {
1502                     Some(j) => j,
1503                     None => i,
1504                 };
1505                 sub_slice_end = self.last_slice_end + last_index;
1506                 break;
1507             }
1508
1509             if !is_comment_connector {
1510                 first_whitespace = None;
1511             }
1512         }
1513
1514         if let (None, true) = (iter.next(), sub_slice_end == self.last_slice_end) {
1515             // This was the last subslice.
1516             sub_slice_end = match first_whitespace {
1517                 Some(i) => self.last_slice_end + i,
1518                 None => self.slice.len(),
1519             };
1520         }
1521
1522         let kind = match self.last_slice_kind {
1523             CodeCharKind::Comment => CodeCharKind::Normal,
1524             CodeCharKind::Normal => CodeCharKind::Comment,
1525         };
1526         let res = (
1527             kind,
1528             self.last_slice_end,
1529             &self.slice[self.last_slice_end..sub_slice_end],
1530         );
1531         self.last_slice_end = sub_slice_end;
1532         self.last_slice_kind = kind;
1533
1534         Some(res)
1535     }
1536 }
1537
1538 /// Checks is `new` didn't miss any comment from `span`, if it removed any, return previous text
1539 /// (if it fits in the width/offset, else return None), else return `new`
1540 pub fn recover_comment_removed(
1541     new: String,
1542     span: Span,
1543     context: &RewriteContext,
1544 ) -> Option<String> {
1545     let snippet = context.snippet(span);
1546     if snippet != new && changed_comment_content(snippet, &new) {
1547         // We missed some comments. Warn and keep the original text.
1548         if context.config.error_on_unformatted() {
1549             context.report.append(
1550                 context.source_map.span_to_filename(span).into(),
1551                 vec![FormattingError::from_span(
1552                     span,
1553                     &context.source_map,
1554                     ErrorKind::LostComment,
1555                 )],
1556             );
1557         }
1558         Some(snippet.to_owned())
1559     } else {
1560         Some(new)
1561     }
1562 }
1563
1564 pub fn filter_normal_code(code: &str) -> String {
1565     let mut buffer = String::with_capacity(code.len());
1566     LineClasses::new(code).for_each(|(kind, line)| match kind {
1567         FullCodeCharKind::Normal
1568         | FullCodeCharKind::StartString
1569         | FullCodeCharKind::InString
1570         | FullCodeCharKind::EndString => {
1571             buffer.push_str(&line);
1572             buffer.push('\n');
1573         }
1574         _ => (),
1575     });
1576     if !code.ends_with('\n') && buffer.ends_with('\n') {
1577         buffer.pop();
1578     }
1579     buffer
1580 }
1581
1582 /// Return true if the two strings of code have the same payload of comments.
1583 /// The payload of comments is everything in the string except:
1584 ///     - actual code (not comments)
1585 ///     - comment start/end marks
1586 ///     - whitespace
1587 ///     - '*' at the beginning of lines in block comments
1588 fn changed_comment_content(orig: &str, new: &str) -> bool {
1589     // Cannot write this as a fn since we cannot return types containing closures
1590     let code_comment_content = |code| {
1591         let slices = UngroupedCommentCodeSlices::new(code);
1592         slices
1593             .filter(|&(ref kind, _, _)| *kind == CodeCharKind::Comment)
1594             .flat_map(|(_, _, s)| CommentReducer::new(s))
1595     };
1596     let res = code_comment_content(orig).ne(code_comment_content(new));
1597     debug!(
1598         "comment::changed_comment_content: {}\norig: '{}'\nnew: '{}'\nraw_old: {}\nraw_new: {}",
1599         res,
1600         orig,
1601         new,
1602         code_comment_content(orig).collect::<String>(),
1603         code_comment_content(new).collect::<String>()
1604     );
1605     res
1606 }
1607
1608 /// Iterator over the 'payload' characters of a comment.
1609 /// It skips whitespace, comment start/end marks, and '*' at the beginning of lines.
1610 /// The comment must be one comment, ie not more than one start mark (no multiple line comments,
1611 /// for example).
1612 struct CommentReducer<'a> {
1613     is_block: bool,
1614     at_start_line: bool,
1615     iter: std::str::Chars<'a>,
1616 }
1617
1618 impl<'a> CommentReducer<'a> {
1619     fn new(comment: &'a str) -> CommentReducer<'a> {
1620         let is_block = comment.starts_with("/*");
1621         let comment = remove_comment_header(comment);
1622         CommentReducer {
1623             is_block,
1624             at_start_line: false, // There are no supplementary '*' on the first line
1625             iter: comment.chars(),
1626         }
1627     }
1628 }
1629
1630 impl<'a> Iterator for CommentReducer<'a> {
1631     type Item = char;
1632
1633     fn next(&mut self) -> Option<Self::Item> {
1634         loop {
1635             let mut c = self.iter.next()?;
1636             if self.is_block && self.at_start_line {
1637                 while c.is_whitespace() {
1638                     c = self.iter.next()?;
1639                 }
1640                 // Ignore leading '*'
1641                 if c == '*' {
1642                     c = self.iter.next()?;
1643                 }
1644             } else if c == '\n' {
1645                 self.at_start_line = true;
1646             }
1647             if !c.is_whitespace() {
1648                 return Some(c);
1649             }
1650         }
1651     }
1652 }
1653
1654 fn remove_comment_header(comment: &str) -> &str {
1655     if comment.starts_with("///") || comment.starts_with("//!") {
1656         &comment[3..]
1657     } else if comment.starts_with("//") {
1658         &comment[2..]
1659     } else if (comment.starts_with("/**") && !comment.starts_with("/**/"))
1660         || comment.starts_with("/*!")
1661     {
1662         &comment[3..comment.len() - 2]
1663     } else {
1664         assert!(
1665             comment.starts_with("/*"),
1666             format!("string '{}' is not a comment", comment)
1667         );
1668         &comment[2..comment.len() - 2]
1669     }
1670 }
1671
1672 #[cfg(test)]
1673 mod test {
1674     use super::*;
1675     use shape::{Indent, Shape};
1676
1677     #[test]
1678     fn char_classes() {
1679         let mut iter = CharClasses::new("//\n\n".chars());
1680
1681         assert_eq!((FullCodeCharKind::StartComment, '/'), iter.next().unwrap());
1682         assert_eq!((FullCodeCharKind::InComment, '/'), iter.next().unwrap());
1683         assert_eq!((FullCodeCharKind::EndComment, '\n'), iter.next().unwrap());
1684         assert_eq!((FullCodeCharKind::Normal, '\n'), iter.next().unwrap());
1685         assert_eq!(None, iter.next());
1686     }
1687
1688     #[test]
1689     fn comment_code_slices() {
1690         let input = "code(); /* test */ 1 + 1";
1691         let mut iter = CommentCodeSlices::new(input);
1692
1693         assert_eq!((CodeCharKind::Normal, 0, "code(); "), iter.next().unwrap());
1694         assert_eq!(
1695             (CodeCharKind::Comment, 8, "/* test */"),
1696             iter.next().unwrap()
1697         );
1698         assert_eq!((CodeCharKind::Normal, 18, " 1 + 1"), iter.next().unwrap());
1699         assert_eq!(None, iter.next());
1700     }
1701
1702     #[test]
1703     fn comment_code_slices_two() {
1704         let input = "// comment\n    test();";
1705         let mut iter = CommentCodeSlices::new(input);
1706
1707         assert_eq!((CodeCharKind::Normal, 0, ""), iter.next().unwrap());
1708         assert_eq!(
1709             (CodeCharKind::Comment, 0, "// comment\n"),
1710             iter.next().unwrap()
1711         );
1712         assert_eq!(
1713             (CodeCharKind::Normal, 11, "    test();"),
1714             iter.next().unwrap()
1715         );
1716         assert_eq!(None, iter.next());
1717     }
1718
1719     #[test]
1720     fn comment_code_slices_three() {
1721         let input = "1 // comment\n    // comment2\n\n";
1722         let mut iter = CommentCodeSlices::new(input);
1723
1724         assert_eq!((CodeCharKind::Normal, 0, "1 "), iter.next().unwrap());
1725         assert_eq!(
1726             (CodeCharKind::Comment, 2, "// comment\n    // comment2\n"),
1727             iter.next().unwrap()
1728         );
1729         assert_eq!((CodeCharKind::Normal, 29, "\n"), iter.next().unwrap());
1730         assert_eq!(None, iter.next());
1731     }
1732
1733     #[test]
1734     #[rustfmt::skip]
1735     fn format_doc_comments() {
1736         let mut wrap_normalize_config: ::config::Config = Default::default();
1737         wrap_normalize_config.set().wrap_comments(true);
1738         wrap_normalize_config.set().normalize_comments(true);
1739
1740         let mut wrap_config: ::config::Config = Default::default();
1741         wrap_config.set().wrap_comments(true);
1742
1743         let comment = rewrite_comment(" //test",
1744                                       true,
1745                                       Shape::legacy(100, Indent::new(0, 100)),
1746                                       &wrap_normalize_config).unwrap();
1747         assert_eq!("/* test */", comment);
1748
1749         let comment = rewrite_comment("// comment on a",
1750                                       false,
1751                                       Shape::legacy(10, Indent::empty()),
1752                                       &wrap_normalize_config).unwrap();
1753         assert_eq!("// comment\n// on a", comment);
1754
1755         let comment = rewrite_comment("//  A multi line comment\n             // between args.",
1756                                       false,
1757                                       Shape::legacy(60, Indent::new(0, 12)),
1758                                       &wrap_normalize_config).unwrap();
1759         assert_eq!("//  A multi line comment\n            // between args.", comment);
1760
1761         let input = "// comment";
1762         let expected =
1763             "/* comment */";
1764         let comment = rewrite_comment(input,
1765                                       true,
1766                                       Shape::legacy(9, Indent::new(0, 69)),
1767                                       &wrap_normalize_config).unwrap();
1768         assert_eq!(expected, comment);
1769
1770         let comment = rewrite_comment("/*   trimmed    */",
1771                                       true,
1772                                       Shape::legacy(100, Indent::new(0, 100)),
1773                                       &wrap_normalize_config).unwrap();
1774         assert_eq!("/* trimmed */", comment);
1775
1776         // check that different comment style are properly recognised
1777         let comment = rewrite_comment(r#"/// test1
1778                                          /// test2
1779                                          /*
1780                                           * test3
1781                                           */"#,
1782                                       false,
1783                                       Shape::legacy(100, Indent::new(0, 0)),
1784                                       &wrap_normalize_config).unwrap();
1785         assert_eq!("/// test1\n/// test2\n// test3", comment);
1786
1787         // check that the blank line marks the end of a commented paragraph
1788         let comment = rewrite_comment(r#"// test1
1789
1790                                          // test2"#,
1791                                       false,
1792                                       Shape::legacy(100, Indent::new(0, 0)),
1793                                       &wrap_normalize_config).unwrap();
1794         assert_eq!("// test1\n\n// test2", comment);
1795
1796         // check that the blank line marks the end of a custom-commented paragraph
1797         let comment = rewrite_comment(r#"//@ test1
1798
1799                                          //@ test2"#,
1800                                       false,
1801                                       Shape::legacy(100, Indent::new(0, 0)),
1802                                       &wrap_normalize_config).unwrap();
1803         assert_eq!("//@ test1\n\n//@ test2", comment);
1804
1805         // check that bare lines are just indented but left unchanged otherwise
1806         let comment = rewrite_comment(r#"// test1
1807                                          /*
1808                                            a bare line!
1809
1810                                                 another bare line!
1811                                           */"#,
1812                                       false,
1813                                       Shape::legacy(100, Indent::new(0, 0)),
1814                                       &wrap_config).unwrap();
1815         assert_eq!("// test1\n/*\n a bare line!\n\n      another bare line!\n*/", comment);
1816     }
1817
1818     // This is probably intended to be a non-test fn, but it is not used. I'm
1819     // keeping it around unless it helps us test stuff.
1820     fn uncommented(text: &str) -> String {
1821         CharClasses::new(text.chars())
1822             .filter_map(|(s, c)| match s {
1823                 FullCodeCharKind::Normal | FullCodeCharKind::InString => Some(c),
1824                 _ => None,
1825             })
1826             .collect()
1827     }
1828
1829     #[test]
1830     fn test_uncommented() {
1831         assert_eq!(&uncommented("abc/*...*/"), "abc");
1832         assert_eq!(
1833             &uncommented("// .... /* \n../* /* *** / */ */a/* // */c\n"),
1834             "..ac\n"
1835         );
1836         assert_eq!(&uncommented("abc \" /* */\" qsdf"), "abc \" /* */\" qsdf");
1837     }
1838
1839     #[test]
1840     fn test_contains_comment() {
1841         assert_eq!(contains_comment("abc"), false);
1842         assert_eq!(contains_comment("abc // qsdf"), true);
1843         assert_eq!(contains_comment("abc /* kqsdf"), true);
1844         assert_eq!(contains_comment("abc \" /* */\" qsdf"), false);
1845     }
1846
1847     #[test]
1848     fn test_find_uncommented() {
1849         fn check(haystack: &str, needle: &str, expected: Option<usize>) {
1850             assert_eq!(expected, haystack.find_uncommented(needle));
1851         }
1852
1853         check("/*/ */test", "test", Some(6));
1854         check("//test\ntest", "test", Some(7));
1855         check("/* comment only */", "whatever", None);
1856         check(
1857             "/* comment */ some text /* more commentary */ result",
1858             "result",
1859             Some(46),
1860         );
1861         check("sup // sup", "p", Some(2));
1862         check("sup", "x", None);
1863         check(r#"π? /**/ π is nice!"#, r#"π is nice"#, Some(9));
1864         check("/*sup yo? \n sup*/ sup", "p", Some(20));
1865         check("hel/*lohello*/lo", "hello", None);
1866         check("acb", "ab", None);
1867         check(",/*A*/ ", ",", Some(0));
1868         check("abc", "abc", Some(0));
1869         check("/* abc */", "abc", None);
1870         check("/**/abc/* */", "abc", Some(4));
1871         check("\"/* abc */\"", "abc", Some(4));
1872         check("\"/* abc", "abc", Some(4));
1873     }
1874
1875     #[test]
1876     fn test_filter_normal_code() {
1877         let s = r#"
1878 fn main() {
1879     println!("hello, world");
1880 }
1881 "#;
1882         assert_eq!(s, filter_normal_code(s));
1883         let s_with_comment = r#"
1884 fn main() {
1885     // hello, world
1886     println!("hello, world");
1887 }
1888 "#;
1889         assert_eq!(s, filter_normal_code(s_with_comment));
1890     }
1891 }