1 // Copyright 2013-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! Markdown formatting for rustdoc
13 //! This module implements markdown formatting through the pulldown-cmark
14 //! rust-library. This module exposes all of the
15 //! functionality through a unit-struct, `Markdown`, which has an implementation
16 //! of `fmt::Display`. Example usage:
19 //! use rustdoc::html::markdown::Markdown;
21 //! let s = "My *markdown* _text_";
22 //! let html = format!("{}", Markdown(s));
23 //! // ... something using html
26 #![allow(non_camel_case_types)]
28 use std::ascii::AsciiExt;
29 use std::cell::RefCell;
30 use std::collections::{HashMap, VecDeque};
31 use std::default::Default;
32 use std::fmt::{self, Write};
34 use syntax::feature_gate::UnstableFeatures;
35 use syntax::codemap::Span;
37 use html::render::derive_id;
38 use html::toc::TocBuilder;
42 use pulldown_cmark::{html, Event, Tag, Parser};
43 use pulldown_cmark::{Options, OPTION_ENABLE_FOOTNOTES, OPTION_ENABLE_TABLES};
45 /// A unit struct which has the `fmt::Display` trait implemented. When
46 /// formatted, this struct will emit the HTML corresponding to the rendered
47 /// version of the contained markdown string.
48 // The second parameter is whether we need a shorter version or not.
49 pub struct Markdown<'a>(pub &'a str);
50 /// A unit struct like `Markdown`, that renders the markdown with a
51 /// table of contents.
52 pub struct MarkdownWithToc<'a>(pub &'a str);
53 /// A unit struct like `Markdown`, that renders the markdown escaping HTML tags.
54 pub struct MarkdownHtml<'a>(pub &'a str);
55 /// A unit struct like `Markdown`, that renders only the first paragraph.
56 pub struct MarkdownSummaryLine<'a>(pub &'a str);
58 /// Returns Some(code) if `s` is a line that should be stripped from
59 /// documentation but used in example code. `code` is the portion of
60 /// `s` that should be used in tests. (None for lines that should be
62 fn stripped_filtered_line<'a>(s: &'a str) -> Option<&'a str> {
63 let trimmed = s.trim();
66 } else if trimmed.starts_with("# ") {
73 /// Convert chars from a title for an id.
75 /// "Hello, world!" -> "hello-world"
76 fn slugify(c: char) -> Option<char> {
77 if c.is_alphanumeric() || c == '-' || c == '_' {
79 Some(c.to_ascii_lowercase())
83 } else if c.is_whitespace() && c.is_ascii() {
90 // Information about the playground if a URL has been specified, containing an
91 // optional crate name and the URL.
92 thread_local!(pub static PLAYGROUND: RefCell<Option<(Option<String>, String)>> = {
96 /// Adds syntax highlighting and playground Run buttons to rust code blocks.
97 struct CodeBlocks<'a, I: Iterator<Item = Event<'a>>> {
101 impl<'a, I: Iterator<Item = Event<'a>>> CodeBlocks<'a, I> {
102 fn new(iter: I) -> Self {
109 impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'a, I> {
110 type Item = Event<'a>;
112 fn next(&mut self) -> Option<Self::Item> {
113 let event = self.inner.next();
114 if let Some(Event::Start(Tag::CodeBlock(lang))) = event {
115 if !LangString::parse(&lang).rust {
116 return Some(Event::Start(Tag::CodeBlock(lang)));
122 let mut origtext = String::new();
123 for event in &mut self.inner {
125 Event::End(Tag::CodeBlock(..)) => break,
126 Event::Text(ref s) => {
127 origtext.push_str(s);
132 let lines = origtext.lines().filter(|l| {
133 stripped_filtered_line(*l).is_none()
135 let text = lines.collect::<Vec<&str>>().join("\n");
136 PLAYGROUND.with(|play| {
137 // insert newline to clearly separate it from the
138 // previous block so we can shorten the html output
139 let mut s = String::from("\n");
140 let playground_button = play.borrow().as_ref().and_then(|&(ref krate, ref url)| {
144 let test = origtext.lines().map(|l| {
145 stripped_filtered_line(l).unwrap_or(l)
146 }).collect::<Vec<&str>>().join("\n");
147 let krate = krate.as_ref().map(|s| &**s);
148 let test = test::maketest(&test, krate, false,
149 &Default::default());
150 let channel = if test.contains("#![feature(") {
151 "&version=nightly"
155 // These characters don't need to be escaped in a URI.
156 // FIXME: use a library function for percent encoding.
157 fn dont_escape(c: u8) -> bool {
158 (b'a' <= c && c <= b'z') ||
159 (b'A' <= c && c <= b'Z') ||
160 (b'0' <= c && c <= b'9') ||
161 c == b'-' || c == b'_' || c == b'.' ||
162 c == b'~' || c == b'!' || c == b'\'' ||
163 c == b'(' || c == b')' || c == b'*'
165 let mut test_escaped = String::new();
166 for b in test.bytes() {
168 test_escaped.push(char::from(b));
170 write!(test_escaped, "%{:02X}", b).unwrap();
174 r#"<a class="test-arrow" target="_blank" href="{}?code={}{}">Run</a>"#,
175 url, test_escaped, channel
178 s.push_str(&highlight::render_with_highlighting(
180 Some("rust-example-rendered"),
182 playground_button.as_ref().map(String::as_str)));
183 Some(Event::Html(s.into()))
188 /// Make headings links with anchor ids and build up TOC.
189 struct HeadingLinks<'a, 'b, I: Iterator<Item = Event<'a>>> {
191 toc: Option<&'b mut TocBuilder>,
192 buf: VecDeque<Event<'a>>,
195 impl<'a, 'b, I: Iterator<Item = Event<'a>>> HeadingLinks<'a, 'b, I> {
196 fn new(iter: I, toc: Option<&'b mut TocBuilder>) -> Self {
200 buf: VecDeque::new(),
205 impl<'a, 'b, I: Iterator<Item = Event<'a>>> Iterator for HeadingLinks<'a, 'b, I> {
206 type Item = Event<'a>;
208 fn next(&mut self) -> Option<Self::Item> {
209 if let Some(e) = self.buf.pop_front() {
213 let event = self.inner.next();
214 if let Some(Event::Start(Tag::Header(level))) = event {
215 let mut id = String::new();
216 for event in &mut self.inner {
218 Event::End(Tag::Header(..)) => break,
219 Event::Text(ref text) => id.extend(text.chars().filter_map(slugify)),
222 self.buf.push_back(event);
224 let id = derive_id(id);
226 if let Some(ref mut builder) = self.toc {
227 let mut html_header = String::new();
228 html::push_html(&mut html_header, self.buf.iter().cloned());
229 let sec = builder.push(level as u32, html_header, id.clone());
230 self.buf.push_front(Event::InlineHtml(format!("{} ", sec).into()));
233 self.buf.push_back(Event::InlineHtml(format!("</a></h{}>", level).into()));
235 let start_tags = format!("<h{level} id=\"{id}\" class=\"section-header\">\
239 return Some(Event::InlineHtml(start_tags.into()));
245 /// Extracts just the first paragraph.
246 struct SummaryLine<'a, I: Iterator<Item = Event<'a>>> {
252 impl<'a, I: Iterator<Item = Event<'a>>> SummaryLine<'a, I> {
253 fn new(iter: I) -> Self {
262 impl<'a, I: Iterator<Item = Event<'a>>> Iterator for SummaryLine<'a, I> {
263 type Item = Event<'a>;
265 fn next(&mut self) -> Option<Self::Item> {
266 if self.started && self.depth == 0 {
272 let event = self.inner.next();
274 Some(Event::Start(..)) => self.depth += 1,
275 Some(Event::End(..)) => self.depth -= 1,
282 /// Moves all footnote definitions to the end and add back links to the
284 struct Footnotes<'a, I: Iterator<Item = Event<'a>>> {
286 footnotes: HashMap<String, (Vec<Event<'a>>, u16)>,
289 impl<'a, I: Iterator<Item = Event<'a>>> Footnotes<'a, I> {
290 fn new(iter: I) -> Self {
293 footnotes: HashMap::new(),
296 fn get_entry(&mut self, key: &str) -> &mut (Vec<Event<'a>>, u16) {
297 let new_id = self.footnotes.keys().count() + 1;
298 let key = key.to_owned();
299 self.footnotes.entry(key).or_insert((Vec::new(), new_id as u16))
303 impl<'a, I: Iterator<Item = Event<'a>>> Iterator for Footnotes<'a, I> {
304 type Item = Event<'a>;
306 fn next(&mut self) -> Option<Self::Item> {
308 match self.inner.next() {
309 Some(Event::FootnoteReference(ref reference)) => {
310 let entry = self.get_entry(&reference);
311 let reference = format!("<sup id=\"supref{0}\"><a href=\"#ref{0}\">{0}\
314 return Some(Event::Html(reference.into()));
316 Some(Event::Start(Tag::FootnoteDefinition(def))) => {
317 let mut content = Vec::new();
318 for event in &mut self.inner {
319 if let Event::End(Tag::FootnoteDefinition(..)) = event {
324 let entry = self.get_entry(&def);
325 (*entry).0 = content;
327 Some(e) => return Some(e),
329 if !self.footnotes.is_empty() {
330 let mut v: Vec<_> = self.footnotes.drain().map(|(_, x)| x).collect();
331 v.sort_by(|a, b| a.1.cmp(&b.1));
332 let mut ret = String::from("<div class=\"footnotes\"><hr><ol>");
333 for (mut content, id) in v {
334 write!(ret, "<li id=\"ref{}\">", id).unwrap();
335 let mut is_paragraph = false;
336 if let Some(&Event::End(Tag::Paragraph)) = content.last() {
340 html::push_html(&mut ret, content.into_iter());
342 " <a href=\"#supref{}\" rev=\"footnote\">↩</a>",
345 ret.push_str("</p>");
347 ret.push_str("</li>");
349 ret.push_str("</ol></div>");
350 return Some(Event::Html(ret.into()));
360 pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector, position: Span) {
361 tests.set_position(position);
363 let mut parser = Parser::new(doc);
364 let mut prev_offset = 0;
365 let mut nb_lines = 0;
366 let mut register_header = None;
367 'main: while let Some(event) = parser.next() {
369 Event::Start(Tag::CodeBlock(s)) => {
370 let block_info = if s.is_empty() {
371 LangString::all_false()
373 LangString::parse(&*s)
375 if !block_info.rust {
378 let mut test_s = String::new();
379 let mut offset = None;
381 let event = parser.next();
382 if let Some(event) = event {
384 Event::End(Tag::CodeBlock(_)) => break,
385 Event::Text(ref s) => {
387 if offset.is_none() {
388 offset = Some(parser.get_offset());
397 let offset = offset.unwrap_or(0);
398 let lines = test_s.lines().map(|l| {
399 stripped_filtered_line(l).unwrap_or(l)
401 let text = lines.collect::<Vec<&str>>().join("\n");
402 nb_lines += doc[prev_offset..offset].lines().count();
403 let line = tests.get_line() + (nb_lines - 1);
404 let filename = tests.get_filename();
405 tests.add_test(text.to_owned(),
406 block_info.should_panic, block_info.no_run,
407 block_info.ignore, block_info.test_harness,
408 block_info.compile_fail, block_info.error_codes,
410 prev_offset = offset;
412 Event::Start(Tag::Header(level)) => {
413 register_header = Some(level as u32);
415 Event::Text(ref s) if register_header.is_some() => {
416 let level = register_header.unwrap();
418 tests.register_header("", level);
420 tests.register_header(s, level);
422 register_header = None;
429 #[derive(Eq, PartialEq, Clone, Debug)]
438 error_codes: Vec<String>,
442 fn all_false() -> LangString {
444 original: String::new(),
448 rust: true, // NB This used to be `notrust = false`
451 error_codes: Vec::new(),
455 fn parse(string: &str) -> LangString {
456 let mut seen_rust_tags = false;
457 let mut seen_other_tags = false;
458 let mut data = LangString::all_false();
459 let mut allow_compile_fail = false;
460 let mut allow_error_code_check = false;
461 if UnstableFeatures::from_environment().is_nightly_build() {
462 allow_compile_fail = true;
463 allow_error_code_check = true;
466 data.original = string.to_owned();
467 let tokens = string.split(|c: char|
468 !(c == '_' || c == '-' || c.is_alphanumeric())
471 for token in tokens {
474 "should_panic" => { data.should_panic = true; seen_rust_tags = true; },
475 "no_run" => { data.no_run = true; seen_rust_tags = true; },
476 "ignore" => { data.ignore = true; seen_rust_tags = true; },
477 "rust" => { data.rust = true; seen_rust_tags = true; },
478 "test_harness" => { data.test_harness = true; seen_rust_tags = true; },
479 "compile_fail" if allow_compile_fail => {
480 data.compile_fail = true;
481 seen_rust_tags = true;
484 x if allow_error_code_check && x.starts_with("E") && x.len() == 5 => {
485 if let Ok(_) = x[1..].parse::<u32>() {
486 data.error_codes.push(x.to_owned());
487 seen_rust_tags = true;
489 seen_other_tags = true;
492 _ => { seen_other_tags = true }
496 data.rust &= !seen_other_tags || seen_rust_tags;
502 impl<'a> fmt::Display for Markdown<'a> {
503 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
504 let Markdown(md) = *self;
505 // This is actually common enough to special-case
506 if md.is_empty() { return Ok(()) }
508 let mut opts = Options::empty();
509 opts.insert(OPTION_ENABLE_TABLES);
510 opts.insert(OPTION_ENABLE_FOOTNOTES);
512 let p = Parser::new_ext(md, opts);
514 let mut s = String::with_capacity(md.len() * 3 / 2);
516 html::push_html(&mut s,
517 Footnotes::new(CodeBlocks::new(HeadingLinks::new(p, None))));
523 impl<'a> fmt::Display for MarkdownWithToc<'a> {
524 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
525 let MarkdownWithToc(md) = *self;
527 let mut opts = Options::empty();
528 opts.insert(OPTION_ENABLE_TABLES);
529 opts.insert(OPTION_ENABLE_FOOTNOTES);
531 let p = Parser::new_ext(md, opts);
533 let mut s = String::with_capacity(md.len() * 3 / 2);
535 let mut toc = TocBuilder::new();
537 html::push_html(&mut s,
538 Footnotes::new(CodeBlocks::new(HeadingLinks::new(p, Some(&mut toc)))));
540 write!(fmt, "<nav id=\"TOC\">{}</nav>", toc.into_toc())?;
546 impl<'a> fmt::Display for MarkdownHtml<'a> {
547 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
548 let MarkdownHtml(md) = *self;
549 // This is actually common enough to special-case
550 if md.is_empty() { return Ok(()) }
552 let mut opts = Options::empty();
553 opts.insert(OPTION_ENABLE_TABLES);
554 opts.insert(OPTION_ENABLE_FOOTNOTES);
556 let p = Parser::new_ext(md, opts);
558 // Treat inline HTML as plain text.
559 let p = p.map(|event| match event {
560 Event::Html(text) | Event::InlineHtml(text) => Event::Text(text),
564 let mut s = String::with_capacity(md.len() * 3 / 2);
566 html::push_html(&mut s,
567 Footnotes::new(CodeBlocks::new(HeadingLinks::new(p, None))));
573 impl<'a> fmt::Display for MarkdownSummaryLine<'a> {
574 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
575 let MarkdownSummaryLine(md) = *self;
576 // This is actually common enough to special-case
577 if md.is_empty() { return Ok(()) }
579 let p = Parser::new(md);
581 let mut s = String::new();
583 html::push_html(&mut s, SummaryLine::new(p));
589 pub fn plain_summary_line(md: &str) -> String {
590 struct ParserWrapper<'a> {
596 impl<'a> Iterator for ParserWrapper<'a> {
599 fn next(&mut self) -> Option<String> {
600 let next_event = self.inner.next();
601 if next_event.is_none() {
604 let next_event = next_event.unwrap();
605 let (ret, is_in) = match next_event {
606 Event::Start(Tag::Paragraph) => (None, 1),
607 Event::Start(Tag::Code) => (Some("`".to_owned()), 1),
608 Event::End(Tag::Code) => (Some("`".to_owned()), -1),
609 Event::Start(Tag::Header(_)) => (None, 1),
610 Event::Text(ref s) if self.is_in > 0 => (Some(s.as_ref().to_owned()), 0),
611 Event::End(Tag::Paragraph) | Event::End(Tag::Header(_)) => (None, -1),
614 if is_in > 0 || (is_in < 0 && self.is_in > 0) {
618 self.is_first = false;
625 let mut s = String::with_capacity(md.len() * 3 / 2);
626 let mut p = ParserWrapper {
627 inner: Parser::new(md),
631 while let Some(t) = p.next() {
641 use super::{LangString, Markdown, MarkdownHtml};
642 use super::plain_summary_line;
643 use html::render::reset_ids;
646 fn test_lang_string_parse() {
648 should_panic: bool, no_run: bool, ignore: bool, rust: bool, test_harness: bool,
649 compile_fail: bool, error_codes: Vec<String>) {
650 assert_eq!(LangString::parse(s), LangString {
651 should_panic: should_panic,
655 test_harness: test_harness,
656 compile_fail: compile_fail,
657 error_codes: error_codes,
658 original: s.to_owned(),
662 // marker | should_panic| no_run| ignore| rust | test_harness| compile_fail
664 t("", false, false, false, true, false, false, Vec::new());
665 t("rust", false, false, false, true, false, false, Vec::new());
666 t("sh", false, false, false, false, false, false, Vec::new());
667 t("ignore", false, false, true, true, false, false, Vec::new());
668 t("should_panic", true, false, false, true, false, false, Vec::new());
669 t("no_run", false, true, false, true, false, false, Vec::new());
670 t("test_harness", false, false, false, true, true, false, Vec::new());
671 t("compile_fail", false, true, false, true, false, true, Vec::new());
672 t("{.no_run .example}", false, true, false, true, false, false, Vec::new());
673 t("{.sh .should_panic}", true, false, false, true, false, false, Vec::new());
674 t("{.example .rust}", false, false, false, true, false, false, Vec::new());
675 t("{.test_harness .rust}", false, false, false, true, true, false, Vec::new());
680 let markdown = "# title";
681 format!("{}", Markdown(markdown));
687 fn t(input: &str, expect: &str) {
688 let output = format!("{}", Markdown(input));
689 assert_eq!(output, expect, "original: {}", input);
693 t("# Foo bar", "<h1 id=\"foo-bar\" class=\"section-header\">\
694 <a href=\"#foo-bar\">Foo bar</a></h1>");
695 t("## Foo-bar_baz qux", "<h2 id=\"foo-bar_baz-qux\" class=\"section-\
696 header\"><a href=\"#foo-bar_baz-qux\">Foo-bar_baz qux</a></h2>");
697 t("### **Foo** *bar* baz!?!& -_qux_-%",
698 "<h3 id=\"foo-bar-baz--qux-\" class=\"section-header\">\
699 <a href=\"#foo-bar-baz--qux-\"><strong>Foo</strong> \
700 <em>bar</em> baz!?!& -<em>qux</em>-%</a></h3>");
701 t("#### **Foo?** & \\*bar?!* _`baz`_ ❤ #qux",
702 "<h4 id=\"foo--bar--baz--qux\" class=\"section-header\">\
703 <a href=\"#foo--bar--baz--qux\"><strong>Foo?</strong> & *bar?!* \
704 <em><code>baz</code></em> ❤ #qux</a></h4>");
708 fn test_header_ids_multiple_blocks() {
709 fn t(input: &str, expect: &str) {
710 let output = format!("{}", Markdown(input));
711 assert_eq!(output, expect, "original: {}", input);
715 t("# Example", "<h1 id=\"example\" class=\"section-header\">\
716 <a href=\"#example\">Example</a></h1>");
717 t("# Panics", "<h1 id=\"panics\" class=\"section-header\">\
718 <a href=\"#panics\">Panics</a></h1>");
719 t("# Example", "<h1 id=\"example-1\" class=\"section-header\">\
720 <a href=\"#example-1\">Example</a></h1>");
721 t("# Main", "<h1 id=\"main-1\" class=\"section-header\">\
722 <a href=\"#main-1\">Main</a></h1>");
723 t("# Example", "<h1 id=\"example-2\" class=\"section-header\">\
724 <a href=\"#example-2\">Example</a></h1>");
725 t("# Panics", "<h1 id=\"panics-1\" class=\"section-header\">\
726 <a href=\"#panics-1\">Panics</a></h1>");
734 fn test_plain_summary_line() {
735 fn t(input: &str, expect: &str) {
736 let output = plain_summary_line(input);
737 assert_eq!(output, expect, "original: {}", input);
740 t("hello [Rust](https://www.rust-lang.org) :)", "hello Rust :)");
741 t("hello [Rust](https://www.rust-lang.org \"Rust\") :)", "hello Rust :)");
742 t("code `let x = i32;` ...", "code `let x = i32;` ...");
743 t("type `Type<'static>` ...", "type `Type<'static>` ...");
744 t("# top header", "top header");
745 t("## header", "header");
749 fn test_markdown_html_escape() {
750 fn t(input: &str, expect: &str) {
751 let output = format!("{}", MarkdownHtml(input));
752 assert_eq!(output, expect, "original: {}", input);
755 t("`Struct<'a, T>`", "<p><code>Struct<'a, T></code></p>\n");
756 t("Struct<'a, T>", "<p>Struct<'a, T></p>\n");
757 t("Struct<br>", "<p>Struct<br></p>\n");