1 // Copyright 2013-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! Markdown formatting for rustdoc
13 //! This module implements markdown formatting through the pulldown-cmark
14 //! rust-library. This module exposes all of the
15 //! functionality through a unit-struct, `Markdown`, which has an implementation
16 //! of `fmt::Display`. Example usage:
19 //! #![feature(rustc_private)]
21 //! use rustdoc::html::markdown::{RenderType, Markdown};
23 //! let s = "My *markdown* _text_";
24 //! let html = format!("{}", Markdown(s, RenderType::Pulldown));
25 //! // ... something using html
28 #![allow(non_camel_case_types)]
33 use std::cell::RefCell;
34 use std::collections::{HashMap, VecDeque};
35 use std::default::Default;
36 use std::fmt::{self, Write};
38 use syntax::feature_gate::UnstableFeatures;
39 use syntax::codemap::Span;
41 use html::render::derive_id;
42 use html::toc::TocBuilder;
44 use html::escape::Escape;
47 use pulldown_cmark::{html, Event, Tag, Parser};
48 use pulldown_cmark::{Options, OPTION_ENABLE_FOOTNOTES, OPTION_ENABLE_TABLES};
50 #[derive(PartialEq, Debug, Clone, Copy)]
56 /// A unit struct which has the `fmt::Display` trait implemented. When
57 /// formatted, this struct will emit the HTML corresponding to the rendered
58 /// version of the contained markdown string.
59 /// The second parameter is a list of link replacements
60 // The third parameter is whether we need a shorter version or not.
61 pub struct Markdown<'a>(pub &'a str, pub &'a [(String, String)], pub RenderType);
62 /// A unit struct like `Markdown`, that renders the markdown with a
63 /// table of contents.
64 pub struct MarkdownWithToc<'a>(pub &'a str, pub RenderType);
65 /// A unit struct like `Markdown`, that renders the markdown escaping HTML tags.
66 pub struct MarkdownHtml<'a>(pub &'a str, pub RenderType);
67 /// A unit struct like `Markdown`, that renders only the first paragraph.
68 pub struct MarkdownSummaryLine<'a>(pub &'a str, pub &'a [(String, String)]);
70 /// Controls whether a line will be hidden or shown in HTML output.
72 /// All lines are used in documentation tests.
79 fn for_html(self) -> Option<&'a str> {
81 Line::Shown(l) => Some(l),
82 Line::Hidden(_) => None,
86 fn for_code(self) -> &'a str {
94 // FIXME: There is a minor inconsistency here. For lines that start with ##, we
95 // have no easy way of removing a potential single space after the hashes, which
96 // is done in the single # case. This inconsistency seems okay, if non-ideal. In
97 // order to fix it we'd have to iterate to find the first non-# character, and
98 // then reallocate to remove it; which would make us return a String.
99 fn map_line(s: &str) -> Line {
100 let trimmed = s.trim();
101 if trimmed.starts_with("##") {
102 Line::Shown(&trimmed[1..])
103 } else if trimmed.starts_with("# ") {
105 Line::Hidden(&trimmed[2..])
106 } else if trimmed == "#" {
107 // We cannot handle '#text' because it could be #[attr].
114 /// Returns a new string with all consecutive whitespace collapsed into
117 /// Any leading or trailing whitespace will be trimmed.
118 fn collapse_whitespace(s: &str) -> String {
119 s.split_whitespace().collect::<Vec<_>>().join(" ")
122 /// Convert chars from a title for an id.
124 /// "Hello, world!" -> "hello-world"
125 fn slugify(c: char) -> Option<char> {
126 if c.is_alphanumeric() || c == '-' || c == '_' {
128 Some(c.to_ascii_lowercase())
132 } else if c.is_whitespace() && c.is_ascii() {
139 // Information about the playground if a URL has been specified, containing an
140 // optional crate name and the URL.
141 thread_local!(pub static PLAYGROUND: RefCell<Option<(Option<String>, String)>> = {
145 /// Adds syntax highlighting and playground Run buttons to rust code blocks.
146 struct CodeBlocks<'a, I: Iterator<Item = Event<'a>>> {
150 impl<'a, I: Iterator<Item = Event<'a>>> CodeBlocks<'a, I> {
151 fn new(iter: I) -> Self {
158 impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'a, I> {
159 type Item = Event<'a>;
161 fn next(&mut self) -> Option<Self::Item> {
162 let event = self.inner.next();
165 if let Some(Event::Start(Tag::CodeBlock(lang))) = event {
166 let parse_result = LangString::parse(&lang);
167 if !parse_result.rust {
168 return Some(Event::Start(Tag::CodeBlock(lang)));
170 compile_fail = parse_result.compile_fail;
171 ignore = parse_result.ignore;
176 let mut origtext = String::new();
177 for event in &mut self.inner {
179 Event::End(Tag::CodeBlock(..)) => break,
180 Event::Text(ref s) => {
181 origtext.push_str(s);
186 let lines = origtext.lines().filter_map(|l| map_line(l).for_html());
187 let text = lines.collect::<Vec<&str>>().join("\n");
188 PLAYGROUND.with(|play| {
189 // insert newline to clearly separate it from the
190 // previous block so we can shorten the html output
191 let mut s = String::from("\n");
192 let playground_button = play.borrow().as_ref().and_then(|&(ref krate, ref url)| {
196 let test = origtext.lines()
197 .map(|l| map_line(l).for_code())
198 .collect::<Vec<&str>>().join("\n");
199 let krate = krate.as_ref().map(|s| &**s);
200 let (test, _) = test::make_test(&test, krate, false,
201 &Default::default());
202 let channel = if test.contains("#![feature(") {
203 "&version=nightly"
207 // These characters don't need to be escaped in a URI.
208 // FIXME: use a library function for percent encoding.
209 fn dont_escape(c: u8) -> bool {
210 (b'a' <= c && c <= b'z') ||
211 (b'A' <= c && c <= b'Z') ||
212 (b'0' <= c && c <= b'9') ||
213 c == b'-' || c == b'_' || c == b'.' ||
214 c == b'~' || c == b'!' || c == b'\'' ||
215 c == b'(' || c == b')' || c == b'*'
217 let mut test_escaped = String::new();
218 for b in test.bytes() {
220 test_escaped.push(char::from(b));
222 write!(test_escaped, "%{:02X}", b).unwrap();
226 r#"<a class="test-arrow" target="_blank" href="{}?code={}{}">Run</a>"#,
227 url, test_escaped, channel
230 let tooltip = if ignore {
231 Some(("This example is not tested", "ignore"))
232 } else if compile_fail {
233 Some(("This example deliberately fails to compile", "compile_fail"))
237 s.push_str(&highlight::render_with_highlighting(
239 Some(&format!("rust-example-rendered{}",
240 if ignore { " ignore" }
241 else if compile_fail { " compile_fail" }
244 playground_button.as_ref().map(String::as_str),
246 Some(Event::Html(s.into()))
251 /// Make headings links with anchor ids and build up TOC.
252 struct LinkReplacer<'a, 'b, I: Iterator<Item = Event<'a>>> {
254 links: &'b [(String, String)]
257 impl<'a, 'b, I: Iterator<Item = Event<'a>>> LinkReplacer<'a, 'b, I> {
258 fn new(iter: I, links: &'b [(String, String)]) -> Self {
266 impl<'a, 'b, I: Iterator<Item = Event<'a>>> Iterator for LinkReplacer<'a, 'b, I> {
267 type Item = Event<'a>;
269 fn next(&mut self) -> Option<Self::Item> {
270 let event = self.inner.next();
271 if let Some(Event::Start(Tag::Link(dest, text))) = event {
272 if let Some(&(_, ref replace)) = self.links.into_iter().find(|link| &*link.0 == &*dest)
274 Some(Event::Start(Tag::Link(replace.to_owned().into(), text)))
276 Some(Event::Start(Tag::Link(dest, text)))
284 /// Make headings links with anchor ids and build up TOC.
285 struct HeadingLinks<'a, 'b, I: Iterator<Item = Event<'a>>> {
287 toc: Option<&'b mut TocBuilder>,
288 buf: VecDeque<Event<'a>>,
291 impl<'a, 'b, I: Iterator<Item = Event<'a>>> HeadingLinks<'a, 'b, I> {
292 fn new(iter: I, toc: Option<&'b mut TocBuilder>) -> Self {
296 buf: VecDeque::new(),
301 impl<'a, 'b, I: Iterator<Item = Event<'a>>> Iterator for HeadingLinks<'a, 'b, I> {
302 type Item = Event<'a>;
304 fn next(&mut self) -> Option<Self::Item> {
305 if let Some(e) = self.buf.pop_front() {
309 let event = self.inner.next();
310 if let Some(Event::Start(Tag::Header(level))) = event {
311 let mut id = String::new();
312 for event in &mut self.inner {
314 Event::End(Tag::Header(..)) => break,
315 Event::Text(ref text) => id.extend(text.chars().filter_map(slugify)),
318 self.buf.push_back(event);
320 let id = derive_id(id);
322 if let Some(ref mut builder) = self.toc {
323 let mut html_header = String::new();
324 html::push_html(&mut html_header, self.buf.iter().cloned());
325 let sec = builder.push(level as u32, html_header, id.clone());
326 self.buf.push_front(Event::InlineHtml(format!("{} ", sec).into()));
329 self.buf.push_back(Event::InlineHtml(format!("</a></h{}>", level).into()));
331 let start_tags = format!("<h{level} id=\"{id}\" class=\"section-header\">\
335 return Some(Event::InlineHtml(start_tags.into()));
341 /// Extracts just the first paragraph.
342 struct SummaryLine<'a, I: Iterator<Item = Event<'a>>> {
348 impl<'a, I: Iterator<Item = Event<'a>>> SummaryLine<'a, I> {
349 fn new(iter: I) -> Self {
358 impl<'a, I: Iterator<Item = Event<'a>>> Iterator for SummaryLine<'a, I> {
359 type Item = Event<'a>;
361 fn next(&mut self) -> Option<Self::Item> {
362 if self.started && self.depth == 0 {
368 let event = self.inner.next();
370 Some(Event::Start(..)) => self.depth += 1,
371 Some(Event::End(..)) => self.depth -= 1,
378 /// Moves all footnote definitions to the end and add back links to the
380 struct Footnotes<'a, I: Iterator<Item = Event<'a>>> {
382 footnotes: HashMap<String, (Vec<Event<'a>>, u16)>,
385 impl<'a, I: Iterator<Item = Event<'a>>> Footnotes<'a, I> {
386 fn new(iter: I) -> Self {
389 footnotes: HashMap::new(),
392 fn get_entry(&mut self, key: &str) -> &mut (Vec<Event<'a>>, u16) {
393 let new_id = self.footnotes.keys().count() + 1;
394 let key = key.to_owned();
395 self.footnotes.entry(key).or_insert((Vec::new(), new_id as u16))
399 impl<'a, I: Iterator<Item = Event<'a>>> Iterator for Footnotes<'a, I> {
400 type Item = Event<'a>;
402 fn next(&mut self) -> Option<Self::Item> {
404 match self.inner.next() {
405 Some(Event::FootnoteReference(ref reference)) => {
406 let entry = self.get_entry(&reference);
407 let reference = format!("<sup id=\"fnref{0}\"><a href=\"#fn{0}\">{0}\
410 return Some(Event::Html(reference.into()));
412 Some(Event::Start(Tag::FootnoteDefinition(def))) => {
413 let mut content = Vec::new();
414 for event in &mut self.inner {
415 if let Event::End(Tag::FootnoteDefinition(..)) = event {
420 let entry = self.get_entry(&def);
421 (*entry).0 = content;
423 Some(e) => return Some(e),
425 if !self.footnotes.is_empty() {
426 let mut v: Vec<_> = self.footnotes.drain().map(|(_, x)| x).collect();
427 v.sort_by(|a, b| a.1.cmp(&b.1));
428 let mut ret = String::from("<div class=\"footnotes\"><hr><ol>");
429 for (mut content, id) in v {
430 write!(ret, "<li id=\"fn{}\">", id).unwrap();
431 let mut is_paragraph = false;
432 if let Some(&Event::End(Tag::Paragraph)) = content.last() {
436 html::push_html(&mut ret, content.into_iter());
438 " <a href=\"#fnref{}\" rev=\"footnote\">↩</a>",
441 ret.push_str("</p>");
443 ret.push_str("</li>");
445 ret.push_str("</ol></div>");
446 return Some(Event::Html(ret.into()));
456 const DEF_OUNIT: libc::size_t = 64;
457 const HOEDOWN_EXT_NO_INTRA_EMPHASIS: libc::c_uint = 1 << 11;
458 const HOEDOWN_EXT_TABLES: libc::c_uint = 1 << 0;
459 const HOEDOWN_EXT_FENCED_CODE: libc::c_uint = 1 << 1;
460 const HOEDOWN_EXT_AUTOLINK: libc::c_uint = 1 << 3;
461 const HOEDOWN_EXT_STRIKETHROUGH: libc::c_uint = 1 << 4;
462 const HOEDOWN_EXT_SUPERSCRIPT: libc::c_uint = 1 << 8;
463 const HOEDOWN_EXT_FOOTNOTES: libc::c_uint = 1 << 2;
464 const HOEDOWN_HTML_ESCAPE: libc::c_uint = 1 << 1;
466 const HOEDOWN_EXTENSIONS: libc::c_uint =
467 HOEDOWN_EXT_NO_INTRA_EMPHASIS | HOEDOWN_EXT_TABLES |
468 HOEDOWN_EXT_FENCED_CODE | HOEDOWN_EXT_AUTOLINK |
469 HOEDOWN_EXT_STRIKETHROUGH | HOEDOWN_EXT_SUPERSCRIPT |
470 HOEDOWN_EXT_FOOTNOTES;
472 enum hoedown_document {}
474 type blockcodefn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
475 *const hoedown_buffer, *const hoedown_renderer_data,
478 type blockquotefn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
479 *const hoedown_renderer_data, libc::size_t);
481 type headerfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
482 libc::c_int, *const hoedown_renderer_data,
485 type blockhtmlfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
486 *const hoedown_renderer_data, libc::size_t);
488 type codespanfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
489 *const hoedown_renderer_data, libc::size_t) -> libc::c_int;
491 type linkfn = extern "C" fn (*mut hoedown_buffer, *const hoedown_buffer,
492 *const hoedown_buffer, *const hoedown_buffer,
493 *const hoedown_renderer_data, libc::size_t) -> libc::c_int;
495 type entityfn = extern "C" fn (*mut hoedown_buffer, *const hoedown_buffer,
496 *const hoedown_renderer_data, libc::size_t);
498 type normaltextfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
499 *const hoedown_renderer_data, libc::size_t);
502 struct hoedown_renderer_data {
503 opaque: *mut libc::c_void,
507 struct hoedown_renderer {
508 opaque: *mut libc::c_void,
510 blockcode: Option<blockcodefn>,
511 blockquote: Option<blockquotefn>,
512 header: Option<headerfn>,
514 other_block_level_callbacks: [libc::size_t; 11],
516 blockhtml: Option<blockhtmlfn>,
518 /* span level callbacks - NULL or return 0 prints the span verbatim */
519 autolink: libc::size_t, // unused
520 codespan: Option<codespanfn>,
521 other_span_level_callbacks_1: [libc::size_t; 7],
522 link: Option<linkfn>,
523 other_span_level_callbacks_2: [libc::size_t; 6],
525 /* low level callbacks - NULL copies input directly into the output */
526 entity: Option<entityfn>,
527 normal_text: Option<normaltextfn>,
529 /* header and footer */
530 other_callbacks: [libc::size_t; 2],
534 struct hoedown_html_renderer_state {
535 opaque: *mut libc::c_void,
536 toc_data: html_toc_data,
538 link_attributes: Option<extern "C" fn(*mut hoedown_buffer,
539 *const hoedown_buffer,
540 *const hoedown_renderer_data)>,
544 struct html_toc_data {
545 header_count: libc::c_int,
546 current_level: libc::c_int,
547 level_offset: libc::c_int,
548 nesting_level: libc::c_int,
552 struct hoedown_buffer {
560 dfltblk: extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
561 *const hoedown_buffer, *const hoedown_renderer_data,
563 toc_builder: Option<TocBuilder>,
564 links_out: Option<Vec<String>>,
565 links_replace: Vec<(String, String)>,
569 fn hoedown_html_renderer_new(render_flags: libc::c_uint,
570 nesting_level: libc::c_int)
571 -> *mut hoedown_renderer;
572 fn hoedown_html_renderer_free(renderer: *mut hoedown_renderer);
574 fn hoedown_document_new(rndr: *const hoedown_renderer,
575 extensions: libc::c_uint,
576 max_nesting: libc::size_t) -> *mut hoedown_document;
577 fn hoedown_document_render(doc: *mut hoedown_document,
578 ob: *mut hoedown_buffer,
580 doc_size: libc::size_t);
581 fn hoedown_document_free(md: *mut hoedown_document);
583 fn hoedown_buffer_new(unit: libc::size_t) -> *mut hoedown_buffer;
584 fn hoedown_buffer_free(b: *mut hoedown_buffer);
585 fn hoedown_buffer_put(b: *mut hoedown_buffer, c: *const u8, len: libc::size_t);
588 impl hoedown_buffer {
589 fn as_bytes(&self) -> &[u8] {
590 unsafe { slice::from_raw_parts(self.data, self.size as usize) }
594 extern fn hoedown_block(ob: *mut hoedown_buffer, orig_text: *const hoedown_buffer,
595 lang: *const hoedown_buffer, data: *const hoedown_renderer_data,
596 line: libc::size_t) {
598 if orig_text.is_null() { return }
600 let opaque = (*data).opaque as *mut hoedown_html_renderer_state;
601 let my_opaque: &MyOpaque = &*((*opaque).opaque as *const MyOpaque);
602 let text = (*orig_text).as_bytes();
603 let origtext = str::from_utf8(text).unwrap();
604 let origtext = origtext.trim_left();
605 debug!("docblock: ==============\n{:?}\n=======", text);
606 let mut compile_fail = false;
607 let mut ignore = false;
609 let rendered = if lang.is_null() || origtext.is_empty() {
612 let rlang = (*lang).as_bytes();
613 let rlang = str::from_utf8(rlang).unwrap();
614 let parse_result = LangString::parse(rlang);
615 compile_fail = parse_result.compile_fail;
616 ignore = parse_result.ignore;
617 if !parse_result.rust {
618 (my_opaque.dfltblk)(ob, orig_text, lang,
619 opaque as *const hoedown_renderer_data,
627 let lines = origtext.lines().filter_map(|l| map_line(l).for_html());
628 let text = lines.collect::<Vec<&str>>().join("\n");
629 if rendered { return }
630 PLAYGROUND.with(|play| {
631 // insert newline to clearly separate it from the
632 // previous block so we can shorten the html output
633 let mut s = String::from("\n");
634 let playground_button = play.borrow().as_ref().and_then(|&(ref krate, ref url)| {
638 let test = origtext.lines()
639 .map(|l| map_line(l).for_code())
640 .collect::<Vec<&str>>().join("\n");
641 let krate = krate.as_ref().map(|s| &**s);
642 let (test, _) = test::make_test(&test, krate, false,
643 &Default::default());
644 let channel = if test.contains("#![feature(") {
645 "&version=nightly"
649 // These characters don't need to be escaped in a URI.
650 // FIXME: use a library function for percent encoding.
651 fn dont_escape(c: u8) -> bool {
652 (b'a' <= c && c <= b'z') ||
653 (b'A' <= c && c <= b'Z') ||
654 (b'0' <= c && c <= b'9') ||
655 c == b'-' || c == b'_' || c == b'.' ||
656 c == b'~' || c == b'!' || c == b'\'' ||
657 c == b'(' || c == b')' || c == b'*'
659 let mut test_escaped = String::new();
660 for b in test.bytes() {
662 test_escaped.push(char::from(b));
664 write!(test_escaped, "%{:02X}", b).unwrap();
668 r#"<a class="test-arrow" target="_blank" href="{}?code={}{}">Run</a>"#,
669 url, test_escaped, channel
672 let tooltip = if ignore {
673 Some(("This example is not tested", "ignore"))
674 } else if compile_fail {
675 Some(("This example deliberately fails to compile", "compile_fail"))
679 s.push_str(&highlight::render_with_highlighting(
681 Some(&format!("rust-example-rendered{}",
682 if ignore { " ignore" }
683 else if compile_fail { " compile_fail" }
686 playground_button.as_ref().map(String::as_str),
688 hoedown_buffer_put(ob, s.as_ptr(), s.len());
693 extern fn hoedown_header(ob: *mut hoedown_buffer, text: *const hoedown_buffer,
694 level: libc::c_int, data: *const hoedown_renderer_data,
696 // hoedown does this, we may as well too
697 unsafe { hoedown_buffer_put(ob, "\n".as_ptr(), 1); }
699 // Extract the text provided
700 let s = if text.is_null() {
703 let s = unsafe { (*text).as_bytes() };
704 str::from_utf8(&s).unwrap().to_owned()
707 // Discard '<em>', '<code>' tags and some escaped characters,
708 // transform the contents of the header into a hyphenated string
709 // without non-alphanumeric characters other than '-' and '_'.
711 // This is a terrible hack working around how hoedown gives us rendered
712 // html for text rather than the raw text.
713 let mut id = s.clone();
714 let repl_sub = vec!["<em>", "</em>", "<code>", "</code>",
715 "<strong>", "</strong>",
716 "<", ">", "&", "'", """];
717 for sub in repl_sub {
718 id = id.replace(sub, "");
720 let id = id.chars().filter_map(|c| {
721 if c.is_alphanumeric() || c == '-' || c == '_' {
723 Some(c.to_ascii_lowercase())
727 } else if c.is_whitespace() && c.is_ascii() {
732 }).collect::<String>();
734 let opaque = unsafe { (*data).opaque as *mut hoedown_html_renderer_state };
735 let opaque = unsafe { &mut *((*opaque).opaque as *mut MyOpaque) };
737 let id = derive_id(id);
739 let sec = opaque.toc_builder.as_mut().map_or("".to_owned(), |builder| {
740 format!("{} ", builder.push(level as u32, s.clone(), id.clone()))
744 let text = format!("<h{lvl} id='{id}' class='section-header'>\
745 <a href='#{id}'>{sec}{}</a></h{lvl}>",
746 s, lvl = level, id = id, sec = sec);
748 unsafe { hoedown_buffer_put(ob, text.as_ptr(), text.len()); }
751 extern fn hoedown_codespan(
752 ob: *mut hoedown_buffer,
753 text: *const hoedown_buffer,
754 _: *const hoedown_renderer_data,
757 let content = if text.is_null() {
760 let bytes = unsafe { (*text).as_bytes() };
761 let s = str::from_utf8(bytes).unwrap();
762 collapse_whitespace(s)
765 let content = format!("<code>{}</code>", Escape(&content));
767 hoedown_buffer_put(ob, content.as_ptr(), content.len());
769 // Return anything except 0, which would mean "also print the code span verbatim".
773 pub fn render(w: &mut fmt::Formatter,
775 links: &[(String, String)],
777 html_flags: libc::c_uint) -> fmt::Result {
778 // copied from pulldown-cmark (MIT license, Google)
779 // https://github.com/google/pulldown-cmark
780 // this is temporary till we remove the hoedown renderer
781 static HREF_SAFE: [u8; 128] = [
782 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
783 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
784 0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,
785 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1,
786 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
787 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,
788 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
789 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,
792 static HEX_CHARS: &'static [u8] = b"0123456789ABCDEF";
794 fn escape_href(ob: &mut String, s: &str) {
796 for i in 0..s.len() {
797 let c = s.as_bytes()[i];
798 if c >= 0x80 || HREF_SAFE[c as usize] == 0 {
799 // character needing escape
801 // write partial substring up to mark
803 ob.push_str(&s[mark..i]);
807 ob.push_str("&");
810 ob.push_str("'");
813 let mut buf = [0u8; 3];
815 buf[1] = HEX_CHARS[((c as usize) >> 4) & 0xF];
816 buf[2] = HEX_CHARS[(c as usize) & 0xF];
817 ob.push_str(str::from_utf8(&buf).unwrap());
820 mark = i + 1; // all escaped characters are ASCII
823 ob.push_str(&s[mark..]);
825 // end code copied from pulldown-cmark
827 extern fn hoedown_link(
828 ob: *mut hoedown_buffer,
829 content: *const hoedown_buffer,
830 link: *const hoedown_buffer,
831 title: *const hoedown_buffer,
832 data: *const hoedown_renderer_data,
839 let opaque = unsafe { (*data).opaque as *mut hoedown_html_renderer_state };
840 let opaque = unsafe { &mut *((*opaque).opaque as *mut MyOpaque) };
843 let s = unsafe { (*link).as_bytes() };
844 str::from_utf8(s).unwrap().to_owned()
847 let link = if let Some(&(_, ref new_target)) = opaque.links_replace
849 .find(|t| &*t.0 == &*link) {
850 new_target.to_owned()
855 let content = unsafe {
856 content.as_ref().map(|c| {
857 let s = c.as_bytes();
858 str::from_utf8(s).unwrap().to_owned()
862 let mut link_buf = String::new();
863 escape_href(&mut link_buf, &link);
866 title.as_ref().map(|t| {
867 let s = t.as_bytes();
868 str::from_utf8(s).unwrap().to_owned()
872 let link_out = format!("<a href=\"{link}\"{title}>{content}</a>",
874 title = title.map_or(String::new(),
875 |t| format!(" title=\"{}\"", Escape(&t))),
876 content = content.unwrap_or(String::new()));
878 unsafe { hoedown_buffer_put(ob, link_out.as_ptr(), link_out.len()); }
880 //return "anything but 0" to show we've written the link in
885 let ob = hoedown_buffer_new(DEF_OUNIT);
886 let renderer = hoedown_html_renderer_new(html_flags, 0);
887 let mut opaque = MyOpaque {
888 dfltblk: (*renderer).blockcode.unwrap(),
889 toc_builder: if print_toc {Some(TocBuilder::new())} else {None},
891 links_replace: links.to_vec(),
893 (*((*renderer).opaque as *mut hoedown_html_renderer_state)).opaque
894 = &mut opaque as *mut _ as *mut libc::c_void;
895 (*renderer).blockcode = Some(hoedown_block);
896 (*renderer).header = Some(hoedown_header);
897 (*renderer).codespan = Some(hoedown_codespan);
898 (*renderer).link = Some(hoedown_link);
900 let document = hoedown_document_new(renderer, HOEDOWN_EXTENSIONS, 16);
901 hoedown_document_render(document, ob, s.as_ptr(),
902 s.len() as libc::size_t);
903 hoedown_document_free(document);
905 hoedown_html_renderer_free(renderer);
907 let mut ret = opaque.toc_builder.map_or(Ok(()), |builder| {
908 write!(w, "<nav id=\"TOC\">{}</nav>", builder.into_toc())
912 let buf = (*ob).as_bytes();
913 ret = w.write_str(str::from_utf8(buf).unwrap());
915 hoedown_buffer_free(ob);
920 pub fn old_find_testable_code(doc: &str, tests: &mut ::test::Collector, position: Span) {
921 extern fn block(_ob: *mut hoedown_buffer,
922 text: *const hoedown_buffer,
923 lang: *const hoedown_buffer,
924 data: *const hoedown_renderer_data,
925 line: libc::size_t) {
927 if text.is_null() { return }
928 let block_info = if lang.is_null() {
929 LangString::all_false()
931 let lang = (*lang).as_bytes();
932 let s = str::from_utf8(lang).unwrap();
935 if !block_info.rust { return }
936 let text = (*text).as_bytes();
937 let opaque = (*data).opaque as *mut hoedown_html_renderer_state;
938 let tests = &mut *((*opaque).opaque as *mut ::test::Collector);
939 let text = str::from_utf8(text).unwrap();
940 let lines = text.lines().map(|l| map_line(l).for_code());
941 let text = lines.collect::<Vec<&str>>().join("\n");
942 let filename = tests.get_filename();
944 if tests.render_type == RenderType::Hoedown {
945 let line = tests.get_line() + line;
946 tests.add_test(text.to_owned(),
947 block_info.should_panic, block_info.no_run,
948 block_info.ignore, block_info.test_harness,
949 block_info.compile_fail, block_info.error_codes,
950 line, filename, block_info.allow_fail);
952 tests.add_old_test(text, filename);
957 extern fn header(_ob: *mut hoedown_buffer,
958 text: *const hoedown_buffer,
959 level: libc::c_int, data: *const hoedown_renderer_data,
962 let opaque = (*data).opaque as *mut hoedown_html_renderer_state;
963 let tests = &mut *((*opaque).opaque as *mut ::test::Collector);
965 tests.register_header("", level as u32);
967 let text = (*text).as_bytes();
968 let text = str::from_utf8(text).unwrap();
969 tests.register_header(text, level as u32);
974 tests.set_position(position);
976 let ob = hoedown_buffer_new(DEF_OUNIT);
977 let renderer = hoedown_html_renderer_new(0, 0);
978 (*renderer).blockcode = Some(block);
979 (*renderer).header = Some(header);
980 (*((*renderer).opaque as *mut hoedown_html_renderer_state)).opaque
981 = tests as *mut _ as *mut libc::c_void;
983 let document = hoedown_document_new(renderer, HOEDOWN_EXTENSIONS, 16);
984 hoedown_document_render(document, ob, doc.as_ptr(),
985 doc.len() as libc::size_t);
986 hoedown_document_free(document);
988 hoedown_html_renderer_free(renderer);
989 hoedown_buffer_free(ob);
993 pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector, position: Span) {
994 tests.set_position(position);
996 let mut parser = Parser::new(doc);
997 let mut prev_offset = 0;
998 let mut nb_lines = 0;
999 let mut register_header = None;
1000 'main: while let Some(event) = parser.next() {
1002 Event::Start(Tag::CodeBlock(s)) => {
1003 let block_info = if s.is_empty() {
1004 LangString::all_false()
1006 LangString::parse(&*s)
1008 if !block_info.rust {
1011 let mut test_s = String::new();
1012 let mut offset = None;
1014 let event = parser.next();
1015 if let Some(event) = event {
1017 Event::End(Tag::CodeBlock(_)) => break,
1018 Event::Text(ref s) => {
1020 if offset.is_none() {
1021 offset = Some(parser.get_offset());
1030 let offset = offset.unwrap_or(0);
1031 let lines = test_s.lines().map(|l| map_line(l).for_code());
1032 let text = lines.collect::<Vec<&str>>().join("\n");
1033 nb_lines += doc[prev_offset..offset].lines().count();
1034 let line = tests.get_line() + (nb_lines - 1);
1035 let filename = tests.get_filename();
1036 tests.add_test(text.to_owned(),
1037 block_info.should_panic, block_info.no_run,
1038 block_info.ignore, block_info.test_harness,
1039 block_info.compile_fail, block_info.error_codes,
1040 line, filename, block_info.allow_fail);
1041 prev_offset = offset;
1043 Event::Start(Tag::Header(level)) => {
1044 register_header = Some(level as u32);
1046 Event::Text(ref s) if register_header.is_some() => {
1047 let level = register_header.unwrap();
1049 tests.register_header("", level);
1051 tests.register_header(s, level);
1053 register_header = None;
1060 #[derive(Eq, PartialEq, Clone, Debug)]
1069 error_codes: Vec<String>,
1074 fn all_false() -> LangString {
1076 original: String::new(),
1077 should_panic: false,
1080 rust: true, // NB This used to be `notrust = false`
1081 test_harness: false,
1082 compile_fail: false,
1083 error_codes: Vec::new(),
1088 fn parse(string: &str) -> LangString {
1089 let mut seen_rust_tags = false;
1090 let mut seen_other_tags = false;
1091 let mut data = LangString::all_false();
1092 let mut allow_error_code_check = false;
1093 if UnstableFeatures::from_environment().is_nightly_build() {
1094 allow_error_code_check = true;
1097 data.original = string.to_owned();
1098 let tokens = string.split(|c: char|
1099 !(c == '_' || c == '-' || c.is_alphanumeric())
1102 for token in tokens {
1103 match token.trim() {
1106 data.should_panic = true;
1107 seen_rust_tags = seen_other_tags == false;
1109 "no_run" => { data.no_run = true; seen_rust_tags = !seen_other_tags; }
1110 "ignore" => { data.ignore = true; seen_rust_tags = !seen_other_tags; }
1111 "allow_fail" => { data.allow_fail = true; seen_rust_tags = !seen_other_tags; }
1112 "rust" => { data.rust = true; seen_rust_tags = true; }
1114 data.test_harness = true;
1115 seen_rust_tags = !seen_other_tags || seen_rust_tags;
1118 data.compile_fail = true;
1119 seen_rust_tags = !seen_other_tags || seen_rust_tags;
1122 x if allow_error_code_check && x.starts_with("E") && x.len() == 5 => {
1123 if let Ok(_) = x[1..].parse::<u32>() {
1124 data.error_codes.push(x.to_owned());
1125 seen_rust_tags = !seen_other_tags || seen_rust_tags;
1127 seen_other_tags = true;
1130 _ => { seen_other_tags = true }
1134 data.rust &= !seen_other_tags || seen_rust_tags;
1140 impl<'a> fmt::Display for Markdown<'a> {
1141 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1142 let Markdown(md, links, render_type) = *self;
1144 // This is actually common enough to special-case
1145 if md.is_empty() { return Ok(()) }
1146 if render_type == RenderType::Hoedown {
1147 render(fmt, md, links, false, 0)
1149 let mut opts = Options::empty();
1150 opts.insert(OPTION_ENABLE_TABLES);
1151 opts.insert(OPTION_ENABLE_FOOTNOTES);
1153 let p = Parser::new_ext(md, opts);
1155 let mut s = String::with_capacity(md.len() * 3 / 2);
1157 html::push_html(&mut s,
1161 HeadingLinks::new(p, None),
1169 impl<'a> fmt::Display for MarkdownWithToc<'a> {
1170 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1171 let MarkdownWithToc(md, render_type) = *self;
1173 if render_type == RenderType::Hoedown {
1174 render(fmt, md, &[], true, 0)
1176 let mut opts = Options::empty();
1177 opts.insert(OPTION_ENABLE_TABLES);
1178 opts.insert(OPTION_ENABLE_FOOTNOTES);
1180 let p = Parser::new_ext(md, opts);
1182 let mut s = String::with_capacity(md.len() * 3 / 2);
1184 let mut toc = TocBuilder::new();
1186 html::push_html(&mut s,
1187 Footnotes::new(CodeBlocks::new(HeadingLinks::new(p, Some(&mut toc)))));
1189 write!(fmt, "<nav id=\"TOC\">{}</nav>", toc.into_toc())?;
1196 impl<'a> fmt::Display for MarkdownHtml<'a> {
1197 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1198 let MarkdownHtml(md, render_type) = *self;
1200 // This is actually common enough to special-case
1201 if md.is_empty() { return Ok(()) }
1202 if render_type == RenderType::Hoedown {
1203 render(fmt, md, &[], false, HOEDOWN_HTML_ESCAPE)
1205 let mut opts = Options::empty();
1206 opts.insert(OPTION_ENABLE_TABLES);
1207 opts.insert(OPTION_ENABLE_FOOTNOTES);
1209 let p = Parser::new_ext(md, opts);
1211 // Treat inline HTML as plain text.
1212 let p = p.map(|event| match event {
1213 Event::Html(text) | Event::InlineHtml(text) => Event::Text(text),
1217 let mut s = String::with_capacity(md.len() * 3 / 2);
1219 html::push_html(&mut s,
1220 Footnotes::new(CodeBlocks::new(HeadingLinks::new(p, None))));
1227 impl<'a> fmt::Display for MarkdownSummaryLine<'a> {
1228 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1229 let MarkdownSummaryLine(md, links) = *self;
1230 // This is actually common enough to special-case
1231 if md.is_empty() { return Ok(()) }
1233 let p = Parser::new(md);
1235 let mut s = String::new();
1237 html::push_html(&mut s, LinkReplacer::new(SummaryLine::new(p), links));
1243 pub fn plain_summary_line(md: &str) -> String {
1244 struct ParserWrapper<'a> {
1250 impl<'a> Iterator for ParserWrapper<'a> {
1253 fn next(&mut self) -> Option<String> {
1254 let next_event = self.inner.next();
1255 if next_event.is_none() {
1258 let next_event = next_event.unwrap();
1259 let (ret, is_in) = match next_event {
1260 Event::Start(Tag::Paragraph) => (None, 1),
1261 Event::Start(Tag::Code) => (Some("`".to_owned()), 1),
1262 Event::End(Tag::Code) => (Some("`".to_owned()), -1),
1263 Event::Start(Tag::Header(_)) => (None, 1),
1264 Event::Text(ref s) if self.is_in > 0 => (Some(s.as_ref().to_owned()), 0),
1265 Event::End(Tag::Paragraph) | Event::End(Tag::Header(_)) => (None, -1),
1268 if is_in > 0 || (is_in < 0 && self.is_in > 0) {
1269 self.is_in += is_in;
1272 self.is_first = false;
1279 let mut s = String::with_capacity(md.len() * 3 / 2);
1280 let mut p = ParserWrapper {
1281 inner: Parser::new(md),
1285 while let Some(t) = p.next() {
1293 pub fn markdown_links(md: &str, render_type: RenderType) -> Vec<String> {
1299 RenderType::Hoedown => {
1300 extern fn hoedown_link(
1301 _ob: *mut hoedown_buffer,
1302 _content: *const hoedown_buffer,
1303 link: *const hoedown_buffer,
1304 _title: *const hoedown_buffer,
1305 data: *const hoedown_renderer_data,
1312 let opaque = unsafe { (*data).opaque as *mut hoedown_html_renderer_state };
1313 let opaque = unsafe { &mut *((*opaque).opaque as *mut MyOpaque) };
1315 if let Some(ref mut links) = opaque.links_out {
1316 let s = unsafe { (*link).as_bytes() };
1317 let s = str::from_utf8(&s).unwrap().to_owned();
1319 debug!("found link: {}", s);
1324 //returning 0 here means "emit the span verbatim", but we're not using the output
1325 //anyway so we don't really care
1330 let ob = hoedown_buffer_new(DEF_OUNIT);
1331 let renderer = hoedown_html_renderer_new(0, 0);
1332 let mut opaque = MyOpaque {
1333 dfltblk: (*renderer).blockcode.unwrap(),
1335 links_out: Some(vec![]),
1336 links_replace: vec![],
1338 (*((*renderer).opaque as *mut hoedown_html_renderer_state)).opaque
1339 = &mut opaque as *mut _ as *mut libc::c_void;
1340 (*renderer).header = Some(hoedown_header);
1341 (*renderer).codespan = Some(hoedown_codespan);
1342 (*renderer).link = Some(hoedown_link);
1344 let document = hoedown_document_new(renderer, HOEDOWN_EXTENSIONS, 16);
1345 hoedown_document_render(document, ob, md.as_ptr(),
1346 md.len() as libc::size_t);
1347 hoedown_document_free(document);
1349 hoedown_html_renderer_free(renderer);
1350 hoedown_buffer_free(ob);
1352 opaque.links_out.unwrap()
1355 RenderType::Pulldown => {
1356 let mut opts = Options::empty();
1357 opts.insert(OPTION_ENABLE_TABLES);
1358 opts.insert(OPTION_ENABLE_FOOTNOTES);
1360 let p = Parser::new_ext(md, opts);
1362 let iter = Footnotes::new(HeadingLinks::new(p, None));
1363 let mut links = vec![];
1366 if let Event::Start(Tag::Link(dest, _)) = ev {
1367 debug!("found link: {}", dest);
1368 links.push(dest.into_owned());
1379 use super::{LangString, Markdown, MarkdownHtml};
1380 use super::plain_summary_line;
1381 use super::RenderType;
1382 use html::render::reset_ids;
1385 fn test_lang_string_parse() {
1387 should_panic: bool, no_run: bool, ignore: bool, rust: bool, test_harness: bool,
1388 compile_fail: bool, allow_fail: bool, error_codes: Vec<String>) {
1389 assert_eq!(LangString::parse(s), LangString {
1397 original: s.to_owned(),
1402 fn v() -> Vec<String> {
1406 // marker | should_panic| no_run| ignore| rust | test_harness| compile_fail
1407 // | allow_fail | error_codes
1408 t("", false, false, false, true, false, false, false, v());
1409 t("rust", false, false, false, true, false, false, false, v());
1410 t("sh", false, false, false, false, false, false, false, v());
1411 t("ignore", false, false, true, true, false, false, false, v());
1412 t("should_panic", true, false, false, true, false, false, false, v());
1413 t("no_run", false, true, false, true, false, false, false, v());
1414 t("test_harness", false, false, false, true, true, false, false, v());
1415 t("compile_fail", false, true, false, true, false, true, false, v());
1416 t("allow_fail", false, false, false, true, false, false, true, v());
1417 t("{.no_run .example}", false, true, false, true, false, false, false, v());
1418 t("{.sh .should_panic}", true, false, false, false, false, false, false, v());
1419 t("{.example .rust}", false, false, false, true, false, false, false, v());
1420 t("{.test_harness .rust}", false, false, false, true, true, false, false, v());
1421 t("text, no_run", false, true, false, false, false, false, false, v());
1422 t("text,no_run", false, true, false, false, false, false, false, v());
1427 let markdown = "# title";
1428 format!("{}", Markdown(markdown, &[], RenderType::Pulldown));
1434 fn t(input: &str, expect: &str) {
1435 let output = format!("{}", Markdown(input, &[], RenderType::Pulldown));
1436 assert_eq!(output, expect, "original: {}", input);
1440 t("# Foo bar", "<h1 id=\"foo-bar\" class=\"section-header\">\
1441 <a href=\"#foo-bar\">Foo bar</a></h1>");
1442 t("## Foo-bar_baz qux", "<h2 id=\"foo-bar_baz-qux\" class=\"section-\
1443 header\"><a href=\"#foo-bar_baz-qux\">Foo-bar_baz qux</a></h2>");
1444 t("### **Foo** *bar* baz!?!& -_qux_-%",
1445 "<h3 id=\"foo-bar-baz--qux-\" class=\"section-header\">\
1446 <a href=\"#foo-bar-baz--qux-\"><strong>Foo</strong> \
1447 <em>bar</em> baz!?!& -<em>qux</em>-%</a></h3>");
1448 t("#### **Foo?** & \\*bar?!* _`baz`_ ❤ #qux",
1449 "<h4 id=\"foo--bar--baz--qux\" class=\"section-header\">\
1450 <a href=\"#foo--bar--baz--qux\"><strong>Foo?</strong> & *bar?!* \
1451 <em><code>baz</code></em> ❤ #qux</a></h4>");
1455 fn test_header_ids_multiple_blocks() {
1456 fn t(input: &str, expect: &str) {
1457 let output = format!("{}", Markdown(input, &[], RenderType::Pulldown));
1458 assert_eq!(output, expect, "original: {}", input);
1462 t("# Example", "<h1 id=\"example\" class=\"section-header\">\
1463 <a href=\"#example\">Example</a></h1>");
1464 t("# Panics", "<h1 id=\"panics\" class=\"section-header\">\
1465 <a href=\"#panics\">Panics</a></h1>");
1466 t("# Example", "<h1 id=\"example-1\" class=\"section-header\">\
1467 <a href=\"#example-1\">Example</a></h1>");
1468 t("# Main", "<h1 id=\"main-1\" class=\"section-header\">\
1469 <a href=\"#main-1\">Main</a></h1>");
1470 t("# Example", "<h1 id=\"example-2\" class=\"section-header\">\
1471 <a href=\"#example-2\">Example</a></h1>");
1472 t("# Panics", "<h1 id=\"panics-1\" class=\"section-header\">\
1473 <a href=\"#panics-1\">Panics</a></h1>");
1481 fn test_plain_summary_line() {
1482 fn t(input: &str, expect: &str) {
1483 let output = plain_summary_line(input);
1484 assert_eq!(output, expect, "original: {}", input);
1487 t("hello [Rust](https://www.rust-lang.org) :)", "hello Rust :)");
1488 t("hello [Rust](https://www.rust-lang.org \"Rust\") :)", "hello Rust :)");
1489 t("code `let x = i32;` ...", "code `let x = i32;` ...");
1490 t("type `Type<'static>` ...", "type `Type<'static>` ...");
1491 t("# top header", "top header");
1492 t("## header", "header");
1496 fn test_markdown_html_escape() {
1497 fn t(input: &str, expect: &str) {
1498 let output = format!("{}", MarkdownHtml(input, RenderType::Pulldown));
1499 assert_eq!(output, expect, "original: {}", input);
1502 t("`Struct<'a, T>`", "<p><code>Struct<'a, T></code></p>\n");
1503 t("Struct<'a, T>", "<p>Struct<'a, T></p>\n");
1504 t("Struct<br>", "<p>Struct<br></p>\n");