1 // Copyright 2013-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! Markdown formatting for rustdoc
13 //! This module implements markdown formatting through the hoedown C-library
14 //! (bundled into the rust runtime). This module self-contains the C bindings
15 //! and necessary legwork to render markdown, and exposes all of the
16 //! functionality through a unit-struct, `Markdown`, which has an implementation
17 //! of `fmt::Display`. Example usage:
20 //! use rustdoc::html::markdown::Markdown;
22 //! let s = "My *markdown* _text_";
23 //! let html = format!("{}", Markdown(s));
24 //! // ... something using html
27 #![allow(non_camel_case_types)]
30 use std::ascii::AsciiExt;
31 use std::cell::RefCell;
32 use std::default::Default;
33 use std::ffi::CString;
38 use html::render::derive_id;
39 use html::toc::TocBuilder;
41 use html::escape::Escape;
44 /// A unit struct which has the `fmt::Display` trait implemented. When
45 /// formatted, this struct will emit the HTML corresponding to the rendered
46 /// version of the contained markdown string.
47 pub struct Markdown<'a>(pub &'a str);
48 /// A unit struct like `Markdown`, that renders the markdown with a
49 /// table of contents.
50 pub struct MarkdownWithToc<'a>(pub &'a str);
52 const DEF_OUNIT: libc::size_t = 64;
53 const HOEDOWN_EXT_NO_INTRA_EMPHASIS: libc::c_uint = 1 << 11;
54 const HOEDOWN_EXT_TABLES: libc::c_uint = 1 << 0;
55 const HOEDOWN_EXT_FENCED_CODE: libc::c_uint = 1 << 1;
56 const HOEDOWN_EXT_AUTOLINK: libc::c_uint = 1 << 3;
57 const HOEDOWN_EXT_STRIKETHROUGH: libc::c_uint = 1 << 4;
58 const HOEDOWN_EXT_SUPERSCRIPT: libc::c_uint = 1 << 8;
59 const HOEDOWN_EXT_FOOTNOTES: libc::c_uint = 1 << 2;
61 const HOEDOWN_EXTENSIONS: libc::c_uint =
62 HOEDOWN_EXT_NO_INTRA_EMPHASIS | HOEDOWN_EXT_TABLES |
63 HOEDOWN_EXT_FENCED_CODE | HOEDOWN_EXT_AUTOLINK |
64 HOEDOWN_EXT_STRIKETHROUGH | HOEDOWN_EXT_SUPERSCRIPT |
65 HOEDOWN_EXT_FOOTNOTES;
67 enum hoedown_document {}
69 type blockcodefn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
70 *const hoedown_buffer, *const hoedown_renderer_data);
72 type blockquotefn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
73 *const hoedown_renderer_data);
75 type headerfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
76 libc::c_int, *const hoedown_renderer_data);
78 type blockhtmlfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
79 *const hoedown_renderer_data);
81 type codespanfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
82 *const hoedown_renderer_data) -> libc::c_int;
84 type linkfn = extern "C" fn (*mut hoedown_buffer, *const hoedown_buffer,
85 *const hoedown_buffer, *const hoedown_buffer,
86 *const hoedown_renderer_data) -> libc::c_int;
88 type entityfn = extern "C" fn (*mut hoedown_buffer, *const hoedown_buffer,
89 *const hoedown_renderer_data);
91 type normaltextfn = extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
92 *const hoedown_renderer_data);
95 struct hoedown_renderer_data {
96 opaque: *mut libc::c_void,
100 struct hoedown_renderer {
101 opaque: *mut libc::c_void,
103 blockcode: Option<blockcodefn>,
104 blockquote: Option<blockquotefn>,
105 header: Option<headerfn>,
107 other_block_level_callbacks: [libc::size_t; 11],
109 blockhtml: Option<blockhtmlfn>,
111 /* span level callbacks - NULL or return 0 prints the span verbatim */
112 autolink: libc::size_t, // unused
113 codespan: Option<codespanfn>,
114 other_span_level_callbacks_1: [libc::size_t; 7],
115 link: Option<linkfn>,
116 other_span_level_callbacks_2: [libc::size_t; 6],
118 /* low level callbacks - NULL copies input directly into the output */
119 entity: Option<entityfn>,
120 normal_text: Option<normaltextfn>,
122 /* header and footer */
123 other_callbacks: [libc::size_t; 2],
127 struct hoedown_html_renderer_state {
128 opaque: *mut libc::c_void,
129 toc_data: html_toc_data,
131 link_attributes: Option<extern "C" fn(*mut hoedown_buffer,
132 *const hoedown_buffer,
133 *const hoedown_renderer_data)>,
137 struct html_toc_data {
138 header_count: libc::c_int,
139 current_level: libc::c_int,
140 level_offset: libc::c_int,
141 nesting_level: libc::c_int,
145 dfltblk: extern "C" fn(*mut hoedown_buffer, *const hoedown_buffer,
146 *const hoedown_buffer, *const hoedown_renderer_data),
147 toc_builder: Option<TocBuilder>,
151 struct hoedown_buffer {
159 #[link(name = "hoedown", kind = "static")]
160 #[cfg(not(cargobuild))]
164 fn hoedown_html_renderer_new(render_flags: libc::c_uint,
165 nesting_level: libc::c_int)
166 -> *mut hoedown_renderer;
167 fn hoedown_html_renderer_free(renderer: *mut hoedown_renderer);
169 fn hoedown_document_new(rndr: *const hoedown_renderer,
170 extensions: libc::c_uint,
171 max_nesting: libc::size_t) -> *mut hoedown_document;
172 fn hoedown_document_render(doc: *mut hoedown_document,
173 ob: *mut hoedown_buffer,
175 doc_size: libc::size_t);
176 fn hoedown_document_free(md: *mut hoedown_document);
178 fn hoedown_buffer_new(unit: libc::size_t) -> *mut hoedown_buffer;
179 fn hoedown_buffer_put(b: *mut hoedown_buffer, c: *const libc::c_char,
181 fn hoedown_buffer_puts(b: *mut hoedown_buffer, c: *const libc::c_char);
182 fn hoedown_buffer_free(b: *mut hoedown_buffer);
186 // hoedown_buffer helpers
187 impl hoedown_buffer {
188 fn as_bytes(&self) -> &[u8] {
189 unsafe { slice::from_raw_parts(self.data, self.size as usize) }
193 /// Returns Some(code) if `s` is a line that should be stripped from
194 /// documentation but used in example code. `code` is the portion of
195 /// `s` that should be used in tests. (None for lines that should be
197 fn stripped_filtered_line<'a>(s: &'a str) -> Option<&'a str> {
198 let trimmed = s.trim();
201 } else if trimmed.starts_with("# ") {
208 /// Returns a new string with all consecutive whitespace collapsed into
211 /// Any leading or trailing whitespace will be trimmed.
212 fn collapse_whitespace(s: &str) -> String {
213 s.split_whitespace().collect::<Vec<_>>().join(" ")
216 thread_local!(pub static PLAYGROUND_KRATE: RefCell<Option<Option<String>>> = {
220 pub fn render(w: &mut fmt::Formatter, s: &str, print_toc: bool) -> fmt::Result {
221 extern fn block(ob: *mut hoedown_buffer, orig_text: *const hoedown_buffer,
222 lang: *const hoedown_buffer, data: *const hoedown_renderer_data) {
224 if orig_text.is_null() { return }
226 let opaque = (*data).opaque as *mut hoedown_html_renderer_state;
227 let my_opaque: &MyOpaque = &*((*opaque).opaque as *const MyOpaque);
228 let text = (*orig_text).as_bytes();
229 let origtext = str::from_utf8(text).unwrap();
230 debug!("docblock: ==============\n{:?}\n=======", text);
231 let rendered = if lang.is_null() {
234 let rlang = (*lang).as_bytes();
235 let rlang = str::from_utf8(rlang).unwrap();
236 if !LangString::parse(rlang).rust {
237 (my_opaque.dfltblk)(ob, orig_text, lang,
238 opaque as *const hoedown_renderer_data);
245 let lines = origtext.lines().filter(|l| {
246 stripped_filtered_line(*l).is_none()
248 let text = lines.collect::<Vec<&str>>().join("\n");
249 if rendered { return }
250 PLAYGROUND_KRATE.with(|krate| {
251 let mut s = String::new();
252 krate.borrow().as_ref().map(|krate| {
253 let test = origtext.lines().map(|l| {
254 stripped_filtered_line(l).unwrap_or(l)
255 }).collect::<Vec<&str>>().join("\n");
256 let krate = krate.as_ref().map(|s| &**s);
257 let test = test::maketest(&test, krate, false,
258 &Default::default());
259 s.push_str(&format!("<span class='rusttest'>{}</span>", Escape(&test)));
261 s.push_str(&highlight::highlight(&text,
262 Some("rust-example-rendered"),
264 let output = CString::new(s).unwrap();
265 hoedown_buffer_puts(ob, output.as_ptr());
270 extern fn header(ob: *mut hoedown_buffer, text: *const hoedown_buffer,
271 level: libc::c_int, data: *const hoedown_renderer_data) {
272 // hoedown does this, we may as well too
273 unsafe { hoedown_buffer_puts(ob, "\n\0".as_ptr() as *const _); }
275 // Extract the text provided
276 let s = if text.is_null() {
279 let s = unsafe { (*text).as_bytes() };
280 str::from_utf8(&s).unwrap().to_owned()
283 // Discard '<em>', '<code>' tags and some escaped characters,
284 // transform the contents of the header into a hyphenated string
285 // without non-alphanumeric characters other than '-' and '_'.
287 // This is a terrible hack working around how hoedown gives us rendered
288 // html for text rather than the raw text.
289 let mut id = s.clone();
290 let repl_sub = vec!["<em>", "</em>", "<code>", "</code>",
291 "<strong>", "</strong>",
292 "<", ">", "&", "'", """];
293 for sub in repl_sub {
294 id = id.replace(sub, "");
296 let id = id.chars().filter_map(|c| {
297 if c.is_alphanumeric() || c == '-' || c == '_' {
299 Some(c.to_ascii_lowercase())
303 } else if c.is_whitespace() && c.is_ascii() {
308 }).collect::<String>();
310 let opaque = unsafe { (*data).opaque as *mut hoedown_html_renderer_state };
311 let opaque = unsafe { &mut *((*opaque).opaque as *mut MyOpaque) };
313 let id = derive_id(id);
315 let sec = opaque.toc_builder.as_mut().map_or("".to_owned(), |builder| {
316 format!("{} ", builder.push(level as u32, s.clone(), id.clone()))
320 let text = format!("<h{lvl} id='{id}' class='section-header'>\
321 <a href='#{id}'>{sec}{}</a></h{lvl}>",
322 s, lvl = level, id = id, sec = sec);
324 let text = CString::new(text).unwrap();
325 unsafe { hoedown_buffer_puts(ob, text.as_ptr()) }
329 ob: *mut hoedown_buffer,
330 text: *const hoedown_buffer,
331 _: *const hoedown_renderer_data,
333 let content = if text.is_null() {
336 let bytes = unsafe { (*text).as_bytes() };
337 let s = str::from_utf8(bytes).unwrap();
338 collapse_whitespace(s)
341 let content = format!("<code>{}</code>", Escape(&content));
342 let element = CString::new(content).unwrap();
343 unsafe { hoedown_buffer_puts(ob, element.as_ptr()); }
344 // Return anything except 0, which would mean "also print the code span verbatim".
349 let ob = hoedown_buffer_new(DEF_OUNIT);
350 let renderer = hoedown_html_renderer_new(0, 0);
351 let mut opaque = MyOpaque {
352 dfltblk: (*renderer).blockcode.unwrap(),
353 toc_builder: if print_toc {Some(TocBuilder::new())} else {None}
355 (*((*renderer).opaque as *mut hoedown_html_renderer_state)).opaque
356 = &mut opaque as *mut _ as *mut libc::c_void;
357 (*renderer).blockcode = Some(block);
358 (*renderer).header = Some(header);
359 (*renderer).codespan = Some(codespan);
361 let document = hoedown_document_new(renderer, HOEDOWN_EXTENSIONS, 16);
362 hoedown_document_render(document, ob, s.as_ptr(),
363 s.len() as libc::size_t);
364 hoedown_document_free(document);
366 hoedown_html_renderer_free(renderer);
368 let mut ret = opaque.toc_builder.map_or(Ok(()), |builder| {
369 write!(w, "<nav id=\"TOC\">{}</nav>", builder.into_toc())
373 let buf = (*ob).as_bytes();
374 ret = w.write_str(str::from_utf8(buf).unwrap());
376 hoedown_buffer_free(ob);
381 pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector) {
382 extern fn block(_ob: *mut hoedown_buffer,
383 text: *const hoedown_buffer,
384 lang: *const hoedown_buffer,
385 data: *const hoedown_renderer_data) {
387 if text.is_null() { return }
388 let block_info = if lang.is_null() {
389 LangString::all_false()
391 let lang = (*lang).as_bytes();
392 let s = str::from_utf8(lang).unwrap();
395 if !block_info.rust { return }
396 let text = (*text).as_bytes();
397 let opaque = (*data).opaque as *mut hoedown_html_renderer_state;
398 let tests = &mut *((*opaque).opaque as *mut ::test::Collector);
399 let text = str::from_utf8(text).unwrap();
400 let lines = text.lines().map(|l| {
401 stripped_filtered_line(l).unwrap_or(l)
403 let text = lines.collect::<Vec<&str>>().join("\n");
404 tests.add_test(text.to_owned(),
405 block_info.should_panic, block_info.no_run,
406 block_info.ignore, block_info.test_harness);
410 extern fn header(_ob: *mut hoedown_buffer,
411 text: *const hoedown_buffer,
412 level: libc::c_int, data: *const hoedown_renderer_data) {
414 let opaque = (*data).opaque as *mut hoedown_html_renderer_state;
415 let tests = &mut *((*opaque).opaque as *mut ::test::Collector);
417 tests.register_header("", level as u32);
419 let text = (*text).as_bytes();
420 let text = str::from_utf8(text).unwrap();
421 tests.register_header(text, level as u32);
427 let ob = hoedown_buffer_new(DEF_OUNIT);
428 let renderer = hoedown_html_renderer_new(0, 0);
429 (*renderer).blockcode = Some(block);
430 (*renderer).header = Some(header);
431 (*((*renderer).opaque as *mut hoedown_html_renderer_state)).opaque
432 = tests as *mut _ as *mut libc::c_void;
434 let document = hoedown_document_new(renderer, HOEDOWN_EXTENSIONS, 16);
435 hoedown_document_render(document, ob, doc.as_ptr(),
436 doc.len() as libc::size_t);
437 hoedown_document_free(document);
439 hoedown_html_renderer_free(renderer);
440 hoedown_buffer_free(ob);
444 #[derive(Eq, PartialEq, Clone, Debug)]
454 fn all_false() -> LangString {
459 rust: true, // NB This used to be `notrust = false`
464 fn parse(string: &str) -> LangString {
465 let mut seen_rust_tags = false;
466 let mut seen_other_tags = false;
467 let mut data = LangString::all_false();
469 let tokens = string.split(|c: char|
470 !(c == '_' || c == '-' || c.is_alphanumeric())
473 for token in tokens {
476 "should_panic" => { data.should_panic = true; seen_rust_tags = true; },
477 "no_run" => { data.no_run = true; seen_rust_tags = true; },
478 "ignore" => { data.ignore = true; seen_rust_tags = true; },
479 "rust" => { data.rust = true; seen_rust_tags = true; },
480 "test_harness" => { data.test_harness = true; seen_rust_tags = true; }
481 _ => { seen_other_tags = true }
485 data.rust &= !seen_other_tags || seen_rust_tags;
491 impl<'a> fmt::Display for Markdown<'a> {
492 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
493 let Markdown(md) = *self;
494 // This is actually common enough to special-case
495 if md.is_empty() { return Ok(()) }
496 render(fmt, md, false)
500 impl<'a> fmt::Display for MarkdownWithToc<'a> {
501 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
502 let MarkdownWithToc(md) = *self;
503 render(fmt, md, true)
507 pub fn plain_summary_line(md: &str) -> String {
508 extern fn link(_ob: *mut hoedown_buffer,
509 _link: *const hoedown_buffer,
510 _title: *const hoedown_buffer,
511 content: *const hoedown_buffer,
512 data: *const hoedown_renderer_data) -> libc::c_int
515 if !content.is_null() && (*content).size > 0 {
516 let ob = (*data).opaque as *mut hoedown_buffer;
517 hoedown_buffer_put(ob, (*content).data as *const libc::c_char,
524 extern fn normal_text(_ob: *mut hoedown_buffer,
525 text: *const hoedown_buffer,
526 data: *const hoedown_renderer_data)
529 let ob = (*data).opaque as *mut hoedown_buffer;
530 hoedown_buffer_put(ob, (*text).data as *const libc::c_char,
536 let ob = hoedown_buffer_new(DEF_OUNIT);
537 let mut plain_renderer: hoedown_renderer = ::std::mem::zeroed();
538 let renderer: *mut hoedown_renderer = &mut plain_renderer;
539 (*renderer).opaque = ob as *mut libc::c_void;
540 (*renderer).link = Some(link);
541 (*renderer).normal_text = Some(normal_text);
543 let document = hoedown_document_new(renderer, HOEDOWN_EXTENSIONS, 16);
544 hoedown_document_render(document, ob, md.as_ptr(),
545 md.len() as libc::size_t);
546 hoedown_document_free(document);
547 let plain_slice = (*ob).as_bytes();
548 let plain = str::from_utf8(plain_slice).unwrap_or("").to_owned();
549 hoedown_buffer_free(ob);
556 use super::{LangString, Markdown};
557 use super::plain_summary_line;
558 use html::render::reset_ids;
561 fn test_lang_string_parse() {
563 should_panic: bool, no_run: bool, ignore: bool, rust: bool, test_harness: bool) {
564 assert_eq!(LangString::parse(s), LangString {
565 should_panic: should_panic,
569 test_harness: test_harness,
573 // marker | should_panic| no_run | ignore | rust | test_harness
574 t("", false, false, false, true, false);
575 t("rust", false, false, false, true, false);
576 t("sh", false, false, false, false, false);
577 t("ignore", false, false, true, true, false);
578 t("should_panic", true, false, false, true, false);
579 t("no_run", false, true, false, true, false);
580 t("test_harness", false, false, false, true, true);
581 t("{.no_run .example}", false, true, false, true, false);
582 t("{.sh .should_panic}", true, false, false, true, false);
583 t("{.example .rust}", false, false, false, true, false);
584 t("{.test_harness .rust}", false, false, false, true, true);
589 let markdown = "# title";
590 format!("{}", Markdown(markdown));
596 fn t(input: &str, expect: &str) {
597 let output = format!("{}", Markdown(input));
598 assert_eq!(output, expect);
602 t("# Foo bar", "\n<h1 id='foo-bar' class='section-header'>\
603 <a href='#foo-bar'>Foo bar</a></h1>");
604 t("## Foo-bar_baz qux", "\n<h2 id='foo-bar_baz-qux' class=\'section-\
605 header'><a href='#foo-bar_baz-qux'>Foo-bar_baz qux</a></h2>");
606 t("### **Foo** *bar* baz!?!& -_qux_-%",
607 "\n<h3 id='foo-bar-baz--_qux_-' class='section-header'>\
608 <a href='#foo-bar-baz--_qux_-'><strong>Foo</strong> \
609 <em>bar</em> baz!?!& -_qux_-%</a></h3>");
610 t("####**Foo?** & \\*bar?!* _`baz`_ ❤ #qux",
611 "\n<h4 id='foo--bar--baz--qux' class='section-header'>\
612 <a href='#foo--bar--baz--qux'><strong>Foo?</strong> & *bar?!* \
613 <em><code>baz</code></em> ❤ #qux</a></h4>");
617 fn test_header_ids_multiple_blocks() {
618 fn t(input: &str, expect: &str) {
619 let output = format!("{}", Markdown(input));
620 assert_eq!(output, expect);
624 t("# Example", "\n<h1 id='example' class='section-header'>\
625 <a href='#example'>Example</a></h1>");
626 t("# Panics", "\n<h1 id='panics' class='section-header'>\
627 <a href='#panics'>Panics</a></h1>");
628 t("# Example", "\n<h1 id='example-1' class='section-header'>\
629 <a href='#example-1'>Example</a></h1>");
630 t("# Main", "\n<h1 id='main-1' class='section-header'>\
631 <a href='#main-1'>Main</a></h1>");
632 t("# Example", "\n<h1 id='example-2' class='section-header'>\
633 <a href='#example-2'>Example</a></h1>");
634 t("# Panics", "\n<h1 id='panics-1' class='section-header'>\
635 <a href='#panics-1'>Panics</a></h1>");
643 fn test_plain_summary_line() {
644 fn t(input: &str, expect: &str) {
645 let output = plain_summary_line(input);
646 assert_eq!(output, expect);
649 t("hello [Rust](https://www.rust-lang.org) :)", "hello Rust :)");
650 t("code `let x = i32;` ...", "code `let x = i32;` ...");
651 t("type `Type<'static>` ...", "type `Type<'static>` ...");
652 t("# top header", "top header");
653 t("## header", "header");