1 //! Basic syntax highlighting functionality.
3 //! This module uses libsyntax's lexer to provide token-based highlighting for
4 //! the HTML documentation generated by rustdoc.
6 //! Use the `render_with_highlighting` to highlight some rust code.
8 use crate::html::escape::Escape;
10 use std::fmt::Display;
12 use std::io::prelude::*;
14 use rustc_parse::lexer;
15 use rustc_session::parse::ParseSess;
16 use rustc_span::source_map::SourceMap;
17 use rustc_span::symbol::{kw, sym};
18 use rustc_span::{FileName, Span};
19 use syntax::token::{self, Token};
21 /// Highlights `src`, returning the HTML output.
22 pub fn render_with_highlighting(
25 playground_button: Option<&str>,
26 tooltip: Option<(&str, &str)>,
28 debug!("highlighting: ================\n{}\n==============", src);
29 let mut out = Vec::new();
30 if let Some((tooltip, class)) = tooltip {
33 "<div class='information'><div class='tooltip {}'>ⓘ<span \
34 class='tooltiptext'>{}</span></div></div>",
40 let sess = ParseSess::with_silent_emitter();
43 .new_source_file(FileName::Custom(String::from("rustdoc-highlighting")), src.to_owned());
44 let highlight_result = rustc_driver::catch_fatal_errors(|| {
45 let lexer = lexer::StringReader::new(&sess, sf, None);
46 let mut classifier = Classifier::new(lexer, sess.source_map());
48 let mut highlighted_source = vec![];
49 if classifier.write_source(&mut highlighted_source).is_err() {
52 Ok(String::from_utf8_lossy(&highlighted_source).into_owned())
57 match highlight_result {
58 Ok(highlighted_source) => {
59 write_header(class, &mut out).unwrap();
60 write!(out, "{}", highlighted_source).unwrap();
61 write_footer(&mut out, playground_button).unwrap();
64 // If errors are encountered while trying to highlight, just emit
65 // the unhighlighted source.
66 write!(out, "<pre><code>{}</code></pre>", Escape(src)).unwrap();
70 String::from_utf8_lossy(&out[..]).into_owned()
73 /// Processes a program (nested in the internal `lexer`), classifying strings of
74 /// text by highlighting category (`Class`). Calls out to a `Writer` to write
75 /// each span of text in sequence.
76 struct Classifier<'a> {
77 lexer: lexer::StringReader<'a>,
78 peek_token: Option<Token>,
79 source_map: &'a SourceMap,
81 // State of the classifier.
84 in_macro_nonterminal: bool,
87 /// How a span of text is classified. Mostly corresponds to token kinds.
88 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
95 // Keywords that do pointer/reference stuff.
111 /// Trait that controls writing the output of syntax highlighting. Users should
112 /// implement this trait to customize writing output.
114 /// The classifier will call into the `Writer` implementation as it finds spans
115 /// of text to highlight. Exactly how that text should be highlighted is up to
116 /// the implementation.
118 /// Called when we start processing a span of text that should be highlighted.
119 /// The `Class` argument specifies how it should be highlighted.
120 fn enter_span(&mut self, _: Class) -> io::Result<()>;
122 /// Called at the end of a span of highlighted text.
123 fn exit_span(&mut self) -> io::Result<()>;
125 /// Called for a span of text. If the text should be highlighted differently from the
126 /// surrounding text, then the `Class` argument will be a value other than `None`.
128 /// The following sequences of callbacks are equivalent:
130 /// enter_span(Foo), string("text", None), exit_span()
131 /// string("text", Foo)
133 /// The latter can be thought of as a shorthand for the former, which is
135 fn string<T: Display>(&mut self, text: T, klass: Class) -> io::Result<()>;
138 // Implement `Writer` for anthing that can be written to, this just implements
139 // the default rustdoc behaviour.
140 impl<U: Write> Writer for U {
141 fn string<T: Display>(&mut self, text: T, klass: Class) -> io::Result<()> {
143 Class::None => write!(self, "{}", text),
144 klass => write!(self, "<span class=\"{}\">{}</span>", klass.rustdoc_class(), text),
148 fn enter_span(&mut self, klass: Class) -> io::Result<()> {
149 write!(self, "<span class=\"{}\">", klass.rustdoc_class())
152 fn exit_span(&mut self) -> io::Result<()> {
153 write!(self, "</span>")
157 enum HighlightError {
162 impl From<io::Error> for HighlightError {
163 fn from(err: io::Error) -> Self {
164 HighlightError::IoError(err)
168 impl<'a> Classifier<'a> {
169 fn new(lexer: lexer::StringReader<'a>, source_map: &'a SourceMap) -> Classifier<'a> {
176 in_macro_nonterminal: false,
180 /// Gets the next token out of the lexer.
181 fn try_next_token(&mut self) -> Result<Token, HighlightError> {
182 if let Some(token) = self.peek_token.take() {
185 let token = self.lexer.next_token();
186 if let token::Unknown(..) = &token.kind {
187 return Err(HighlightError::LexError);
192 fn peek(&mut self) -> Result<&Token, HighlightError> {
193 if self.peek_token.is_none() {
194 let token = self.lexer.next_token();
195 if let token::Unknown(..) = &token.kind {
196 return Err(HighlightError::LexError);
198 self.peek_token = Some(token);
200 Ok(self.peek_token.as_ref().unwrap())
203 /// Exhausts the `lexer` writing the output into `out`.
205 /// The general structure for this method is to iterate over each token,
206 /// possibly giving it an HTML span with a class specifying what flavor of token
207 /// is used. All source code emission is done as slices from the source map,
208 /// not from the tokens themselves, in order to stay true to the original
210 fn write_source<W: Writer>(&mut self, out: &mut W) -> Result<(), HighlightError> {
212 let next = self.try_next_token()?;
213 if next == token::Eof {
217 self.write_token(out, next)?;
223 // Handles an individual token from the lexer.
224 fn write_token<W: Writer>(&mut self, out: &mut W, token: Token) -> Result<(), HighlightError> {
225 let klass = match token.kind {
226 token::Shebang(s) => {
227 out.string(Escape(&s.as_str()), Class::None)?;
231 token::Whitespace | token::Unknown(..) => Class::None,
232 token::Comment => Class::Comment,
233 token::DocComment(..) => Class::DocComment,
235 // If this '&' or '*' token is followed by a non-whitespace token, assume that it's the
236 // reference or dereference operator or a reference or pointer type, instead of the
237 // bit-and or multiplication operator.
238 token::BinOp(token::And) | token::BinOp(token::Star)
239 if self.peek()? != &token::Whitespace =>
244 // Consider this as part of a macro invocation if there was a
245 // leading identifier.
246 token::Not if self.in_macro => {
247 self.in_macro = false;
265 | token::FatArrow => Class::Op,
267 // Miscellaneous, no highlighting.
277 | token::OpenDelim(_)
278 | token::CloseDelim(token::Brace)
279 | token::CloseDelim(token::Paren)
280 | token::CloseDelim(token::NoDelim) => Class::None,
282 token::Question => Class::QuestionMark,
285 if self.peek()?.is_ident() {
286 self.in_macro_nonterminal = true;
287 Class::MacroNonTerminal
293 // This might be the start of an attribute. We're going to want to
294 // continue highlighting it as an attribute until the ending ']' is
295 // seen, so skip out early. Down below we terminate the attribute
296 // span when we see the ']'.
298 // We can't be sure that our # begins an attribute (it could
299 // just be appearing in a macro) until we read either `#![` or
300 // `#[` from the input stream.
302 // We don't want to start highlighting as an attribute until
303 // we're confident there is going to be a ] coming up, as
304 // otherwise # tokens in macros highlight the rest of the input
307 // Case 1: #![inner_attribute]
308 if self.peek()? == &token::Not {
309 self.try_next_token()?; // NOTE: consumes `!` token!
310 if self.peek()? == &token::OpenDelim(token::Bracket) {
311 self.in_attribute = true;
312 out.enter_span(Class::Attribute)?;
314 out.string("#", Class::None)?;
315 out.string("!", Class::None)?;
319 // Case 2: #[outer_attribute]
320 if self.peek()? == &token::OpenDelim(token::Bracket) {
321 self.in_attribute = true;
322 out.enter_span(Class::Attribute)?;
324 out.string("#", Class::None)?;
327 token::CloseDelim(token::Bracket) => {
328 if self.in_attribute {
329 self.in_attribute = false;
330 out.string("]", Class::None)?;
338 token::Literal(lit) => {
345 | token::ByteStrRaw(..)
347 | token::StrRaw(..) => Class::String,
350 token::Integer | token::Float => Class::Number,
352 token::Bool => panic!("literal token contains `Lit::Bool`"),
356 // Keywords are also included in the identifier set.
357 token::Ident(name, is_raw) => match name {
358 kw::Ref | kw::Mut if !is_raw => Class::RefKeyWord,
360 kw::SelfLower | kw::SelfUpper => Class::Self_,
361 kw::False | kw::True if !is_raw => Class::Bool,
363 sym::Option | sym::Result => Class::PreludeTy,
364 sym::Some | sym::None | sym::Ok | sym::Err => Class::PreludeVal,
366 _ if token.is_reserved_ident() => Class::KeyWord,
369 if self.in_macro_nonterminal {
370 self.in_macro_nonterminal = false;
371 Class::MacroNonTerminal
372 } else if self.peek()? == &token::Not {
373 self.in_macro = true;
381 token::Lifetime(..) => Class::Lifetime,
384 | token::Interpolated(..)
387 | token::SingleQuote => Class::None,
390 // Anything that didn't return above is the simple case where we the
391 // class just spans a single token, so we can use the `string` method.
392 out.string(Escape(&self.snip(token.span)), klass)?;
397 // Helper function to get a snippet from the source_map.
398 fn snip(&self, sp: Span) -> String {
399 self.source_map.span_to_snippet(sp).unwrap()
404 /// Returns the css class expected by rustdoc for each `Class`.
405 fn rustdoc_class(self) -> &'static str {
408 Class::Comment => "comment",
409 Class::DocComment => "doccomment",
410 Class::Attribute => "attribute",
411 Class::KeyWord => "kw",
412 Class::RefKeyWord => "kw-2",
413 Class::Self_ => "self",
415 Class::Macro => "macro",
416 Class::MacroNonTerminal => "macro-nonterminal",
417 Class::String => "string",
418 Class::Number => "number",
419 Class::Bool => "bool-val",
420 Class::Ident => "ident",
421 Class::Lifetime => "lifetime",
422 Class::PreludeTy => "prelude-ty",
423 Class::PreludeVal => "prelude-val",
424 Class::QuestionMark => "question-mark",
429 fn write_header(class: Option<&str>, out: &mut dyn Write) -> io::Result<()> {
430 write!(out, "<div class=\"example-wrap\"><pre class=\"rust {}\">\n", class.unwrap_or(""))
433 fn write_footer(out: &mut dyn Write, playground_button: Option<&str>) -> io::Result<()> {
434 write!(out, "</pre>{}</div>\n", if let Some(button) = playground_button { button } else { "" })