1 //! Basic syntax highlighting functionality.
3 //! This module uses librustc_ast's lexer to provide token-based highlighting for
4 //! the HTML documentation generated by rustdoc.
6 //! Use the `render_with_highlighting` to highlight some rust code.
8 use crate::clean::PrimitiveType;
9 use crate::html::escape::Escape;
10 use crate::html::render::{Context, LinkFromSrc};
12 use std::collections::VecDeque;
13 use std::fmt::{Display, Write};
15 use rustc_data_structures::fx::FxHashMap;
16 use rustc_lexer::{Cursor, LiteralKind, TokenKind};
17 use rustc_span::edition::Edition;
18 use rustc_span::symbol::Symbol;
19 use rustc_span::{BytePos, Span, DUMMY_SP};
21 use super::format::{self, Buffer};
23 /// This type is needed in case we want to render links on items to allow to go to their definition.
24 pub(crate) struct HrefContext<'a, 'tcx> {
25 pub(crate) context: &'a Context<'tcx>,
26 /// This span contains the current file we're going through.
27 pub(crate) file_span: Span,
28 /// This field is used to know "how far" from the top of the directory we are to link to either
29 /// documentation pages or other source pages.
30 pub(crate) root_path: &'a str,
31 /// This field is used to calculate precise local URLs.
32 pub(crate) current_href: String,
35 /// Decorations are represented as a map from CSS class to vector of character ranges.
36 /// Each range will be wrapped in a span with that class.
38 pub(crate) struct DecorationInfo(pub(crate) FxHashMap<&'static str, Vec<(u32, u32)>>);
40 #[derive(Eq, PartialEq, Clone, Copy)]
41 pub(crate) enum Tooltip {
49 /// Highlights `src` as an inline example, returning the HTML output.
50 pub(crate) fn render_example_with_highlighting(
54 playground_button: Option<&str>,
56 write_header(out, "rust-example-rendered", None, tooltip);
57 write_code(out, src, None, None);
58 write_footer(out, playground_button);
61 /// Highlights `src` as a macro, returning the HTML output.
62 pub(crate) fn render_macro_with_highlighting(src: &str, out: &mut Buffer) {
63 write_header(out, "macro", None, Tooltip::None);
64 write_code(out, src, None, None);
65 write_footer(out, None);
68 /// Highlights `src` as a source code page, returning the HTML output.
69 pub(crate) fn render_source_with_highlighting(
73 href_context: HrefContext<'_, '_>,
74 decoration_info: DecorationInfo,
77 write_header(out, "", Some(line_numbers), Tooltip::None);
78 if let Some(extra) = extra {
81 write_code(out, src, Some(href_context), Some(decoration_info));
82 write_footer(out, None);
85 fn write_header(out: &mut Buffer, class: &str, extra_content: Option<Buffer>, tooltip: Tooltip) {
88 "<div class=\"example-wrap{}\">",
90 Tooltip::Ignore => " ignore",
91 Tooltip::CompileFail => " compile_fail",
92 Tooltip::ShouldPanic => " should_panic",
93 Tooltip::Edition(_) => " edition",
98 if tooltip != Tooltip::None {
101 "<div class='tooltip'{}>ⓘ</div>",
102 if let Tooltip::Edition(edition_info) = tooltip {
103 format!(" data-edition=\"{}\"", edition_info)
110 if let Some(extra) = extra_content {
111 out.push_buffer(extra);
113 if class.is_empty() {
114 write!(out, "<pre class=\"rust\">");
116 write!(out, "<pre class=\"rust {class}\">");
118 write!(out, "<code>");
121 /// Check if two `Class` can be merged together. In the following rules, "unclassified" means `None`
122 /// basically (since it's `Option<Class>`). The following rules apply:
124 /// * If two `Class` have the same variant, then they can be merged.
125 /// * If the other `Class` is unclassified and only contains white characters (backline,
126 /// whitespace, etc), it can be merged.
127 /// * `Class::Ident` is considered the same as unclassified (because it doesn't have an associated
129 fn can_merge(class1: Option<Class>, class2: Option<Class>, text: &str) -> bool {
130 match (class1, class2) {
131 (Some(c1), Some(c2)) => c1.is_equal_to(c2),
132 (Some(Class::Ident(_)), None) | (None, Some(Class::Ident(_))) => true,
133 (Some(_), None) | (None, Some(_)) => text.trim().is_empty(),
134 (None, None) => true,
138 /// This type is used as a conveniency to prevent having to pass all its fields as arguments into
139 /// the various functions (which became its methods).
140 struct TokenHandler<'a, 'tcx> {
142 /// It contains the closing tag and the associated `Class`.
143 closing_tags: Vec<(&'static str, Class)>,
144 /// This is used because we don't automatically generate the closing tag on `ExitSpan` in
145 /// case an `EnterSpan` event with the same class follows.
146 pending_exit_span: Option<Class>,
147 /// `current_class` and `pending_elems` are used to group HTML elements with same `class`
148 /// attributes to reduce the DOM size.
149 current_class: Option<Class>,
150 /// We need to keep the `Class` for each element because it could contain a `Span` which is
151 /// used to generate links.
152 pending_elems: Vec<(&'a str, Option<Class>)>,
153 href_context: Option<HrefContext<'a, 'tcx>>,
156 impl<'a, 'tcx> TokenHandler<'a, 'tcx> {
157 fn handle_exit_span(&mut self) {
158 // We can't get the last `closing_tags` element using `pop()` because `closing_tags` is
159 // being used in `write_pending_elems`.
160 let class = self.closing_tags.last().expect("ExitSpan without EnterSpan").1;
161 // We flush everything just in case...
162 self.write_pending_elems(Some(class));
164 exit_span(self.out, self.closing_tags.pop().expect("ExitSpan without EnterSpan").0);
165 self.pending_exit_span = None;
168 /// Write all the pending elements sharing a same (or at mergeable) `Class`.
170 /// If there is a "parent" (if a `EnterSpan` event was encountered) and the parent can be merged
171 /// with the elements' class, then we simply write the elements since the `ExitSpan` event will
174 /// Otherwise, if there is only one pending element, we let the `string` function handle both
175 /// opening and closing the tag, otherwise we do it into this function.
177 /// It returns `true` if `current_class` must be set to `None` afterwards.
178 fn write_pending_elems(&mut self, current_class: Option<Class>) -> bool {
179 if self.pending_elems.is_empty() {
182 if let Some((_, parent_class)) = self.closing_tags.last() &&
183 can_merge(current_class, Some(*parent_class), "")
185 for (text, class) in self.pending_elems.iter() {
186 string(self.out, Escape(text), *class, &self.href_context, false);
189 // We only want to "open" the tag ourselves if we have more than one pending and if the
190 // current parent tag is not the same as our pending content.
191 let close_tag = if self.pending_elems.len() > 1 && current_class.is_some() {
192 Some(enter_span(self.out, current_class.unwrap(), &self.href_context))
196 for (text, class) in self.pending_elems.iter() {
197 string(self.out, Escape(text), *class, &self.href_context, close_tag.is_none());
199 if let Some(close_tag) = close_tag {
200 exit_span(self.out, close_tag);
203 self.pending_elems.clear();
208 impl<'a, 'tcx> Drop for TokenHandler<'a, 'tcx> {
209 /// When leaving, we need to flush all pending data to not have missing content.
211 if self.pending_exit_span.is_some() {
212 self.handle_exit_span();
214 self.write_pending_elems(self.current_class);
219 /// Convert the given `src` source code into HTML by adding classes for highlighting.
221 /// This code is used to render code blocks (in the documentation) as well as the source code pages.
223 /// Some explanations on the last arguments:
225 /// In case we are rendering a code block and not a source code file, `href_context` will be `None`.
226 /// To put it more simply: if `href_context` is `None`, the code won't try to generate links to an
229 /// More explanations about spans and how we use them here are provided in the
233 href_context: Option<HrefContext<'_, '_>>,
234 decoration_info: Option<DecorationInfo>,
236 // This replace allows to fix how the code source with DOS backline characters is displayed.
237 let src = src.replace("\r\n", "\n");
238 let mut token_handler = TokenHandler {
240 closing_tags: Vec::new(),
241 pending_exit_span: None,
243 pending_elems: Vec::new(),
249 token_handler.href_context.as_ref().map(|c| c.file_span).unwrap_or(DUMMY_SP),
252 .highlight(&mut |highlight| {
254 Highlight::Token { text, class } => {
255 // If we received a `ExitSpan` event and then have a non-compatible `Class`, we
256 // need to close the `<span>`.
257 let need_current_class_update = if let Some(pending) = token_handler.pending_exit_span &&
258 !can_merge(Some(pending), class, text) {
259 token_handler.handle_exit_span();
261 // If the two `Class` are different, time to flush the current content and start
263 } else if !can_merge(token_handler.current_class, class, text) {
264 token_handler.write_pending_elems(token_handler.current_class);
267 token_handler.current_class.is_none()
270 if need_current_class_update {
271 token_handler.current_class = class.map(Class::dummy);
273 token_handler.pending_elems.push((text, class));
275 Highlight::EnterSpan { class } => {
276 let mut should_add = true;
277 if let Some(pending_exit_span) = token_handler.pending_exit_span {
278 if class.is_equal_to(pending_exit_span) {
281 token_handler.handle_exit_span();
284 // We flush everything just in case...
285 if token_handler.write_pending_elems(token_handler.current_class) {
286 token_handler.current_class = None;
290 let closing_tag = enter_span(token_handler.out, class, &token_handler.href_context);
291 token_handler.closing_tags.push((closing_tag, class));
294 token_handler.current_class = None;
295 token_handler.pending_exit_span = None;
297 Highlight::ExitSpan => {
298 token_handler.current_class = None;
299 token_handler.pending_exit_span =
300 Some(token_handler.closing_tags.last().as_ref().expect("ExitSpan without EnterSpan").1);
306 fn write_footer(out: &mut Buffer, playground_button: Option<&str>) {
307 writeln!(out, "</code></pre>{}</div>", playground_button.unwrap_or_default());
310 /// How a span of text is classified. Mostly corresponds to token kinds.
311 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
317 /// Keywords that do pointer/reference stuff.
325 /// `Ident` isn't rendered in the HTML but we still need it for the `Span` it contains.
331 Decoration(&'static str),
335 /// It is only looking at the variant, not the variant content.
337 /// It is used mostly to group multiple similar HTML elements into one `<span>` instead of
339 fn is_equal_to(self, other: Self) -> bool {
340 match (self, other) {
341 (Self::Self_(_), Self::Self_(_))
342 | (Self::Macro(_), Self::Macro(_))
343 | (Self::Ident(_), Self::Ident(_)) => true,
344 (Self::Decoration(c1), Self::Decoration(c2)) => c1 == c2,
349 /// If `self` contains a `Span`, it'll be replaced with `DUMMY_SP` to prevent creating links
350 /// on "empty content" (because of the attributes merge).
351 fn dummy(self) -> Self {
353 Self::Self_(_) => Self::Self_(DUMMY_SP),
354 Self::Macro(_) => Self::Macro(DUMMY_SP),
355 Self::Ident(_) => Self::Ident(DUMMY_SP),
360 /// Returns the css class expected by rustdoc for each `Class`.
361 fn as_html(self) -> &'static str {
363 Class::Comment => "comment",
364 Class::DocComment => "doccomment",
365 Class::Attribute => "attr",
366 Class::KeyWord => "kw",
367 Class::RefKeyWord => "kw-2",
368 Class::Self_(_) => "self",
369 Class::Macro(_) => "macro",
370 Class::MacroNonTerminal => "macro-nonterminal",
371 Class::String => "string",
372 Class::Number => "number",
373 Class::Bool => "bool-val",
374 Class::Ident(_) => "",
375 Class::Lifetime => "lifetime",
376 Class::PreludeTy => "prelude-ty",
377 Class::PreludeVal => "prelude-val",
378 Class::QuestionMark => "question-mark",
379 Class::Decoration(kind) => kind,
383 /// In case this is an item which can be converted into a link to a definition, it'll contain
384 /// a "span" (a tuple representing `(lo, hi)` equivalent of `Span`).
385 fn get_span(self) -> Option<Span> {
387 Self::Ident(sp) | Self::Self_(sp) | Self::Macro(sp) => Some(sp),
393 | Self::MacroNonTerminal
401 | Self::Decoration(_) => None,
407 Token { text: &'a str, class: Option<Class> },
408 EnterSpan { class: Class },
412 struct TokenIter<'a> {
417 impl<'a> Iterator for TokenIter<'a> {
418 type Item = (TokenKind, &'a str);
419 fn next(&mut self) -> Option<(TokenKind, &'a str)> {
420 let token = self.cursor.advance_token();
421 if token.kind == TokenKind::Eof {
424 let (text, rest) = self.src.split_at(token.len as usize);
426 Some((token.kind, text))
430 /// Classifies into identifier class; returns `None` if this is a non-keyword identifier.
431 fn get_real_ident_class(text: &str, allow_path_keywords: bool) -> Option<Class> {
432 let ignore: &[&str] =
433 if allow_path_keywords { &["self", "Self", "super", "crate"] } else { &["self", "Self"] };
434 if ignore.iter().any(|k| *k == text) {
438 "ref" | "mut" => Class::RefKeyWord,
439 "false" | "true" => Class::Bool,
440 _ if Symbol::intern(text).is_reserved(|| Edition::Edition2021) => Class::KeyWord,
445 /// This iterator comes from the same idea than "Peekable" except that it allows to "peek" more than
446 /// just the next item by using `peek_next`. The `peek` method always returns the next item after
447 /// the current one whereas `peek_next` will return the next item after the last one peeked.
449 /// You can use both `peek` and `peek_next` at the same time without problem.
450 struct PeekIter<'a> {
451 stored: VecDeque<(TokenKind, &'a str)>,
452 /// This position is reinitialized when using `next`. It is used in `peek_next`.
457 impl<'a> PeekIter<'a> {
458 fn new(iter: TokenIter<'a>) -> Self {
459 Self { stored: VecDeque::new(), peek_pos: 0, iter }
461 /// Returns the next item after the current one. It doesn't interfere with `peek_next` output.
462 fn peek(&mut self) -> Option<&(TokenKind, &'a str)> {
463 if self.stored.is_empty() {
464 if let Some(next) = self.iter.next() {
465 self.stored.push_back(next);
470 /// Returns the next item after the last one peeked. It doesn't interfere with `peek` output.
471 fn peek_next(&mut self) -> Option<&(TokenKind, &'a str)> {
473 if self.peek_pos - 1 < self.stored.len() {
474 self.stored.get(self.peek_pos - 1)
475 } else if let Some(next) = self.iter.next() {
476 self.stored.push_back(next);
484 impl<'a> Iterator for PeekIter<'a> {
485 type Item = (TokenKind, &'a str);
486 fn next(&mut self) -> Option<Self::Item> {
488 if let Some(first) = self.stored.pop_front() { Some(first) } else { self.iter.next() }
492 /// Custom spans inserted into the source. Eg --scrape-examples uses this to highlight function calls
494 starts: Vec<(u32, &'static str)>,
499 fn new(info: DecorationInfo) -> Self {
500 // Extract tuples (start, end, kind) into separate sequences of (start, kind) and (end).
501 let (mut starts, mut ends): (Vec<_>, Vec<_>) = info
504 .flat_map(|(kind, ranges)| ranges.into_iter().map(move |(lo, hi)| ((lo, kind), hi)))
507 // Sort the sequences in document order.
508 starts.sort_by_key(|(lo, _)| *lo);
511 Decorations { starts, ends }
515 /// Processes program tokens, classifying strings of text by highlighting
516 /// category (`Class`).
517 struct Classifier<'src> {
518 tokens: PeekIter<'src>,
521 in_macro_nonterminal: bool,
525 decorations: Option<Decorations>,
528 impl<'src> Classifier<'src> {
529 /// Takes as argument the source code to HTML-ify, the rust edition to use and the source code
530 /// file span which will be used later on by the `span_correspondance_map`.
531 fn new(src: &str, file_span: Span, decoration_info: Option<DecorationInfo>) -> Classifier<'_> {
532 let tokens = PeekIter::new(TokenIter { src, cursor: Cursor::new(src) });
533 let decorations = decoration_info.map(Decorations::new);
538 in_macro_nonterminal: false,
546 /// Convenient wrapper to create a [`Span`] from a position in the file.
547 fn new_span(&self, lo: u32, text: &str) -> Span {
548 let hi = lo + text.len() as u32;
549 let file_lo = self.file_span.lo();
550 self.file_span.with_lo(file_lo + BytePos(lo)).with_hi(file_lo + BytePos(hi))
553 /// Concatenate colons and idents as one when possible.
554 fn get_full_ident_path(&mut self) -> Vec<(TokenKind, usize, usize)> {
555 let start = self.byte_pos as usize;
557 let mut has_ident = false;
561 while let Some((TokenKind::Colon, _)) = self.tokens.peek() {
565 // Ident path can start with "::" but if we already have content in the ident path,
566 // the "::" is mandatory.
567 if has_ident && nb == 0 {
568 return vec![(TokenKind::Ident, start, pos)];
569 } else if nb != 0 && nb != 2 {
571 return vec![(TokenKind::Ident, start, pos), (TokenKind::Colon, pos, pos + nb)];
573 return vec![(TokenKind::Colon, start, pos + nb)];
577 if let Some((None, text)) = self.tokens.peek().map(|(token, text)| {
578 if *token == TokenKind::Ident {
579 let class = get_real_ident_class(text, true);
582 // Doesn't matter which Class we put in here...
583 (Some(Class::Comment), text)
586 // We only "add" the colon if there is an ident behind.
587 pos += text.len() + nb;
590 } else if nb > 0 && has_ident {
591 return vec![(TokenKind::Ident, start, pos), (TokenKind::Colon, pos, pos + nb)];
593 return vec![(TokenKind::Colon, start, start + nb)];
594 } else if has_ident {
595 return vec![(TokenKind::Ident, start, pos)];
602 /// Wraps the tokens iteration to ensure that the `byte_pos` is always correct.
604 /// It returns the token's kind, the token as a string and its byte position in the source
606 fn next(&mut self) -> Option<(TokenKind, &'src str, u32)> {
607 if let Some((kind, text)) = self.tokens.next() {
608 let before = self.byte_pos;
609 self.byte_pos += text.len() as u32;
610 Some((kind, text, before))
616 /// Exhausts the `Classifier` writing the output into `sink`.
618 /// The general structure for this method is to iterate over each token,
619 /// possibly giving it an HTML span with a class specifying what flavor of
621 fn highlight(mut self, sink: &mut dyn FnMut(Highlight<'src>)) {
623 if let Some(decs) = self.decorations.as_mut() {
624 let byte_pos = self.byte_pos;
625 let n_starts = decs.starts.iter().filter(|(i, _)| byte_pos >= *i).count();
626 for (_, kind) in decs.starts.drain(0..n_starts) {
627 sink(Highlight::EnterSpan { class: Class::Decoration(kind) });
630 let n_ends = decs.ends.iter().filter(|i| byte_pos >= **i).count();
631 for _ in decs.ends.drain(0..n_ends) {
632 sink(Highlight::ExitSpan);
639 .map(|t| matches!(t.0, TokenKind::Colon | TokenKind::Ident))
642 let tokens = self.get_full_ident_path();
643 for (token, start, end) in &tokens {
644 let text = &self.src[*start..*end];
645 self.advance(*token, text, sink, *start as u32);
646 self.byte_pos += text.len() as u32;
648 if !tokens.is_empty() {
652 if let Some((token, text, before)) = self.next() {
653 self.advance(token, text, sink, before);
660 /// Single step of highlighting. This will classify `token`, but maybe also a couple of
661 /// following ones as well.
663 /// `before` is the position of the given token in the `source` string and is used as "lo" byte
664 /// in case we want to try to generate a link for this token using the
665 /// `span_correspondance_map`.
670 sink: &mut dyn FnMut(Highlight<'src>),
673 let lookahead = self.peek();
674 let no_highlight = |sink: &mut dyn FnMut(_)| sink(Highlight::Token { text, class: None });
675 let class = match token {
676 TokenKind::Whitespace => return no_highlight(sink),
677 TokenKind::LineComment { doc_style } | TokenKind::BlockComment { doc_style, .. } => {
678 if doc_style.is_some() {
684 // Consider this as part of a macro invocation if there was a
685 // leading identifier.
686 TokenKind::Bang if self.in_macro => {
687 self.in_macro = false;
688 sink(Highlight::Token { text, class: None });
689 sink(Highlight::ExitSpan);
693 // Assume that '&' or '*' is the reference or dereference operator
694 // or a reference or pointer type. Unless, of course, it looks like
695 // a logical and or a multiplication operator: `&&` or `* `.
696 TokenKind::Star => match self.tokens.peek() {
697 Some((TokenKind::Whitespace, _)) => return no_highlight(sink),
698 Some((TokenKind::Ident, "mut")) => {
700 sink(Highlight::Token { text: "*mut", class: Some(Class::RefKeyWord) });
703 Some((TokenKind::Ident, "const")) => {
705 sink(Highlight::Token { text: "*const", class: Some(Class::RefKeyWord) });
708 _ => Class::RefKeyWord,
710 TokenKind::And => match self.tokens.peek() {
711 Some((TokenKind::And, _)) => {
713 sink(Highlight::Token { text: "&&", class: None });
716 Some((TokenKind::Eq, _)) => {
718 sink(Highlight::Token { text: "&=", class: None });
721 Some((TokenKind::Whitespace, _)) => return no_highlight(sink),
722 Some((TokenKind::Ident, "mut")) => {
724 sink(Highlight::Token { text: "&mut", class: Some(Class::RefKeyWord) });
727 _ => Class::RefKeyWord,
730 // These can either be operators, or arrows.
731 TokenKind::Eq => match lookahead {
732 Some(TokenKind::Eq) => {
734 sink(Highlight::Token { text: "==", class: None });
737 Some(TokenKind::Gt) => {
739 sink(Highlight::Token { text: "=>", class: None });
742 _ => return no_highlight(sink),
744 TokenKind::Minus if lookahead == Some(TokenKind::Gt) => {
746 sink(Highlight::Token { text: "->", class: None });
759 | TokenKind::Gt => return no_highlight(sink),
761 // Miscellaneous, no highlighting.
765 | TokenKind::OpenParen
766 | TokenKind::CloseParen
767 | TokenKind::OpenBrace
768 | TokenKind::CloseBrace
769 | TokenKind::OpenBracket
773 | TokenKind::Unknown => return no_highlight(sink),
775 TokenKind::Question => Class::QuestionMark,
777 TokenKind::Dollar => match lookahead {
778 Some(TokenKind::Ident) => {
779 self.in_macro_nonterminal = true;
780 Class::MacroNonTerminal
782 _ => return no_highlight(sink),
785 // This might be the start of an attribute. We're going to want to
786 // continue highlighting it as an attribute until the ending ']' is
787 // seen, so skip out early. Down below we terminate the attribute
788 // span when we see the ']'.
789 TokenKind::Pound => {
791 // Case 1: #![inner_attribute]
792 Some(TokenKind::Bang) => {
794 if let Some(TokenKind::OpenBracket) = self.peek() {
795 self.in_attribute = true;
796 sink(Highlight::EnterSpan { class: Class::Attribute });
798 sink(Highlight::Token { text: "#", class: None });
799 sink(Highlight::Token { text: "!", class: None });
802 // Case 2: #[outer_attribute]
803 Some(TokenKind::OpenBracket) => {
804 self.in_attribute = true;
805 sink(Highlight::EnterSpan { class: Class::Attribute });
809 return no_highlight(sink);
811 TokenKind::CloseBracket => {
812 if self.in_attribute {
813 self.in_attribute = false;
814 sink(Highlight::Token { text: "]", class: None });
815 sink(Highlight::ExitSpan);
818 return no_highlight(sink);
820 TokenKind::Literal { kind, .. } => match kind {
822 LiteralKind::Byte { .. }
823 | LiteralKind::Char { .. }
824 | LiteralKind::Str { .. }
825 | LiteralKind::ByteStr { .. }
826 | LiteralKind::RawStr { .. }
827 | LiteralKind::RawByteStr { .. } => Class::String,
829 LiteralKind::Float { .. } | LiteralKind::Int { .. } => Class::Number,
831 TokenKind::Ident | TokenKind::RawIdent if lookahead == Some(TokenKind::Bang) => {
832 self.in_macro = true;
833 sink(Highlight::EnterSpan { class: Class::Macro(self.new_span(before, text)) });
834 sink(Highlight::Token { text, class: None });
837 TokenKind::Ident => match get_real_ident_class(text, false) {
839 "Option" | "Result" => Class::PreludeTy,
840 "Some" | "None" | "Ok" | "Err" => Class::PreludeVal,
841 // "union" is a weak keyword and is only considered as a keyword when declaring
843 "union" if self.check_if_is_union_keyword() => Class::KeyWord,
844 _ if self.in_macro_nonterminal => {
845 self.in_macro_nonterminal = false;
846 Class::MacroNonTerminal
848 "self" | "Self" => Class::Self_(self.new_span(before, text)),
849 _ => Class::Ident(self.new_span(before, text)),
853 TokenKind::RawIdent | TokenKind::UnknownPrefix | TokenKind::InvalidIdent => {
854 Class::Ident(self.new_span(before, text))
856 TokenKind::Lifetime { .. } => Class::Lifetime,
857 TokenKind::Eof => panic!("Eof in advance"),
859 // Anything that didn't return above is the simple case where we the
860 // class just spans a single token, so we can use the `string` method.
861 sink(Highlight::Token { text, class: Some(class) });
864 fn peek(&mut self) -> Option<TokenKind> {
865 self.tokens.peek().map(|(token_kind, _text)| *token_kind)
868 fn check_if_is_union_keyword(&mut self) -> bool {
869 while let Some(kind) = self.tokens.peek_next().map(|(token_kind, _text)| token_kind) {
870 if *kind == TokenKind::Whitespace {
873 return *kind == TokenKind::Ident;
879 /// Called when we start processing a span of text that should be highlighted.
880 /// The `Class` argument specifies how it should be highlighted.
884 href_context: &Option<HrefContext<'_, '_>>,
886 string_without_closing_tag(out, "", Some(klass), href_context, true).expect(
887 "internal error: enter_span was called with Some(klass) but did not return a \
892 /// Called at the end of a span of highlighted text.
893 fn exit_span(out: &mut Buffer, closing_tag: &str) {
894 out.write_str(closing_tag);
897 /// Called for a span of text. If the text should be highlighted differently
898 /// from the surrounding text, then the `Class` argument will be a value other
901 /// The following sequences of callbacks are equivalent:
903 /// enter_span(Foo), string("text", None), exit_span()
904 /// string("text", Foo)
907 /// The latter can be thought of as a shorthand for the former, which is more
910 /// Note that if `context` is not `None` and that the given `klass` contains a `Span`, the function
911 /// will then try to find this `span` in the `span_correspondance_map`. If found, it'll then
912 /// generate a link for this element (which corresponds to where its definition is located).
913 fn string<T: Display>(
916 klass: Option<Class>,
917 href_context: &Option<HrefContext<'_, '_>>,
920 if let Some(closing_tag) = string_without_closing_tag(out, text, klass, href_context, open_tag)
922 out.write_str(closing_tag);
926 /// This function writes `text` into `out` with some modifications depending on `klass`:
928 /// * If `klass` is `None`, `text` is written into `out` with no modification.
929 /// * If `klass` is `Some` but `klass.get_span()` is `None`, it writes the text wrapped in a
930 /// `<span>` with the provided `klass`.
931 /// * If `klass` is `Some` and has a [`rustc_span::Span`], it then tries to generate a link (`<a>`
932 /// element) by retrieving the link information from the `span_correspondance_map` that was filled
933 /// in `span_map.rs::collect_spans_and_sources`. If it cannot retrieve the information, then it's
934 /// the same as the second point (`klass` is `Some` but doesn't have a [`rustc_span::Span`]).
935 fn string_without_closing_tag<T: Display>(
938 klass: Option<Class>,
939 href_context: &Option<HrefContext<'_, '_>>,
941 ) -> Option<&'static str> {
942 let Some(klass) = klass
944 write!(out, "{}", text);
947 let Some(def_span) = klass.get_span()
950 write!(out, "{}", text);
953 write!(out, "<span class=\"{}\">{}", klass.as_html(), text);
954 return Some("</span>");
957 let mut text_s = text.to_string();
958 if text_s.contains("::") {
959 text_s = text_s.split("::").intersperse("::").fold(String::new(), |mut path, t| {
961 "self" | "Self" => write!(
963 "<span class=\"{}\">{}</span>",
964 Class::Self_(DUMMY_SP).as_html(),
967 "crate" | "super" => {
968 write!(&mut path, "<span class=\"{}\">{}</span>", Class::KeyWord.as_html(), t)
970 t => write!(&mut path, "{}", t),
972 .expect("Failed to build source HTML path");
977 if let Some(href_context) = href_context {
979 href_context.context.shared.span_correspondance_map.get(&def_span).and_then(|href| {
980 let context = href_context.context;
981 // FIXME: later on, it'd be nice to provide two links (if possible) for all items:
982 // one to the documentation page and one to the source definition.
983 // FIXME: currently, external items only generate a link to their documentation,
984 // a link to their definition can be generated using this:
985 // https://github.com/rust-lang/rust/blob/60f1a2fc4b535ead9c85ce085fdce49b1b097531/src/librustdoc/html/render/context.rs#L315-L338
987 LinkFromSrc::Local(span) => {
988 context.href_from_span_relative(*span, &href_context.current_href)
990 LinkFromSrc::External(def_id) => {
991 format::href_with_root_path(*def_id, context, Some(href_context.root_path))
993 .map(|(url, _, _)| url)
995 LinkFromSrc::Primitive(prim) => format::href_with_root_path(
996 PrimitiveType::primitive_locations(context.tcx())[prim],
998 Some(href_context.root_path),
1001 .map(|(url, _, _)| url),
1006 // We're already inside an element which has the same klass, no need to give it
1008 write!(out, "<a href=\"{}\">{}", href, text_s);
1010 let klass_s = klass.as_html();
1011 if klass_s.is_empty() {
1012 write!(out, "<a href=\"{}\">{}", href, text_s);
1014 write!(out, "<a class=\"{}\" href=\"{}\">{}", klass_s, href, text_s);
1017 return Some("</a>");
1021 write!(out, "{}", text_s);
1024 let klass_s = klass.as_html();
1025 if klass_s.is_empty() {
1026 write!(out, "{}", text_s);
1029 write!(out, "<span class=\"{}\">{}", klass_s, text_s);