parse/lexer: support `StringReader::retokenize` called on external files.

diff --git a/src/librustc_parse/lexer/mod.rs b/src/librustc_parse/lexer/mod.rs
index f7fb704fcbc2c13c2a5984c742d32a68235ca98f..6c0b2c40c76221db44b5ff2925d6e9b77069bd6b 100644 (file)
--- a/src/librustc_parse/lexer/mod.rs
+++ b/src/librustc_parse/lexer/mod.rs
@@ -46,12 +46,20 @@ pub fn new(
         source_file: Lrc<rustc_span::SourceFile>,
         override_span: Option<Span>,
     ) -> Self {
-        if source_file.src.is_none() {
+        // Make sure external source is loaded first, before accessing it.
+        // While this can't show up during normal parsing, `retokenize` may
+        // be called with a source file from an external crate.
+        sess.source_map().ensure_source_file_source_present(source_file.clone());
+
+        // FIXME(eddyb) use `Lrc<str>` or similar to avoid cloning the `String`.
+        let src = if let Some(src) = &source_file.src {
+            src.clone()
+        } else if let Some(src) = source_file.external_src.borrow().get_source() {
+            src.clone()
+        } else {
             sess.span_diagnostic
                 .bug(&format!("cannot lex `source_file` without source: {}", source_file.name));
-        }
-
-        let src = (*source_file.src.as_ref().unwrap()).clone();
+        };
 
         StringReader {
             sess,

diff --git a/src/librustc_span/lib.rs b/src/librustc_span/lib.rs
index dbc180114f1c1dafd90a9f296a057d1023457736..28864737072b291051012d2349b06587c0cd21bf 100644 (file)
--- a/src/librustc_span/lib.rs
+++ b/src/librustc_span/lib.rs
@@ -856,7 +856,7 @@ pub enum ExternalSource {
 #[derive(PartialEq, Eq, Clone, Debug)]
 pub enum ExternalSourceKind {
     /// The external source has been loaded already.
-    Present(String),
+    Present(Lrc<String>),
     /// No attempt has been made to load the external source.
     AbsentOk,
     /// A failed attempt has been made to load the external source.
@@ -872,7 +872,7 @@ pub fn is_absent(&self) -> bool {
         }
     }
 
-    pub fn get_source(&self) -> Option<&str> {
+    pub fn get_source(&self) -> Option<&Lrc<String>> {
         match self {
             ExternalSource::Foreign { kind: ExternalSourceKind::Present(ref src), .. } => Some(src),
             _ => None,
@@ -1138,7 +1138,7 @@ pub fn add_external_src<F>(&self, get_src: F) -> bool
                     hasher.write(src.as_bytes());
 
                     if hasher.finish::<u128>() == self.src_hash {
-                        *src_kind = ExternalSourceKind::Present(src);
+                        *src_kind = ExternalSourceKind::Present(Lrc::new(src));
                         return true;
                     }
                 } else {