Normalize source when loading external foreign source into SourceMap

The compiler normalizes source when reading files initially (removes BOMs, etc), but not when loading external sources.

Fixes #70874 by normalizing when loading external sources too. Adds a test to verify normalization.

diff --git a/src/librustc_span/lib.rs b/src/librustc_span/lib.rs
index 0d9f3f214fb21afa031497863edd1726a9e085e7..85a870ae34c11b0bbd97a5fdb96b3c1a3ba63a39 100644 (file)
--- a/src/librustc_span/lib.rs
+++ b/src/librustc_span/lib.rs
@@ -1192,8 +1192,10 @@ pub fn add_external_src<F>(&self, get_src: F) -> bool
                 kind: src_kind @ ExternalSourceKind::AbsentOk, ..
             } = &mut *external_src
             {
-                if let Some(src) = src {
+                if let Some(mut src) = src {
+                    // The src_hash needs to be computed on the pre-normalized src.
                     if self.src_hash.matches(&src) {
+                        normalize_src(&mut src, BytePos::from_usize(0));
                         *src_kind = ExternalSourceKind::Present(Lrc::new(src));
                         return true;
                     }

diff --git a/src/librustc_span/source_map/tests.rs b/src/librustc_span/source_map/tests.rs
index 79df1884f0db0abf95095515174efc115e87f146..b8459eee4ecf0e6a84ba86cd6e0ff48b32a5c114 100644 (file)
--- a/src/librustc_span/source_map/tests.rs
+++ b/src/librustc_span/source_map/tests.rs
@@ -168,6 +168,62 @@ fn span_merging_fail() {
     assert!(sm.merge_spans(span1, span2).is_none());
 }
 
+/// Tests loading an external source file that requires normalization.
+#[test]
+fn t10() {
+    let sm = SourceMap::new(FilePathMapping::empty());
+    let unnormalized = "first line.\r\nsecond line";
+    let normalized = "first line.\nsecond line";
+
+    let src_file = sm.new_source_file(PathBuf::from("blork.rs").into(), unnormalized.to_string());
+
+    assert_eq!(src_file.src.as_ref().unwrap().as_ref(), normalized);
+    assert!(
+        src_file.src_hash.matches(unnormalized),
+        "src_hash should use the source before normalization"
+    );
+
+    let SourceFile {
+        name,
+        name_was_remapped,
+        src_hash,
+        start_pos,
+        end_pos,
+        lines,
+        multibyte_chars,
+        non_narrow_chars,
+        normalized_pos,
+        name_hash,
+        ..
+    } = (*src_file).clone();
+
+    let imported_src_file = sm.new_imported_source_file(
+        name,
+        name_was_remapped,
+        src_hash,
+        name_hash,
+        (end_pos - start_pos).to_usize(),
+        CrateNum::new(0),
+        lines,
+        multibyte_chars,
+        non_narrow_chars,
+        normalized_pos,
+        start_pos,
+        end_pos,
+    );
+
+    assert!(
+        imported_src_file.external_src.borrow().get_source().is_none(),
+        "imported source file should not have source yet"
+    );
+    imported_src_file.add_external_src(|| Some(unnormalized.to_string()));
+    assert_eq!(
+        imported_src_file.external_src.borrow().get_source().unwrap().as_ref(),
+        normalized,
+        "imported source file should be normalized"
+    );
+}
+
 /// Returns the span corresponding to the `n`th occurrence of `substring` in `source_text`.
 trait SourceMapExtension {
     fn span_substr(