Set metadata for vtable-related loads

Give LLVM much more information about vtable pointers. Without the extra
information, LLVM has to be rather pessimistic about vtables, preventing
a number of obvious optimisations.

* Makes the vtable pointer argument noalias and readonly.
* Marks loads of the vtable pointer as nonnull.
* Marks load from the vtable with `!invariant.load` metadata.

Fixes #39992

diff --git a/src/librustc_trans/abi.rs b/src/librustc_trans/abi.rs
index a476b1d29e5fb823ef8ffb9a605c2ab474210ba5..f742cca5b980a28cbbc9481b87a9a2a92a6a5bad 100644 (file)
--- a/src/librustc_trans/abi.rs
+++ b/src/librustc_trans/abi.rs
@@ -506,7 +506,11 @@ pub fn unadjusted<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>,
                 if let Some(inner) = rust_ptr_attrs(ty, &mut data) {
                     data.attrs.set(ArgAttribute::NonNull);
                     if ccx.tcx().struct_tail(inner).is_trait() {
+                        // vtables can be safely marked non-null, readonly
+                        // and noalias.
                         info.attrs.set(ArgAttribute::NonNull);
+                        info.attrs.set(ArgAttribute::ReadOnly);
+                        info.attrs.set(ArgAttribute::NoAlias);
                     }
                 }
                 args.push(data);

diff --git a/src/librustc_trans/base.rs b/src/librustc_trans/base.rs
index 41c0eaa52a77d1bc1a2f1d6cec13bc84a850a496..0f7a510c363854b91b547f139930f1f9ff640208 100644 (file)
--- a/src/librustc_trans/base.rs
+++ b/src/librustc_trans/base.rs
@@ -472,8 +472,15 @@ pub fn load_fat_ptr<'a, 'tcx>(
         b.load(ptr, alignment.to_align())
     };
 
-    // FIXME: emit metadata on `meta`.
-    let meta = b.load(get_meta(b, src), alignment.to_align());
+    let meta = get_meta(b, src);
+    let meta_ty = val_ty(meta);
+    // If the 'meta' field is a pointer, it's a vtable, so use load_nonnull
+    // instead
+    let meta = if meta_ty.element_type().kind() == llvm::TypeKind::Pointer {
+        b.load_nonnull(meta, None)
+    } else {
+        b.load(meta, None)
+    };
 
     (ptr, meta)
 }

diff --git a/src/librustc_trans/builder.rs b/src/librustc_trans/builder.rs
index f64e581c1773ee2a81e42a68c55f8526abb06c2b..99738dd6872e8df6d12929ecdbf3ed318996c216 100644 (file)
--- a/src/librustc_trans/builder.rs
+++ b/src/librustc_trans/builder.rs
@@ -1149,6 +1149,13 @@ pub fn add_incoming_to_phi(&self, phi: ValueRef, val: ValueRef, bb: BasicBlockRe
         }
     }
 
+    pub fn set_invariant_load(&self, load: ValueRef) {
+        unsafe {
+            llvm::LLVMSetMetadata(load, llvm::MD_invariant_load as c_uint,
+                                  llvm::LLVMMDNodeInContext(self.ccx.llcx(), ptr::null(), 0));
+        }
+    }
+
     /// Returns the ptr value that should be used for storing `val`.
     fn check_store<'b>(&self,
                        val: ValueRef,

diff --git a/src/librustc_trans/glue.rs b/src/librustc_trans/glue.rs
index d66ea4d650f7b4638c226adc826d2316a086c9be..9963514acd73619b0b939b488e03d0a7fe16bf87 100644 (file)
--- a/src/librustc_trans/glue.rs
+++ b/src/librustc_trans/glue.rs
@@ -386,7 +386,15 @@ pub fn size_and_align_of_dst<'a, 'tcx>(bcx: &Builder<'a, 'tcx>, t: Ty<'tcx>, inf
             let info = bcx.pointercast(info, Type::int(bcx.ccx).ptr_to());
             let size_ptr = bcx.gepi(info, &[1]);
             let align_ptr = bcx.gepi(info, &[2]);
-            (bcx.load(size_ptr, None), bcx.load(align_ptr, None))
+
+            let size = bcx.load(size_ptr, None);
+            let align = bcx.load(align_ptr, None);
+
+            // Vtable loads are invariant
+            bcx.set_invariant_load(size);
+            bcx.set_invariant_load(align);
+
+            (size, align)
         }
         ty::TySlice(_) | ty::TyStr => {
             let unit_ty = t.sequence_element_type(bcx.tcx());

diff --git a/src/librustc_trans/meth.rs b/src/librustc_trans/meth.rs
index 3033ae61d20c80ded2e4101210084e57e825e862..a3f4168e96f2a0c5e2bdc13d824a0846e34c7a5b 100644 (file)
--- a/src/librustc_trans/meth.rs
+++ b/src/librustc_trans/meth.rs
@@ -30,13 +30,15 @@
 /// Extracts a method from a trait object's vtable, at the specified index.
 pub fn get_virtual_method<'a, 'tcx>(bcx: &Builder<'a, 'tcx>,
                                     llvtable: ValueRef,
-                                    vtable_index: usize)
-                                    -> ValueRef {
+                                    vtable_index: usize) -> ValueRef {
     // Load the data pointer from the object.
     debug!("get_virtual_method(vtable_index={}, llvtable={:?})",
            vtable_index, Value(llvtable));
 
-    bcx.load(bcx.gepi(llvtable, &[vtable_index + VTABLE_OFFSET]), None)
+    let ptr = bcx.load_nonnull(bcx.gepi(llvtable, &[vtable_index + VTABLE_OFFSET]), None);
+    // Vtable loads are invariant
+    bcx.set_invariant_load(ptr);
+    ptr
 }
 
 /// Generate a shim function that allows an object type like `SomeTrait` to