1 use super::place::PlaceRef;
2 use super::{FunctionCx, LocalRef};
10 use rustc::mir::interpret::{ConstValue, ErrorHandled, Pointer, Scalar};
11 use rustc::ty::layout::{self, Align, LayoutOf, Size, TyLayout};
16 /// The representation of a Rust value. The enum variant is in fact
17 /// uniquely determined by the value's type, but is kept as a
19 #[derive(Copy, Clone, Debug)]
20 pub enum OperandValue<V> {
21 /// A reference to the actual operand. The data is guaranteed
22 /// to be valid for the operand's lifetime.
23 /// The second value, if any, is the extra data (vtable or length)
24 /// which indicates that it refers to an unsized rvalue.
25 Ref(V, Option<V>, Align),
26 /// A single LLVM value.
28 /// A pair of immediate LLVM values. Used by fat pointers too.
32 /// An `OperandRef` is an "SSA" reference to a Rust value, along with
35 /// NOTE: unless you know a value's type exactly, you should not
36 /// generate LLVM opcodes acting on it and instead act via methods,
37 /// to avoid nasty edge cases. In particular, using `Builder::store`
38 /// directly is sure to cause problems -- use `OperandRef::store`
40 #[derive(Copy, Clone)]
41 pub struct OperandRef<'tcx, V> {
43 pub val: OperandValue<V>,
45 // The layout of value, based on its Rust type.
46 pub layout: TyLayout<'tcx>,
49 impl<V: CodegenObject> fmt::Debug for OperandRef<'tcx, V> {
50 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
51 write!(f, "OperandRef({:?} @ {:?})", self.val, self.layout)
55 impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
56 pub fn new_zst<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
58 layout: TyLayout<'tcx>,
59 ) -> OperandRef<'tcx, V> {
60 assert!(layout.is_zst());
62 val: OperandValue::Immediate(bx.const_undef(bx.immediate_backend_type(layout))),
67 pub fn from_const<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
69 val: ConstValue<'tcx>,
72 let layout = bx.layout_of(ty);
75 return OperandRef::new_zst(bx, layout);
79 ConstValue::Scalar(x) => {
80 let scalar = match layout.abi {
81 layout::Abi::Scalar(ref x) => x,
82 _ => bug!("from_const: invalid ByVal layout: {:#?}", layout),
84 let llval = bx.scalar_to_backend(x, scalar, bx.immediate_backend_type(layout));
85 OperandValue::Immediate(llval)
87 ConstValue::Slice { data, start, end } => {
88 let a_scalar = match layout.abi {
89 layout::Abi::ScalarPair(ref a, _) => a,
90 _ => bug!("from_const: invalid ScalarPair layout: {:#?}", layout),
92 let a = Scalar::from(Pointer::new(
93 bx.tcx().alloc_map.lock().create_memory_alloc(data),
94 Size::from_bytes(start as u64),
97 let a_llval = bx.scalar_to_backend(
100 bx.scalar_pair_element_backend_type(layout, 0, true),
102 let b_llval = bx.const_usize((end - start) as u64);
103 OperandValue::Pair(a_llval, b_llval)
105 ConstValue::ByRef { alloc, offset } => {
106 return bx.load_operand(bx.from_const_alloc(layout, alloc, offset));
110 OperandRef { val, layout }
113 /// Asserts that this operand refers to a scalar and returns
114 /// a reference to its value.
115 pub fn immediate(self) -> V {
117 OperandValue::Immediate(s) => s,
118 _ => bug!("not immediate: {:?}", self),
122 pub fn deref<Cx: LayoutTypeMethods<'tcx>>(self, cx: &Cx) -> PlaceRef<'tcx, V> {
123 let projected_ty = self
127 .unwrap_or_else(|| bug!("deref of non-pointer {:?}", self))
129 let (llptr, llextra) = match self.val {
130 OperandValue::Immediate(llptr) => (llptr, None),
131 OperandValue::Pair(llptr, llextra) => (llptr, Some(llextra)),
132 OperandValue::Ref(..) => bug!("Deref of by-Ref operand {:?}", self),
134 let layout = cx.layout_of(projected_ty);
135 PlaceRef { llval: llptr, llextra, layout, align: layout.align.abi }
138 /// If this operand is a `Pair`, we return an aggregate with the two values.
139 /// For other cases, see `immediate`.
140 pub fn immediate_or_packed_pair<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
144 if let OperandValue::Pair(a, b) = self.val {
145 let llty = bx.cx().backend_type(self.layout);
146 debug!("Operand::immediate_or_packed_pair: packing {:?} into {:?}", self, llty);
147 // Reconstruct the immediate aggregate.
148 let mut llpair = bx.cx().const_undef(llty);
149 let imm_a = base::from_immediate(bx, a);
150 let imm_b = base::from_immediate(bx, b);
151 llpair = bx.insert_value(llpair, imm_a, 0);
152 llpair = bx.insert_value(llpair, imm_b, 1);
159 /// If the type is a pair, we return a `Pair`, otherwise, an `Immediate`.
160 pub fn from_immediate_or_packed_pair<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
163 layout: TyLayout<'tcx>,
165 let val = if let layout::Abi::ScalarPair(ref a, ref b) = layout.abi {
166 debug!("Operand::from_immediate_or_packed_pair: unpacking {:?} @ {:?}", llval, layout);
168 // Deconstruct the immediate aggregate.
169 let a_llval = bx.extract_value(llval, 0);
170 let a_llval = base::to_immediate_scalar(bx, a_llval, a);
171 let b_llval = bx.extract_value(llval, 1);
172 let b_llval = base::to_immediate_scalar(bx, b_llval, b);
173 OperandValue::Pair(a_llval, b_llval)
175 OperandValue::Immediate(llval)
177 OperandRef { val, layout }
180 pub fn extract_field<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
185 let field = self.layout.field(bx.cx(), i);
186 let offset = self.layout.fields.offset(i);
188 let mut val = match (self.val, &self.layout.abi) {
189 // If the field is ZST, it has no data.
190 _ if field.is_zst() => {
191 return OperandRef::new_zst(bx, field);
194 // Newtype of a scalar, scalar pair or vector.
195 (OperandValue::Immediate(_), _) | (OperandValue::Pair(..), _)
196 if field.size == self.layout.size =>
198 assert_eq!(offset.bytes(), 0);
202 // Extract a scalar component from a pair.
203 (OperandValue::Pair(a_llval, b_llval), &layout::Abi::ScalarPair(ref a, ref b)) => {
204 if offset.bytes() == 0 {
205 assert_eq!(field.size, a.value.size(bx.cx()));
206 OperandValue::Immediate(a_llval)
208 assert_eq!(offset, a.value.size(bx.cx()).align_to(b.value.align(bx.cx()).abi));
209 assert_eq!(field.size, b.value.size(bx.cx()));
210 OperandValue::Immediate(b_llval)
214 // `#[repr(simd)]` types are also immediate.
215 (OperandValue::Immediate(llval), &layout::Abi::Vector { .. }) => {
216 OperandValue::Immediate(bx.extract_element(llval, bx.cx().const_usize(i as u64)))
219 _ => bug!("OperandRef::extract_field({:?}): not applicable", self),
222 // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types.
223 // Bools in union fields needs to be truncated.
224 let to_immediate_or_cast = |bx: &mut Bx, val, ty| {
225 if ty == bx.cx().type_i1() { bx.trunc(val, ty) } else { bx.bitcast(val, ty) }
229 OperandValue::Immediate(ref mut llval) => {
230 *llval = to_immediate_or_cast(bx, *llval, bx.cx().immediate_backend_type(field));
232 OperandValue::Pair(ref mut a, ref mut b) => {
233 *a = to_immediate_or_cast(
236 bx.cx().scalar_pair_element_backend_type(field, 0, true),
238 *b = to_immediate_or_cast(
241 bx.cx().scalar_pair_element_backend_type(field, 1, true),
244 OperandValue::Ref(..) => bug!(),
247 OperandRef { val, layout: field }
251 impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
252 pub fn store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
255 dest: PlaceRef<'tcx, V>,
257 self.store_with_flags(bx, dest, MemFlags::empty());
260 pub fn volatile_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
263 dest: PlaceRef<'tcx, V>,
265 self.store_with_flags(bx, dest, MemFlags::VOLATILE);
268 pub fn unaligned_volatile_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
271 dest: PlaceRef<'tcx, V>,
273 self.store_with_flags(bx, dest, MemFlags::VOLATILE | MemFlags::UNALIGNED);
276 pub fn nontemporal_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
279 dest: PlaceRef<'tcx, V>,
281 self.store_with_flags(bx, dest, MemFlags::NONTEMPORAL);
284 fn store_with_flags<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
287 dest: PlaceRef<'tcx, V>,
290 debug!("OperandRef::store: operand={:?}, dest={:?}", self, dest);
291 // Avoid generating stores of zero-sized values, because the only way to have a zero-sized
292 // value is through `undef`, and store itself is useless.
293 if dest.layout.is_zst() {
297 OperandValue::Ref(r, None, source_align) => {
298 base::memcpy_ty(bx, dest.llval, dest.align, r, source_align, dest.layout, flags)
300 OperandValue::Ref(_, Some(_), _) => {
301 bug!("cannot directly store unsized values");
303 OperandValue::Immediate(s) => {
304 let val = base::from_immediate(bx, s);
305 bx.store_with_flags(val, dest.llval, dest.align, flags);
307 OperandValue::Pair(a, b) => {
308 let (a_scalar, b_scalar) = match dest.layout.abi {
309 layout::Abi::ScalarPair(ref a, ref b) => (a, b),
310 _ => bug!("store_with_flags: invalid ScalarPair layout: {:#?}", dest.layout),
312 let b_offset = a_scalar.value.size(bx).align_to(b_scalar.value.align(bx).abi);
314 let llptr = bx.struct_gep(dest.llval, 0);
315 let val = base::from_immediate(bx, a);
316 let align = dest.align;
317 bx.store_with_flags(val, llptr, align, flags);
319 let llptr = bx.struct_gep(dest.llval, 1);
320 let val = base::from_immediate(bx, b);
321 let align = dest.align.restrict_for_offset(b_offset);
322 bx.store_with_flags(val, llptr, align, flags);
327 pub fn store_unsized<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
330 indirect_dest: PlaceRef<'tcx, V>,
332 debug!("OperandRef::store_unsized: operand={:?}, indirect_dest={:?}", self, indirect_dest);
333 let flags = MemFlags::empty();
335 // `indirect_dest` must have `*mut T` type. We extract `T` out of it.
336 let unsized_ty = indirect_dest
340 .unwrap_or_else(|| bug!("indirect_dest has non-pointer type: {:?}", indirect_dest))
343 let (llptr, llextra) = if let OperandValue::Ref(llptr, Some(llextra), _) = self {
346 bug!("store_unsized called with a sized value")
349 // FIXME: choose an appropriate alignment, or use dynamic align somehow
350 let max_align = Align::from_bits(128).unwrap();
351 let min_align = Align::from_bits(8).unwrap();
353 // Allocate an appropriate region on the stack, and copy the value into it
354 let (llsize, _) = glue::size_and_align_of_dst(bx, unsized_ty, Some(llextra));
355 let lldst = bx.array_alloca(bx.cx().type_i8(), llsize, max_align);
356 bx.memcpy(lldst, max_align, llptr, min_align, llsize, flags);
358 // Store the allocated region and the extra to the indirect place.
359 let indirect_operand = OperandValue::Pair(lldst, llextra);
360 indirect_operand.store(bx, indirect_dest);
364 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
365 fn maybe_codegen_consume_direct(
368 place_ref: mir::PlaceRef<'_, 'tcx>,
369 ) -> Option<OperandRef<'tcx, Bx::Value>> {
370 debug!("maybe_codegen_consume_direct(place_ref={:?})", place_ref);
372 match self.locals[place_ref.local] {
373 LocalRef::Operand(Some(mut o)) => {
374 // Moves out of scalar and scalar pair fields are trivial.
375 for elem in place_ref.projection.iter() {
377 mir::ProjectionElem::Field(ref f, _) => {
378 o = o.extract_field(bx, f.index());
380 mir::ProjectionElem::Index(_)
381 | mir::ProjectionElem::ConstantIndex { .. } => {
382 // ZSTs don't require any actual memory access.
383 // FIXME(eddyb) deduplicate this with the identical
384 // checks in `codegen_consume` and `extract_field`.
385 let elem = o.layout.field(bx.cx(), 0);
387 o = OperandRef::new_zst(bx, elem);
398 LocalRef::Operand(None) => {
399 bug!("use of {:?} before def", place_ref);
401 LocalRef::Place(..) | LocalRef::UnsizedPlace(..) => {
402 // watch out for locals that do not have an
403 // alloca; they are handled somewhat differently
409 pub fn codegen_consume(
412 place_ref: mir::PlaceRef<'_, 'tcx>,
413 ) -> OperandRef<'tcx, Bx::Value> {
414 debug!("codegen_consume(place_ref={:?})", place_ref);
416 let ty = self.monomorphized_place_ty(place_ref);
417 let layout = bx.cx().layout_of(ty);
419 // ZSTs don't require any actual memory access.
421 return OperandRef::new_zst(bx, layout);
424 if let Some(o) = self.maybe_codegen_consume_direct(bx, place_ref) {
428 // for most places, to consume them we just load them
429 // out from their home
430 let place = self.codegen_place(bx, place_ref);
431 bx.load_operand(place)
434 pub fn codegen_operand(
437 operand: &mir::Operand<'tcx>,
438 ) -> OperandRef<'tcx, Bx::Value> {
439 debug!("codegen_operand(operand={:?})", operand);
442 mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => {
443 self.codegen_consume(bx, place.as_ref())
446 mir::Operand::Constant(ref constant) => {
447 self.eval_mir_constant_to_operand(bx, constant).unwrap_or_else(|err| {
449 // errored or at least linted
450 ErrorHandled::Reported => {}
451 ErrorHandled::TooGeneric => bug!("codgen encountered polymorphic constant"),
453 // Allow RalfJ to sleep soundly knowing that even refactorings that remove
454 // the above error (or silence it under some conditions) will not cause UB.
456 // We still have to return an operand but it doesn't matter,
457 // this code is unreachable.
458 let ty = self.monomorphize(&constant.literal.ty);
459 let layout = bx.cx().layout_of(ty);
460 bx.load_operand(PlaceRef::new_sized(
461 bx.cx().const_undef(bx.cx().type_ptr_to(bx.cx().backend_type(layout))),