1 use rustc::mir::interpret::{ConstValue, ErrorHandled, Pointer, Scalar};
4 use rustc::ty::layout::{self, Align, LayoutOf, TyLayout, Size};
14 use super::{FunctionCx, LocalRef};
15 use super::place::PlaceRef;
17 /// The representation of a Rust value. The enum variant is in fact
18 /// uniquely determined by the value's type, but is kept as a
20 #[derive(Copy, Clone, Debug)]
21 pub enum OperandValue<V> {
22 /// A reference to the actual operand. The data is guaranteed
23 /// to be valid for the operand's lifetime.
24 /// The second value, if any, is the extra data (vtable or length)
25 /// which indicates that it refers to an unsized rvalue.
26 Ref(V, Option<V>, Align),
27 /// A single LLVM value.
29 /// A pair of immediate LLVM values. Used by fat pointers too.
33 /// An `OperandRef` is an "SSA" reference to a Rust value, along with
36 /// NOTE: unless you know a value's type exactly, you should not
37 /// generate LLVM opcodes acting on it and instead act via methods,
38 /// to avoid nasty edge cases. In particular, using `Builder::store`
39 /// directly is sure to cause problems -- use `OperandRef::store`
41 #[derive(Copy, Clone)]
42 pub struct OperandRef<'tcx, V> {
44 pub val: OperandValue<V>,
46 // The layout of value, based on its Rust type.
47 pub layout: TyLayout<'tcx>,
50 impl<V: CodegenObject> fmt::Debug for OperandRef<'tcx, V> {
51 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
52 write!(f, "OperandRef({:?} @ {:?})", self.val, self.layout)
56 impl<'a, 'tcx: 'a, V: CodegenObject> OperandRef<'tcx, V> {
57 pub fn new_zst<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
59 layout: TyLayout<'tcx>
60 ) -> OperandRef<'tcx, V> {
61 assert!(layout.is_zst());
63 val: OperandValue::Immediate(bx.const_undef(bx.immediate_backend_type(layout))),
68 pub fn from_const<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
70 val: &'tcx ty::Const<'tcx>
71 ) -> Result<Self, ErrorHandled> {
72 let layout = bx.layout_of(val.ty);
75 return Ok(OperandRef::new_zst(bx, layout));
78 let val = match val.val {
79 ConstValue::Unevaluated(..) => bug!("unevaluated constant in `OperandRef::from_const`"),
80 ConstValue::Param(_) => bug!("encountered a ConstValue::Param in codegen"),
81 ConstValue::Infer(_) => bug!("encountered a ConstValue::Infer in codegen"),
82 ConstValue::Placeholder(_) => bug!("encountered a ConstValue::Placeholder in codegen"),
83 ConstValue::Scalar(x) => {
84 let scalar = match layout.abi {
85 layout::Abi::Scalar(ref x) => x,
86 _ => bug!("from_const: invalid ByVal layout: {:#?}", layout)
88 let llval = bx.scalar_to_backend(
91 bx.immediate_backend_type(layout),
93 OperandValue::Immediate(llval)
95 ConstValue::Slice { data, start, end } => {
96 let a_scalar = match layout.abi {
97 layout::Abi::ScalarPair(ref a, _) => a,
98 _ => bug!("from_const: invalid ScalarPair layout: {:#?}", layout)
100 let a = Scalar::from(Pointer::new(
101 bx.tcx().alloc_map.lock().allocate(data),
102 Size::from_bytes(start as u64),
104 let a_llval = bx.scalar_to_backend(
107 bx.scalar_pair_element_backend_type(layout, 0, true),
109 let b_llval = bx.const_usize((end - start) as u64);
110 OperandValue::Pair(a_llval, b_llval)
112 ConstValue::ByRef(ptr, alloc) => {
113 return Ok(bx.load_operand(bx.from_const_alloc(layout, alloc, ptr.offset)));
123 /// Asserts that this operand refers to a scalar and returns
124 /// a reference to its value.
125 pub fn immediate(self) -> V {
127 OperandValue::Immediate(s) => s,
128 _ => bug!("not immediate: {:?}", self)
132 pub fn deref<Cx: LayoutTypeMethods<'tcx>>(
135 ) -> PlaceRef<'tcx, V> {
136 let projected_ty = self.layout.ty.builtin_deref(true)
137 .unwrap_or_else(|| bug!("deref of non-pointer {:?}", self)).ty;
138 let (llptr, llextra) = match self.val {
139 OperandValue::Immediate(llptr) => (llptr, None),
140 OperandValue::Pair(llptr, llextra) => (llptr, Some(llextra)),
141 OperandValue::Ref(..) => bug!("Deref of by-Ref operand {:?}", self)
143 let layout = cx.layout_of(projected_ty);
148 align: layout.align.abi,
152 /// If this operand is a `Pair`, we return an aggregate with the two values.
153 /// For other cases, see `immediate`.
154 pub fn immediate_or_packed_pair<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
158 if let OperandValue::Pair(a, b) = self.val {
159 let llty = bx.cx().backend_type(self.layout);
160 debug!("Operand::immediate_or_packed_pair: packing {:?} into {:?}",
162 // Reconstruct the immediate aggregate.
163 let mut llpair = bx.cx().const_undef(llty);
164 let imm_a = base::from_immediate(bx, a);
165 let imm_b = base::from_immediate(bx, b);
166 llpair = bx.insert_value(llpair, imm_a, 0);
167 llpair = bx.insert_value(llpair, imm_b, 1);
174 /// If the type is a pair, we return a `Pair`, otherwise, an `Immediate`.
175 pub fn from_immediate_or_packed_pair<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
178 layout: TyLayout<'tcx>
180 let val = if let layout::Abi::ScalarPair(ref a, ref b) = layout.abi {
181 debug!("Operand::from_immediate_or_packed_pair: unpacking {:?} @ {:?}",
184 // Deconstruct the immediate aggregate.
185 let a_llval = bx.extract_value(llval, 0);
186 let a_llval = base::to_immediate_scalar(bx, a_llval, a);
187 let b_llval = bx.extract_value(llval, 1);
188 let b_llval = base::to_immediate_scalar(bx, b_llval, b);
189 OperandValue::Pair(a_llval, b_llval)
191 OperandValue::Immediate(llval)
193 OperandRef { val, layout }
196 pub fn extract_field<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
201 let field = self.layout.field(bx.cx(), i);
202 let offset = self.layout.fields.offset(i);
204 let mut val = match (self.val, &self.layout.abi) {
205 // If the field is ZST, it has no data.
206 _ if field.is_zst() => {
207 return OperandRef::new_zst(bx, field);
210 // Newtype of a scalar, scalar pair or vector.
211 (OperandValue::Immediate(_), _) |
212 (OperandValue::Pair(..), _) if field.size == self.layout.size => {
213 assert_eq!(offset.bytes(), 0);
217 // Extract a scalar component from a pair.
218 (OperandValue::Pair(a_llval, b_llval), &layout::Abi::ScalarPair(ref a, ref b)) => {
219 if offset.bytes() == 0 {
220 assert_eq!(field.size, a.value.size(bx.cx()));
221 OperandValue::Immediate(a_llval)
223 assert_eq!(offset, a.value.size(bx.cx())
224 .align_to(b.value.align(bx.cx()).abi));
225 assert_eq!(field.size, b.value.size(bx.cx()));
226 OperandValue::Immediate(b_llval)
230 // `#[repr(simd)]` types are also immediate.
231 (OperandValue::Immediate(llval), &layout::Abi::Vector { .. }) => {
232 OperandValue::Immediate(
233 bx.extract_element(llval, bx.cx().const_usize(i as u64)))
236 _ => bug!("OperandRef::extract_field({:?}): not applicable", self)
239 // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types.
240 // Bools in union fields needs to be truncated.
241 let to_immediate_or_cast = |bx: &mut Bx, val, ty| {
242 if ty == bx.cx().type_i1() {
250 OperandValue::Immediate(ref mut llval) => {
251 *llval = to_immediate_or_cast(bx, *llval, bx.cx().immediate_backend_type(field));
253 OperandValue::Pair(ref mut a, ref mut b) => {
254 *a = to_immediate_or_cast(bx, *a, bx.cx()
255 .scalar_pair_element_backend_type(field, 0, true));
256 *b = to_immediate_or_cast(bx, *b, bx.cx()
257 .scalar_pair_element_backend_type(field, 1, true));
259 OperandValue::Ref(..) => bug!()
269 impl<'a, 'tcx: 'a, V: CodegenObject> OperandValue<V> {
270 pub fn store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
273 dest: PlaceRef<'tcx, V>
275 self.store_with_flags(bx, dest, MemFlags::empty());
278 pub fn volatile_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
281 dest: PlaceRef<'tcx, V>
283 self.store_with_flags(bx, dest, MemFlags::VOLATILE);
286 pub fn unaligned_volatile_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
289 dest: PlaceRef<'tcx, V>,
291 self.store_with_flags(bx, dest, MemFlags::VOLATILE | MemFlags::UNALIGNED);
294 pub fn nontemporal_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
297 dest: PlaceRef<'tcx, V>
299 self.store_with_flags(bx, dest, MemFlags::NONTEMPORAL);
302 fn store_with_flags<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
305 dest: PlaceRef<'tcx, V>,
308 debug!("OperandRef::store: operand={:?}, dest={:?}", self, dest);
309 // Avoid generating stores of zero-sized values, because the only way to have a zero-sized
310 // value is through `undef`, and store itself is useless.
311 if dest.layout.is_zst() {
315 OperandValue::Ref(r, None, source_align) => {
316 base::memcpy_ty(bx, dest.llval, dest.align, r, source_align,
319 OperandValue::Ref(_, Some(_), _) => {
320 bug!("cannot directly store unsized values");
322 OperandValue::Immediate(s) => {
323 let val = base::from_immediate(bx, s);
324 bx.store_with_flags(val, dest.llval, dest.align, flags);
326 OperandValue::Pair(a, b) => {
327 let (a_scalar, b_scalar) = match dest.layout.abi {
328 layout::Abi::ScalarPair(ref a, ref b) => (a, b),
329 _ => bug!("store_with_flags: invalid ScalarPair layout: {:#?}", dest.layout)
331 let b_offset = a_scalar.value.size(bx).align_to(b_scalar.value.align(bx).abi);
333 let llptr = bx.struct_gep(dest.llval, 0);
334 let val = base::from_immediate(bx, a);
335 let align = dest.align;
336 bx.store_with_flags(val, llptr, align, flags);
338 let llptr = bx.struct_gep(dest.llval, 1);
339 let val = base::from_immediate(bx, b);
340 let align = dest.align.restrict_for_offset(b_offset);
341 bx.store_with_flags(val, llptr, align, flags);
345 pub fn store_unsized<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
348 indirect_dest: PlaceRef<'tcx, V>
350 debug!("OperandRef::store_unsized: operand={:?}, indirect_dest={:?}", self, indirect_dest);
351 let flags = MemFlags::empty();
353 // `indirect_dest` must have `*mut T` type. We extract `T` out of it.
354 let unsized_ty = indirect_dest.layout.ty.builtin_deref(true)
355 .unwrap_or_else(|| bug!("indirect_dest has non-pointer type: {:?}", indirect_dest)).ty;
357 let (llptr, llextra) =
358 if let OperandValue::Ref(llptr, Some(llextra), _) = self {
361 bug!("store_unsized called with a sized value")
364 // FIXME: choose an appropriate alignment, or use dynamic align somehow
365 let max_align = Align::from_bits(128).unwrap();
366 let min_align = Align::from_bits(8).unwrap();
368 // Allocate an appropriate region on the stack, and copy the value into it
369 let (llsize, _) = glue::size_and_align_of_dst(bx, unsized_ty, Some(llextra));
370 let lldst = bx.array_alloca(bx.cx().type_i8(), llsize, "unsized_tmp", max_align);
371 bx.memcpy(lldst, max_align, llptr, min_align, llsize, flags);
373 // Store the allocated region and the extra to the indirect place.
374 let indirect_operand = OperandValue::Pair(lldst, llextra);
375 indirect_operand.store(bx, indirect_dest);
379 impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
380 fn maybe_codegen_consume_direct(
383 place: &mir::Place<'tcx>
384 ) -> Option<OperandRef<'tcx, Bx::Value>> {
385 debug!("maybe_codegen_consume_direct(place={:?})", place);
387 place.iterate(|place_base, place_projection| {
388 if let mir::PlaceBase::Local(index) = place_base {
389 match self.locals[*index] {
390 LocalRef::Operand(Some(mut o)) => {
391 // Moves out of scalar and scalar pair fields are trivial.
392 for proj in place_projection {
394 mir::ProjectionElem::Field(ref f, _) => {
395 o = o.extract_field(bx, f.index());
397 mir::ProjectionElem::Index(_) |
398 mir::ProjectionElem::ConstantIndex { .. } => {
399 // ZSTs don't require any actual memory access.
400 // FIXME(eddyb) deduplicate this with the identical
401 // checks in `codegen_consume` and `extract_field`.
402 let elem = o.layout.field(bx.cx(), 0);
404 o = OperandRef::new_zst(bx, elem);
415 LocalRef::Operand(None) => {
416 bug!("use of {:?} before def", place);
418 LocalRef::Place(..) | LocalRef::UnsizedPlace(..) => {
419 // watch out for locals that do not have an
420 // alloca; they are handled somewhat differently
430 pub fn codegen_consume(
433 place: &mir::Place<'tcx>
434 ) -> OperandRef<'tcx, Bx::Value> {
435 debug!("codegen_consume(place={:?})", place);
437 let ty = self.monomorphized_place_ty(place);
438 let layout = bx.cx().layout_of(ty);
440 // ZSTs don't require any actual memory access.
442 return OperandRef::new_zst(bx, layout);
445 if let Some(o) = self.maybe_codegen_consume_direct(bx, place) {
449 // for most places, to consume them we just load them
450 // out from their home
451 let place = self.codegen_place(bx, place);
452 bx.load_operand(place)
455 pub fn codegen_operand(
458 operand: &mir::Operand<'tcx>
459 ) -> OperandRef<'tcx, Bx::Value> {
460 debug!("codegen_operand(operand={:?})", operand);
463 mir::Operand::Copy(ref place) |
464 mir::Operand::Move(ref place) => {
465 self.codegen_consume(bx, place)
468 mir::Operand::Constant(ref constant) => {
469 let ty = self.monomorphize(&constant.ty);
470 self.eval_mir_constant(constant)
471 .and_then(|c| OperandRef::from_const(bx, c))
472 .unwrap_or_else(|err| {
474 // errored or at least linted
475 ErrorHandled::Reported => {},
476 ErrorHandled::TooGeneric => {
477 bug!("codgen encountered polymorphic constant")
480 // Allow RalfJ to sleep soundly knowing that even refactorings that remove
481 // the above error (or silence it under some conditions) will not cause UB
483 // We've errored, so we don't have to produce working code.
484 let layout = bx.cx().layout_of(ty);
485 bx.load_operand(PlaceRef::new_sized(
486 bx.cx().const_undef(bx.cx().type_ptr_to(bx.cx().backend_type(layout))),