1 use super::place::PlaceRef;
2 use super::{FunctionCx, LocalRef};
10 use rustc_middle::mir::interpret::{ConstValue, Pointer, Scalar};
11 use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
12 use rustc_middle::ty::Ty;
13 use rustc_target::abi::{Abi, Align, Size};
17 /// The representation of a Rust value. The enum variant is in fact
18 /// uniquely determined by the value's type, but is kept as a
20 #[derive(Copy, Clone, Debug)]
21 pub enum OperandValue<V> {
22 /// A reference to the actual operand. The data is guaranteed
23 /// to be valid for the operand's lifetime.
24 /// The second value, if any, is the extra data (vtable or length)
25 /// which indicates that it refers to an unsized rvalue.
26 Ref(V, Option<V>, Align),
27 /// A single LLVM value.
29 /// A pair of immediate LLVM values. Used by fat pointers too.
33 /// An `OperandRef` is an "SSA" reference to a Rust value, along with
36 /// NOTE: unless you know a value's type exactly, you should not
37 /// generate LLVM opcodes acting on it and instead act via methods,
38 /// to avoid nasty edge cases. In particular, using `Builder::store`
39 /// directly is sure to cause problems -- use `OperandRef::store`
41 #[derive(Copy, Clone)]
42 pub struct OperandRef<'tcx, V> {
44 pub val: OperandValue<V>,
46 // The layout of value, based on its Rust type.
47 pub layout: TyAndLayout<'tcx>,
50 impl<V: CodegenObject> fmt::Debug for OperandRef<'_, V> {
51 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
52 write!(f, "OperandRef({:?} @ {:?})", self.val, self.layout)
56 impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
57 pub fn new_zst<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
59 layout: TyAndLayout<'tcx>,
60 ) -> OperandRef<'tcx, V> {
61 assert!(layout.is_zst());
63 val: OperandValue::Immediate(bx.const_undef(bx.immediate_backend_type(layout))),
68 pub fn from_const<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
70 val: ConstValue<'tcx>,
73 let layout = bx.layout_of(ty);
76 return OperandRef::new_zst(bx, layout);
80 ConstValue::Scalar(x) => {
81 let Abi::Scalar(scalar) = layout.abi else {
82 bug!("from_const: invalid ByVal layout: {:#?}", layout);
84 let llval = bx.scalar_to_backend(x, scalar, bx.immediate_backend_type(layout));
85 OperandValue::Immediate(llval)
87 ConstValue::ZeroSized => {
88 let llval = bx.zst_to_backend(bx.immediate_backend_type(layout));
89 OperandValue::Immediate(llval)
91 ConstValue::Slice { data, start, end } => {
92 let Abi::ScalarPair(a_scalar, _) = layout.abi else {
93 bug!("from_const: invalid ScalarPair layout: {:#?}", layout);
95 let a = Scalar::from_pointer(
96 Pointer::new(bx.tcx().create_memory_alloc(data), Size::from_bytes(start)),
99 let a_llval = bx.scalar_to_backend(
102 bx.scalar_pair_element_backend_type(layout, 0, true),
104 let b_llval = bx.const_usize((end - start) as u64);
105 OperandValue::Pair(a_llval, b_llval)
107 ConstValue::ByRef { alloc, offset } => {
108 return bx.load_operand(bx.from_const_alloc(layout, alloc, offset));
112 OperandRef { val, layout }
115 /// Asserts that this operand refers to a scalar and returns
116 /// a reference to its value.
117 pub fn immediate(self) -> V {
119 OperandValue::Immediate(s) => s,
120 _ => bug!("not immediate: {:?}", self),
124 pub fn deref<Cx: LayoutTypeMethods<'tcx>>(self, cx: &Cx) -> PlaceRef<'tcx, V> {
125 if self.layout.ty.is_box() {
126 bug!("dereferencing {:?} in codegen", self.layout.ty);
129 let projected_ty = self
133 .unwrap_or_else(|| bug!("deref of non-pointer {:?}", self))
136 let (llptr, llextra) = match self.val {
137 OperandValue::Immediate(llptr) => (llptr, None),
138 OperandValue::Pair(llptr, llextra) => (llptr, Some(llextra)),
139 OperandValue::Ref(..) => bug!("Deref of by-Ref operand {:?}", self),
141 let layout = cx.layout_of(projected_ty);
142 PlaceRef { llval: llptr, llextra, layout, align: layout.align.abi }
145 /// If this operand is a `Pair`, we return an aggregate with the two values.
146 /// For other cases, see `immediate`.
147 pub fn immediate_or_packed_pair<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
151 if let OperandValue::Pair(a, b) = self.val {
152 let llty = bx.cx().backend_type(self.layout);
153 debug!("Operand::immediate_or_packed_pair: packing {:?} into {:?}", self, llty);
154 // Reconstruct the immediate aggregate.
155 let mut llpair = bx.cx().const_undef(llty);
156 let imm_a = bx.from_immediate(a);
157 let imm_b = bx.from_immediate(b);
158 llpair = bx.insert_value(llpair, imm_a, 0);
159 llpair = bx.insert_value(llpair, imm_b, 1);
166 /// If the type is a pair, we return a `Pair`, otherwise, an `Immediate`.
167 pub fn from_immediate_or_packed_pair<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
170 layout: TyAndLayout<'tcx>,
172 let val = if let Abi::ScalarPair(a, b) = layout.abi {
173 debug!("Operand::from_immediate_or_packed_pair: unpacking {:?} @ {:?}", llval, layout);
175 // Deconstruct the immediate aggregate.
176 let a_llval = bx.extract_value(llval, 0);
177 let a_llval = bx.to_immediate_scalar(a_llval, a);
178 let b_llval = bx.extract_value(llval, 1);
179 let b_llval = bx.to_immediate_scalar(b_llval, b);
180 OperandValue::Pair(a_llval, b_llval)
182 OperandValue::Immediate(llval)
184 OperandRef { val, layout }
187 pub fn extract_field<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
192 let field = self.layout.field(bx.cx(), i);
193 let offset = self.layout.fields.offset(i);
195 let mut val = match (self.val, self.layout.abi) {
196 // If the field is ZST, it has no data.
197 _ if field.is_zst() => {
198 return OperandRef::new_zst(bx, field);
201 // Newtype of a scalar, scalar pair or vector.
202 (OperandValue::Immediate(_) | OperandValue::Pair(..), _)
203 if field.size == self.layout.size =>
205 assert_eq!(offset.bytes(), 0);
209 // Extract a scalar component from a pair.
210 (OperandValue::Pair(a_llval, b_llval), Abi::ScalarPair(a, b)) => {
211 if offset.bytes() == 0 {
212 assert_eq!(field.size, a.size(bx.cx()));
213 OperandValue::Immediate(a_llval)
215 assert_eq!(offset, a.size(bx.cx()).align_to(b.align(bx.cx()).abi));
216 assert_eq!(field.size, b.size(bx.cx()));
217 OperandValue::Immediate(b_llval)
221 // `#[repr(simd)]` types are also immediate.
222 (OperandValue::Immediate(llval), Abi::Vector { .. }) => {
223 OperandValue::Immediate(bx.extract_element(llval, bx.cx().const_usize(i as u64)))
226 _ => bug!("OperandRef::extract_field({:?}): not applicable", self),
229 match (&mut val, field.abi) {
230 (OperandValue::Immediate(llval), _) => {
231 // Bools in union fields needs to be truncated.
232 *llval = bx.to_immediate(*llval, field);
233 // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types.
234 *llval = bx.bitcast(*llval, bx.cx().immediate_backend_type(field));
236 (OperandValue::Pair(a, b), Abi::ScalarPair(a_abi, b_abi)) => {
237 // Bools in union fields needs to be truncated.
238 *a = bx.to_immediate_scalar(*a, a_abi);
239 *b = bx.to_immediate_scalar(*b, b_abi);
240 // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types.
241 *a = bx.bitcast(*a, bx.cx().scalar_pair_element_backend_type(field, 0, true));
242 *b = bx.bitcast(*b, bx.cx().scalar_pair_element_backend_type(field, 1, true));
244 (OperandValue::Pair(..), _) => bug!(),
245 (OperandValue::Ref(..), _) => bug!(),
248 OperandRef { val, layout: field }
252 impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
253 pub fn store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
256 dest: PlaceRef<'tcx, V>,
258 self.store_with_flags(bx, dest, MemFlags::empty());
261 pub fn volatile_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
264 dest: PlaceRef<'tcx, V>,
266 self.store_with_flags(bx, dest, MemFlags::VOLATILE);
269 pub fn unaligned_volatile_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
272 dest: PlaceRef<'tcx, V>,
274 self.store_with_flags(bx, dest, MemFlags::VOLATILE | MemFlags::UNALIGNED);
277 pub fn nontemporal_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
280 dest: PlaceRef<'tcx, V>,
282 self.store_with_flags(bx, dest, MemFlags::NONTEMPORAL);
285 fn store_with_flags<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
288 dest: PlaceRef<'tcx, V>,
291 debug!("OperandRef::store: operand={:?}, dest={:?}", self, dest);
292 // Avoid generating stores of zero-sized values, because the only way to have a zero-sized
293 // value is through `undef`, and store itself is useless.
294 if dest.layout.is_zst() {
298 OperandValue::Ref(r, None, source_align) => {
299 if flags.contains(MemFlags::NONTEMPORAL) {
300 // HACK(nox): This is inefficient but there is no nontemporal memcpy.
301 let ty = bx.backend_type(dest.layout);
302 let ptr = bx.pointercast(r, bx.type_ptr_to(ty));
303 let val = bx.load(ty, ptr, source_align);
304 bx.store_with_flags(val, dest.llval, dest.align, flags);
307 base::memcpy_ty(bx, dest.llval, dest.align, r, source_align, dest.layout, flags)
309 OperandValue::Ref(_, Some(_), _) => {
310 bug!("cannot directly store unsized values");
312 OperandValue::Immediate(s) => {
313 let val = bx.from_immediate(s);
314 bx.store_with_flags(val, dest.llval, dest.align, flags);
316 OperandValue::Pair(a, b) => {
317 let Abi::ScalarPair(a_scalar, b_scalar) = dest.layout.abi else {
318 bug!("store_with_flags: invalid ScalarPair layout: {:#?}", dest.layout);
320 let ty = bx.backend_type(dest.layout);
321 let b_offset = a_scalar.size(bx).align_to(b_scalar.align(bx).abi);
323 let llptr = bx.struct_gep(ty, dest.llval, 0);
324 let val = bx.from_immediate(a);
325 let align = dest.align;
326 bx.store_with_flags(val, llptr, align, flags);
328 let llptr = bx.struct_gep(ty, dest.llval, 1);
329 let val = bx.from_immediate(b);
330 let align = dest.align.restrict_for_offset(b_offset);
331 bx.store_with_flags(val, llptr, align, flags);
336 pub fn store_unsized<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
339 indirect_dest: PlaceRef<'tcx, V>,
341 debug!("OperandRef::store_unsized: operand={:?}, indirect_dest={:?}", self, indirect_dest);
342 let flags = MemFlags::empty();
344 // `indirect_dest` must have `*mut T` type. We extract `T` out of it.
345 let unsized_ty = indirect_dest
349 .unwrap_or_else(|| bug!("indirect_dest has non-pointer type: {:?}", indirect_dest))
352 let OperandValue::Ref(llptr, Some(llextra), _) = self else {
353 bug!("store_unsized called with a sized value")
356 // FIXME: choose an appropriate alignment, or use dynamic align somehow
357 let max_align = Align::from_bits(128).unwrap();
358 let min_align = Align::from_bits(8).unwrap();
360 // Allocate an appropriate region on the stack, and copy the value into it
361 let (llsize, _) = glue::size_and_align_of_dst(bx, unsized_ty, Some(llextra));
362 let lldst = bx.array_alloca(bx.cx().type_i8(), llsize, max_align);
363 bx.memcpy(lldst, max_align, llptr, min_align, llsize, flags);
365 // Store the allocated region and the extra to the indirect place.
366 let indirect_operand = OperandValue::Pair(lldst, llextra);
367 indirect_operand.store(bx, indirect_dest);
371 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
372 fn maybe_codegen_consume_direct(
375 place_ref: mir::PlaceRef<'tcx>,
376 ) -> Option<OperandRef<'tcx, Bx::Value>> {
377 debug!("maybe_codegen_consume_direct(place_ref={:?})", place_ref);
379 match self.locals[place_ref.local] {
380 LocalRef::Operand(Some(mut o)) => {
381 // Moves out of scalar and scalar pair fields are trivial.
382 for elem in place_ref.projection.iter() {
384 mir::ProjectionElem::Field(ref f, _) => {
385 o = o.extract_field(bx, f.index());
387 mir::ProjectionElem::Index(_)
388 | mir::ProjectionElem::ConstantIndex { .. } => {
389 // ZSTs don't require any actual memory access.
390 // FIXME(eddyb) deduplicate this with the identical
391 // checks in `codegen_consume` and `extract_field`.
392 let elem = o.layout.field(bx.cx(), 0);
394 o = OperandRef::new_zst(bx, elem);
405 LocalRef::Operand(None) => {
406 bug!("use of {:?} before def", place_ref);
408 LocalRef::Place(..) | LocalRef::UnsizedPlace(..) => {
409 // watch out for locals that do not have an
410 // alloca; they are handled somewhat differently
416 pub fn codegen_consume(
419 place_ref: mir::PlaceRef<'tcx>,
420 ) -> OperandRef<'tcx, Bx::Value> {
421 debug!("codegen_consume(place_ref={:?})", place_ref);
423 let ty = self.monomorphized_place_ty(place_ref);
424 let layout = bx.cx().layout_of(ty);
426 // ZSTs don't require any actual memory access.
428 return OperandRef::new_zst(bx, layout);
431 if let Some(o) = self.maybe_codegen_consume_direct(bx, place_ref) {
435 // for most places, to consume them we just load them
436 // out from their home
437 let place = self.codegen_place(bx, place_ref);
438 bx.load_operand(place)
441 pub fn codegen_operand(
444 operand: &mir::Operand<'tcx>,
445 ) -> OperandRef<'tcx, Bx::Value> {
446 debug!("codegen_operand(operand={:?})", operand);
449 mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => {
450 self.codegen_consume(bx, place.as_ref())
453 mir::Operand::Constant(ref constant) => {
454 // This cannot fail because we checked all required_consts in advance.
455 self.eval_mir_constant_to_operand(bx, constant).unwrap_or_else(|_err| {
456 span_bug!(constant.span, "erroneous constant not captured by required_consts")