3 use std::convert::TryFrom;
18 use rustc_apfloat::{ieee, Float, Round, Status};
19 use rustc_codegen_ssa::MemFlags;
20 use rustc_codegen_ssa::common::{
21 AtomicOrdering, AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope, TypeKind,
23 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
24 use rustc_codegen_ssa::mir::place::PlaceRef;
25 use rustc_codegen_ssa::traits::{
36 use rustc_data_structures::fx::FxHashSet;
37 use rustc_middle::bug;
38 use rustc_middle::ty::{ParamEnv, Ty, TyCtxt};
39 use rustc_middle::ty::layout::{FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, HasTyCtxt, LayoutError, LayoutOfHelpers, TyAndLayout};
41 use rustc_span::def_id::DefId;
42 use rustc_target::abi::{
51 use rustc_target::spec::{HasTargetSpec, Target};
53 use crate::common::{SignType, TypeReflection, type_is_pointer};
54 use crate::context::CodegenCx;
55 use crate::intrinsic::llvm;
56 use crate::type_of::LayoutGccExt;
61 // TODO(antoyo): remove this variable.
62 static mut RETURN_VALUE_COUNT: usize = 0;
64 enum ExtremumOperation {
69 pub struct Builder<'a: 'gcc, 'gcc, 'tcx> {
70 pub cx: &'a CodegenCx<'gcc, 'tcx>,
71 pub block: Block<'gcc>,
72 stack_var_count: Cell<usize>,
75 impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
76 fn with_cx(cx: &'a CodegenCx<'gcc, 'tcx>, block: Block<'gcc>) -> Self {
80 stack_var_count: Cell::new(0),
84 fn atomic_extremum(&mut self, operation: ExtremumOperation, dst: RValue<'gcc>, src: RValue<'gcc>, order: AtomicOrdering) -> RValue<'gcc> {
85 let size = src.get_type().get_size();
87 let func = self.current_func();
91 // TODO(antoyo): does this make sense?
92 AtomicOrdering::AcquireRelease | AtomicOrdering::Release => AtomicOrdering::Acquire,
95 let previous_value = self.atomic_load(dst.get_type(), dst, load_ordering, Size::from_bytes(size));
96 let previous_var = func.new_local(None, previous_value.get_type(), "previous_value");
97 let return_value = func.new_local(None, previous_value.get_type(), "return_value");
98 self.llbb().add_assignment(None, previous_var, previous_value);
99 self.llbb().add_assignment(None, return_value, previous_var.to_rvalue());
101 let while_block = func.new_block("while");
102 let after_block = func.new_block("after_while");
103 self.llbb().end_with_jump(None, while_block);
105 // NOTE: since jumps were added and compare_exchange doesn't expect this, the current block in the
106 // state need to be updated.
107 self.switch_to_block(while_block);
109 let comparison_operator =
111 ExtremumOperation::Max => ComparisonOp::LessThan,
112 ExtremumOperation::Min => ComparisonOp::GreaterThan,
115 let cond1 = self.context.new_comparison(None, comparison_operator, previous_var.to_rvalue(), self.context.new_cast(None, src, previous_value.get_type()));
116 let compare_exchange = self.compare_exchange(dst, previous_var, src, order, load_ordering, false);
117 let cond2 = self.cx.context.new_unary_op(None, UnaryOp::LogicalNegate, compare_exchange.get_type(), compare_exchange);
118 let cond = self.cx.context.new_binary_op(None, BinaryOp::LogicalAnd, self.cx.bool_type, cond1, cond2);
120 while_block.end_with_conditional(None, cond, while_block, after_block);
122 // NOTE: since jumps were added in a place rustc does not expect, the current block in the
123 // state need to be updated.
124 self.switch_to_block(after_block);
126 return_value.to_rvalue()
129 fn compare_exchange(&self, dst: RValue<'gcc>, cmp: LValue<'gcc>, src: RValue<'gcc>, order: AtomicOrdering, failure_order: AtomicOrdering, weak: bool) -> RValue<'gcc> {
130 let size = src.get_type().get_size();
131 let compare_exchange = self.context.get_builtin_function(&format!("__atomic_compare_exchange_{}", size));
132 let order = self.context.new_rvalue_from_int(self.i32_type, order.to_gcc());
133 let failure_order = self.context.new_rvalue_from_int(self.i32_type, failure_order.to_gcc());
134 let weak = self.context.new_rvalue_from_int(self.bool_type, weak as i32);
136 let void_ptr_type = self.context.new_type::<*mut ()>();
137 let volatile_void_ptr_type = void_ptr_type.make_volatile();
138 let dst = self.context.new_cast(None, dst, volatile_void_ptr_type);
139 let expected = self.context.new_cast(None, cmp.get_address(None), void_ptr_type);
141 // NOTE: not sure why, but we have the wrong type here.
142 let int_type = compare_exchange.get_param(2).to_rvalue().get_type();
143 let src = self.context.new_cast(None, src, int_type);
144 self.context.new_call(None, compare_exchange, &[dst, expected, src, weak, order, failure_order])
147 pub fn assign(&self, lvalue: LValue<'gcc>, value: RValue<'gcc>) {
148 self.llbb().add_assignment(None, lvalue, value);
151 fn check_call<'b>(&mut self, _typ: &str, func: Function<'gcc>, args: &'b [RValue<'gcc>]) -> Cow<'b, [RValue<'gcc>]> {
152 let mut all_args_match = true;
153 let mut param_types = vec![];
154 let param_count = func.get_param_count();
155 for (index, arg) in args.iter().enumerate().take(param_count) {
156 let param = func.get_param(index as i32);
157 let param = param.to_rvalue().get_type();
158 if param != arg.get_type() {
159 all_args_match = false;
161 param_types.push(param);
165 return Cow::Borrowed(args);
168 let casted_args: Vec<_> = param_types
172 .map(|(_i, (expected_ty, &actual_val))| {
173 let actual_ty = actual_val.get_type();
174 if expected_ty != actual_ty {
175 self.bitcast(actual_val, expected_ty)
183 Cow::Owned(casted_args)
186 fn check_ptr_call<'b>(&mut self, _typ: &str, func_ptr: RValue<'gcc>, args: &'b [RValue<'gcc>]) -> Cow<'b, [RValue<'gcc>]> {
187 let mut all_args_match = true;
188 let mut param_types = vec![];
189 let gcc_func = func_ptr.get_type().dyncast_function_ptr_type().expect("function ptr");
190 for (index, arg) in args.iter().enumerate().take(gcc_func.get_param_count()) {
191 let param = gcc_func.get_param_type(index);
192 if param != arg.get_type() {
193 all_args_match = false;
195 param_types.push(param);
198 let mut on_stack_param_indices = FxHashSet::default();
199 if let Some(indices) = self.on_stack_params.borrow().get(&gcc_func) {
200 on_stack_param_indices = indices.clone();
204 return Cow::Borrowed(args);
207 let func_name = format!("{:?}", func_ptr);
209 let casted_args: Vec<_> = param_types
213 .map(|(index, (expected_ty, &actual_val))| {
214 if llvm::ignore_arg_cast(&func_name, index, args.len()) {
218 let actual_ty = actual_val.get_type();
219 if expected_ty != actual_ty {
220 if !actual_ty.is_vector() && !expected_ty.is_vector() && actual_ty.is_integral() && expected_ty.is_integral() && actual_ty.get_size() != expected_ty.get_size() {
221 self.context.new_cast(None, actual_val, expected_ty)
223 else if on_stack_param_indices.contains(&index) {
224 actual_val.dereference(None).to_rvalue()
227 assert!(!((actual_ty.is_vector() && !expected_ty.is_vector()) || (!actual_ty.is_vector() && expected_ty.is_vector())), "{:?} ({}) -> {:?} ({}), index: {:?}[{}]", actual_ty, actual_ty.is_vector(), expected_ty, expected_ty.is_vector(), func_ptr, index);
228 // TODO(antoyo): perhaps use __builtin_convertvector for vector casting.
229 self.bitcast(actual_val, expected_ty)
238 Cow::Owned(casted_args)
241 fn check_store(&mut self, val: RValue<'gcc>, ptr: RValue<'gcc>) -> RValue<'gcc> {
242 let dest_ptr_ty = self.cx.val_ty(ptr).make_pointer(); // TODO(antoyo): make sure make_pointer() is okay here.
243 let stored_ty = self.cx.val_ty(val);
244 let stored_ptr_ty = self.cx.type_ptr_to(stored_ty);
246 if dest_ptr_ty == stored_ptr_ty {
250 self.bitcast(ptr, stored_ptr_ty)
254 pub fn current_func(&self) -> Function<'gcc> {
255 self.block.get_function()
258 fn function_call(&mut self, func: RValue<'gcc>, args: &[RValue<'gcc>], _funclet: Option<&Funclet>) -> RValue<'gcc> {
259 // TODO(antoyo): remove when the API supports a different type for functions.
260 let func: Function<'gcc> = self.cx.rvalue_as_function(func);
261 let args = self.check_call("call", func, args);
263 // gccjit requires to use the result of functions, even when it's not used.
264 // That's why we assign the result to a local or call add_eval().
265 let return_type = func.get_return_type();
266 let void_type = self.context.new_type::<()>();
267 let current_func = self.block.get_function();
268 if return_type != void_type {
269 unsafe { RETURN_VALUE_COUNT += 1 };
270 let result = current_func.new_local(None, return_type, &format!("returnValue{}", unsafe { RETURN_VALUE_COUNT }));
271 self.block.add_assignment(None, result, self.cx.context.new_call(None, func, &args));
275 self.block.add_eval(None, self.cx.context.new_call(None, func, &args));
276 // Return dummy value when not having return value.
277 self.context.new_rvalue_from_long(self.isize_type, 0)
281 fn function_ptr_call(&mut self, func_ptr: RValue<'gcc>, args: &[RValue<'gcc>], _funclet: Option<&Funclet>) -> RValue<'gcc> {
282 let args = self.check_ptr_call("call", func_ptr, args);
284 // gccjit requires to use the result of functions, even when it's not used.
285 // That's why we assign the result to a local or call add_eval().
286 let gcc_func = func_ptr.get_type().dyncast_function_ptr_type().expect("function ptr");
287 let return_type = gcc_func.get_return_type();
288 let void_type = self.context.new_type::<()>();
289 let current_func = self.block.get_function();
291 if return_type != void_type {
292 unsafe { RETURN_VALUE_COUNT += 1 };
293 let result = current_func.new_local(None, return_type, &format!("ptrReturnValue{}", unsafe { RETURN_VALUE_COUNT }));
294 let func_name = format!("{:?}", func_ptr);
295 let args = llvm::adjust_intrinsic_arguments(&self, gcc_func, args, &func_name);
296 self.block.add_assignment(None, result, self.cx.context.new_call_through_ptr(None, func_ptr, &args));
300 #[cfg(not(feature="master"))]
301 if gcc_func.get_param_count() == 0 {
302 // FIXME(antoyo): As a temporary workaround for unsupported LLVM intrinsics.
303 self.block.add_eval(None, self.cx.context.new_call_through_ptr(None, func_ptr, &[]));
306 self.block.add_eval(None, self.cx.context.new_call_through_ptr(None, func_ptr, &args));
308 #[cfg(feature="master")]
309 self.block.add_eval(None, self.cx.context.new_call_through_ptr(None, func_ptr, &args));
310 // Return dummy value when not having return value.
311 let result = current_func.new_local(None, self.isize_type, "dummyValueThatShouldNeverBeUsed");
312 self.block.add_assignment(None, result, self.context.new_rvalue_from_long(self.isize_type, 0));
317 pub fn overflow_call(&self, func: Function<'gcc>, args: &[RValue<'gcc>], _funclet: Option<&Funclet>) -> RValue<'gcc> {
318 // gccjit requires to use the result of functions, even when it's not used.
319 // That's why we assign the result to a local.
320 let return_type = self.context.new_type::<bool>();
321 let current_func = self.block.get_function();
322 // TODO(antoyo): return the new_call() directly? Since the overflow function has no side-effects.
323 unsafe { RETURN_VALUE_COUNT += 1 };
324 let result = current_func.new_local(None, return_type, &format!("overflowReturnValue{}", unsafe { RETURN_VALUE_COUNT }));
325 self.block.add_assignment(None, result, self.cx.context.new_call(None, func, &args));
330 impl<'gcc, 'tcx> HasCodegen<'tcx> for Builder<'_, 'gcc, 'tcx> {
331 type CodegenCx = CodegenCx<'gcc, 'tcx>;
334 impl<'tcx> HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> {
335 fn tcx(&self) -> TyCtxt<'tcx> {
340 impl HasDataLayout for Builder<'_, '_, '_> {
341 fn data_layout(&self) -> &TargetDataLayout {
342 self.cx.data_layout()
346 impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
347 type LayoutOfResult = TyAndLayout<'tcx>;
350 fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
351 self.cx.handle_layout_err(err, span, ty)
355 impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
356 type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>;
359 fn handle_fn_abi_err(
361 err: FnAbiError<'tcx>,
363 fn_abi_request: FnAbiRequest<'tcx>,
365 self.cx.handle_fn_abi_err(err, span, fn_abi_request)
369 impl<'gcc, 'tcx> Deref for Builder<'_, 'gcc, 'tcx> {
370 type Target = CodegenCx<'gcc, 'tcx>;
372 fn deref(&self) -> &Self::Target {
377 impl<'gcc, 'tcx> BackendTypes for Builder<'_, 'gcc, 'tcx> {
378 type Value = <CodegenCx<'gcc, 'tcx> as BackendTypes>::Value;
379 type Function = <CodegenCx<'gcc, 'tcx> as BackendTypes>::Function;
380 type BasicBlock = <CodegenCx<'gcc, 'tcx> as BackendTypes>::BasicBlock;
381 type Type = <CodegenCx<'gcc, 'tcx> as BackendTypes>::Type;
382 type Funclet = <CodegenCx<'gcc, 'tcx> as BackendTypes>::Funclet;
384 type DIScope = <CodegenCx<'gcc, 'tcx> as BackendTypes>::DIScope;
385 type DILocation = <CodegenCx<'gcc, 'tcx> as BackendTypes>::DILocation;
386 type DIVariable = <CodegenCx<'gcc, 'tcx> as BackendTypes>::DIVariable;
389 impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
390 fn build(cx: &'a CodegenCx<'gcc, 'tcx>, block: Block<'gcc>) -> Self {
391 Builder::with_cx(cx, block)
394 fn llbb(&self) -> Block<'gcc> {
398 fn append_block(cx: &'a CodegenCx<'gcc, 'tcx>, func: RValue<'gcc>, name: &str) -> Block<'gcc> {
399 let func = cx.rvalue_as_function(func);
403 fn append_sibling_block(&mut self, name: &str) -> Block<'gcc> {
404 let func = self.current_func();
408 fn switch_to_block(&mut self, block: Self::BasicBlock) {
412 fn ret_void(&mut self) {
413 self.llbb().end_with_void_return(None)
416 fn ret(&mut self, value: RValue<'gcc>) {
418 if self.structs_as_pointer.borrow().contains(&value) {
419 // NOTE: hack to workaround a limitation of the rustc API: see comment on
420 // CodegenCx.structs_as_pointer
421 value.dereference(None).to_rvalue()
426 self.llbb().end_with_return(None, value);
429 fn br(&mut self, dest: Block<'gcc>) {
430 self.llbb().end_with_jump(None, dest)
433 fn cond_br(&mut self, cond: RValue<'gcc>, then_block: Block<'gcc>, else_block: Block<'gcc>) {
434 self.llbb().end_with_conditional(None, cond, then_block, else_block)
437 fn switch(&mut self, value: RValue<'gcc>, default_block: Block<'gcc>, cases: impl ExactSizeIterator<Item = (u128, Block<'gcc>)>) {
438 let mut gcc_cases = vec![];
439 let typ = self.val_ty(value);
440 for (on_val, dest) in cases {
441 let on_val = self.const_uint_big(typ, on_val);
442 gcc_cases.push(self.context.new_case(on_val, on_val, dest));
444 self.block.end_with_switch(None, value, default_block, &gcc_cases);
450 fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
452 args: &[RValue<'gcc>],
455 _funclet: Option<&Funclet>,
457 // TODO(bjorn3): Properly implement unwinding.
458 let call_site = self.call(typ, None, func, args, None);
459 let condition = self.context.new_rvalue_from_int(self.bool_type, 1);
460 self.llbb().end_with_conditional(None, condition, then, catch);
461 if let Some(_fn_abi) = fn_abi {
462 // TODO(bjorn3): Apply function attributes
467 fn unreachable(&mut self) {
468 let func = self.context.get_builtin_function("__builtin_unreachable");
469 self.block.add_eval(None, self.context.new_call(None, func, &[]));
470 let return_type = self.block.get_function().get_return_type();
471 let void_type = self.context.new_type::<()>();
472 if return_type == void_type {
473 self.block.end_with_void_return(None)
476 let return_value = self.current_func()
477 .new_local(None, return_type, "unreachableReturn");
478 self.block.end_with_return(None, return_value)
482 fn add(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
486 fn fadd(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
490 fn sub(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
494 fn fsub(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
498 fn mul(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
502 fn fmul(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
506 fn udiv(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
510 fn exactudiv(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
511 // TODO(antoyo): poison if not exact.
512 let a_type = a.get_type().to_unsigned(self);
513 let a = self.gcc_int_cast(a, a_type);
514 let b_type = b.get_type().to_unsigned(self);
515 let b = self.gcc_int_cast(b, b_type);
519 fn sdiv(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
523 fn exactsdiv(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
524 // TODO(antoyo): poison if not exact.
525 // FIXME(antoyo): rustc_codegen_ssa::mir::intrinsic uses different types for a and b but they
526 // should be the same.
527 let typ = a.get_type().to_signed(self);
528 let b = self.context.new_cast(None, b, typ);
532 fn fdiv(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
536 fn urem(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
540 fn srem(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
544 fn frem(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
545 if a.get_type().is_compatible_with(self.cx.float_type) {
546 let fmodf = self.context.get_builtin_function("fmodf");
547 // FIXME(antoyo): this seems to produce the wrong result.
548 return self.context.new_call(None, fmodf, &[a, b]);
550 assert_eq!(a.get_type().unqualified(), self.cx.double_type);
552 let fmod = self.context.get_builtin_function("fmod");
553 return self.context.new_call(None, fmod, &[a, b]);
556 fn shl(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
560 fn lshr(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
564 fn ashr(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
565 // TODO(antoyo): check whether behavior is an arithmetic shift for >> .
566 // It seems to be if the value is signed.
570 fn and(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
574 fn or(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
578 fn xor(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
582 fn neg(&mut self, a: RValue<'gcc>) -> RValue<'gcc> {
586 fn fneg(&mut self, a: RValue<'gcc>) -> RValue<'gcc> {
587 self.cx.context.new_unary_op(None, UnaryOp::Minus, a.get_type(), a)
590 fn not(&mut self, a: RValue<'gcc>) -> RValue<'gcc> {
594 fn unchecked_sadd(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
598 fn unchecked_uadd(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
602 fn unchecked_ssub(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
606 fn unchecked_usub(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
607 // TODO(antoyo): should generate poison value?
611 fn unchecked_smul(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
615 fn unchecked_umul(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
619 fn fadd_fast(&mut self, _lhs: RValue<'gcc>, _rhs: RValue<'gcc>) -> RValue<'gcc> {
623 fn fsub_fast(&mut self, _lhs: RValue<'gcc>, _rhs: RValue<'gcc>) -> RValue<'gcc> {
627 fn fmul_fast(&mut self, _lhs: RValue<'gcc>, _rhs: RValue<'gcc>) -> RValue<'gcc> {
631 fn fdiv_fast(&mut self, _lhs: RValue<'gcc>, _rhs: RValue<'gcc>) -> RValue<'gcc> {
635 fn frem_fast(&mut self, _lhs: RValue<'gcc>, _rhs: RValue<'gcc>) -> RValue<'gcc> {
639 fn checked_binop(&mut self, oop: OverflowOp, typ: Ty<'_>, lhs: Self::Value, rhs: Self::Value) -> (Self::Value, Self::Value) {
640 self.gcc_checked_binop(oop, typ, lhs, rhs)
643 fn alloca(&mut self, ty: Type<'gcc>, align: Align) -> RValue<'gcc> {
644 // FIXME(antoyo): this check that we don't call get_aligned() a second time on a type.
645 // Ideally, we shouldn't need to do this check.
647 if ty == self.cx.u128_type || ty == self.cx.i128_type {
651 ty.get_aligned(align.bytes())
653 // TODO(antoyo): It might be better to return a LValue, but fixing the rustc API is non-trivial.
654 self.stack_var_count.set(self.stack_var_count.get() + 1);
655 self.current_func().new_local(None, aligned_type, &format!("stack_var_{}", self.stack_var_count.get())).get_address(None)
658 fn byte_array_alloca(&mut self, _len: RValue<'gcc>, _align: Align) -> RValue<'gcc> {
662 fn load(&mut self, pointee_ty: Type<'gcc>, ptr: RValue<'gcc>, _align: Align) -> RValue<'gcc> {
663 let block = self.llbb();
664 let function = block.get_function();
665 // NOTE: instead of returning the dereference here, we have to assign it to a variable in
666 // the current basic block. Otherwise, it could be used in another basic block, causing a
667 // dereference after a drop, for instance.
668 // TODO(antoyo): handle align of the load instruction.
669 let ptr = self.context.new_cast(None, ptr, pointee_ty.make_pointer());
670 let deref = ptr.dereference(None).to_rvalue();
671 unsafe { RETURN_VALUE_COUNT += 1 };
672 let loaded_value = function.new_local(None, pointee_ty, &format!("loadedValue{}", unsafe { RETURN_VALUE_COUNT }));
673 block.add_assignment(None, loaded_value, deref);
674 loaded_value.to_rvalue()
677 fn volatile_load(&mut self, _ty: Type<'gcc>, ptr: RValue<'gcc>) -> RValue<'gcc> {
678 // TODO(antoyo): use ty.
679 let ptr = self.context.new_cast(None, ptr, ptr.get_type().make_volatile());
680 ptr.dereference(None).to_rvalue()
683 fn atomic_load(&mut self, _ty: Type<'gcc>, ptr: RValue<'gcc>, order: AtomicOrdering, size: Size) -> RValue<'gcc> {
684 // TODO(antoyo): use ty.
685 // TODO(antoyo): handle alignment.
686 let atomic_load = self.context.get_builtin_function(&format!("__atomic_load_{}", size.bytes()));
687 let ordering = self.context.new_rvalue_from_int(self.i32_type, order.to_gcc());
689 let volatile_const_void_ptr_type = self.context.new_type::<()>()
693 let ptr = self.context.new_cast(None, ptr, volatile_const_void_ptr_type);
694 self.context.new_call(None, atomic_load, &[ptr, ordering])
697 fn load_operand(&mut self, place: PlaceRef<'tcx, RValue<'gcc>>) -> OperandRef<'tcx, RValue<'gcc>> {
698 assert_eq!(place.llextra.is_some(), place.layout.is_unsized());
700 if place.layout.is_zst() {
701 return OperandRef::new_zst(self, place.layout);
704 fn scalar_load_metadata<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>, load: RValue<'gcc>, scalar: &abi::Scalar) {
705 let vr = scalar.valid_range(bx);
706 match scalar.primitive() {
708 if !scalar.is_always_valid(bx) {
709 bx.range_metadata(load, vr);
712 abi::Pointer if vr.start < vr.end && !vr.contains(0) => {
713 bx.nonnull_metadata(load);
720 if let Some(llextra) = place.llextra {
721 OperandValue::Ref(place.llval, Some(llextra), place.align)
723 else if place.layout.is_gcc_immediate() {
724 let load = self.load(
725 place.layout.gcc_type(self, false),
729 if let abi::Abi::Scalar(ref scalar) = place.layout.abi {
730 scalar_load_metadata(self, load, scalar);
732 OperandValue::Immediate(self.to_immediate(load, place.layout))
734 else if let abi::Abi::ScalarPair(ref a, ref b) = place.layout.abi {
735 let b_offset = a.size(self).align_to(b.align(self).abi);
736 let pair_type = place.layout.gcc_type(self, false);
738 let mut load = |i, scalar: &abi::Scalar, align| {
739 let llptr = self.struct_gep(pair_type, place.llval, i as u64);
740 let llty = place.layout.scalar_pair_element_gcc_type(self, i, false);
741 let load = self.load(llty, llptr, align);
742 scalar_load_metadata(self, load, scalar);
743 if scalar.is_bool() { self.trunc(load, self.type_i1()) } else { load }
747 load(0, a, place.align),
748 load(1, b, place.align.restrict_for_offset(b_offset)),
752 OperandValue::Ref(place.llval, None, place.align)
755 OperandRef { val, layout: place.layout }
758 fn write_operand_repeatedly(mut self, cg_elem: OperandRef<'tcx, RValue<'gcc>>, count: u64, dest: PlaceRef<'tcx, RValue<'gcc>>) -> Self {
759 let zero = self.const_usize(0);
760 let count = self.const_usize(count);
761 let start = dest.project_index(&mut self, zero).llval;
762 let end = dest.project_index(&mut self, count).llval;
764 let header_bb = self.append_sibling_block("repeat_loop_header");
765 let body_bb = self.append_sibling_block("repeat_loop_body");
766 let next_bb = self.append_sibling_block("repeat_loop_next");
768 let ptr_type = start.get_type();
769 let current = self.llbb().get_function().new_local(None, ptr_type, "loop_var");
770 let current_val = current.to_rvalue();
771 self.assign(current, start);
775 self.switch_to_block(header_bb);
776 let keep_going = self.icmp(IntPredicate::IntNE, current_val, end);
777 self.cond_br(keep_going, body_bb, next_bb);
779 self.switch_to_block(body_bb);
780 let align = dest.align.restrict_for_offset(dest.layout.field(self.cx(), 0).size);
781 cg_elem.val.store(&mut self, PlaceRef::new_sized_aligned(current_val, cg_elem.layout, align));
783 let next = self.inbounds_gep(self.backend_type(cg_elem.layout), current.to_rvalue(), &[self.const_usize(1)]);
784 self.llbb().add_assignment(None, current, next);
787 self.switch_to_block(next_bb);
791 fn range_metadata(&mut self, _load: RValue<'gcc>, _range: WrappingRange) {
795 fn nonnull_metadata(&mut self, _load: RValue<'gcc>) {
799 fn store(&mut self, val: RValue<'gcc>, ptr: RValue<'gcc>, align: Align) -> RValue<'gcc> {
800 self.store_with_flags(val, ptr, align, MemFlags::empty())
803 fn store_with_flags(&mut self, val: RValue<'gcc>, ptr: RValue<'gcc>, align: Align, _flags: MemFlags) -> RValue<'gcc> {
804 let ptr = self.check_store(val, ptr);
805 let destination = ptr.dereference(None);
806 // NOTE: libgccjit does not support specifying the alignment on the assignment, so we cast
807 // to type so it gets the proper alignment.
808 let destination_type = destination.to_rvalue().get_type().unqualified();
809 let aligned_type = destination_type.get_aligned(align.bytes()).make_pointer();
810 let aligned_destination = self.cx.context.new_bitcast(None, ptr, aligned_type);
811 let aligned_destination = aligned_destination.dereference(None);
812 self.llbb().add_assignment(None, aligned_destination, val);
813 // TODO(antoyo): handle align and flags.
814 // NOTE: dummy value here since it's never used. FIXME(antoyo): API should not return a value here?
815 self.cx.context.new_rvalue_zero(self.type_i32())
818 fn atomic_store(&mut self, value: RValue<'gcc>, ptr: RValue<'gcc>, order: AtomicOrdering, size: Size) {
819 // TODO(antoyo): handle alignment.
820 let atomic_store = self.context.get_builtin_function(&format!("__atomic_store_{}", size.bytes()));
821 let ordering = self.context.new_rvalue_from_int(self.i32_type, order.to_gcc());
822 let volatile_const_void_ptr_type = self.context.new_type::<()>()
825 let ptr = self.context.new_cast(None, ptr, volatile_const_void_ptr_type);
827 // FIXME(antoyo): fix libgccjit to allow comparing an integer type with an aligned integer type because
828 // the following cast is required to avoid this error:
829 // gcc_jit_context_new_call: mismatching types for argument 2 of function "__atomic_store_4": assignment to param arg1 (type: int) from loadedValue3577 (type: unsigned int __attribute__((aligned(4))))
830 let int_type = atomic_store.get_param(1).to_rvalue().get_type();
831 let value = self.context.new_cast(None, value, int_type);
833 .add_eval(None, self.context.new_call(None, atomic_store, &[ptr, value, ordering]));
836 fn gep(&mut self, _typ: Type<'gcc>, ptr: RValue<'gcc>, indices: &[RValue<'gcc>]) -> RValue<'gcc> {
837 let mut result = ptr;
838 for index in indices {
839 result = self.context.new_array_access(None, result, *index).get_address(None).to_rvalue();
844 fn inbounds_gep(&mut self, _typ: Type<'gcc>, ptr: RValue<'gcc>, indices: &[RValue<'gcc>]) -> RValue<'gcc> {
845 // FIXME(antoyo): would be safer if doing the same thing (loop) as gep.
846 // TODO(antoyo): specify inbounds somehow.
847 match indices.len() {
849 self.context.new_array_access(None, ptr, indices[0]).get_address(None)
852 let array = ptr.dereference(None); // TODO(antoyo): assert that first index is 0?
853 self.context.new_array_access(None, array, indices[1]).get_address(None)
855 _ => unimplemented!(),
859 fn struct_gep(&mut self, value_type: Type<'gcc>, ptr: RValue<'gcc>, idx: u64) -> RValue<'gcc> {
860 // FIXME(antoyo): it would be better if the API only called this on struct, not on arrays.
861 assert_eq!(idx as usize as u64, idx);
862 let value = ptr.dereference(None).to_rvalue();
864 if value_type.dyncast_array().is_some() {
865 let index = self.context.new_rvalue_from_long(self.u64_type, i64::try_from(idx).expect("i64::try_from"));
866 let element = self.context.new_array_access(None, value, index);
867 element.get_address(None)
869 else if let Some(vector_type) = value_type.dyncast_vector() {
870 let array_type = vector_type.get_element_type().make_pointer();
871 let array = self.bitcast(ptr, array_type);
872 let index = self.context.new_rvalue_from_long(self.u64_type, i64::try_from(idx).expect("i64::try_from"));
873 let element = self.context.new_array_access(None, array, index);
874 element.get_address(None)
876 else if let Some(struct_type) = value_type.is_struct() {
877 ptr.dereference_field(None, struct_type.get_field(idx as i32)).get_address(None)
880 panic!("Unexpected type {:?}", value_type);
885 fn trunc(&mut self, value: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> {
886 // TODO(antoyo): check that it indeed truncate the value.
887 self.gcc_int_cast(value, dest_ty)
890 fn sext(&mut self, value: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> {
891 // TODO(antoyo): check that it indeed sign extend the value.
892 if dest_ty.dyncast_vector().is_some() {
893 // TODO(antoyo): nothing to do as it is only for LLVM?
896 self.context.new_cast(None, value, dest_ty)
899 fn fptoui(&mut self, value: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> {
900 self.gcc_float_to_uint_cast(value, dest_ty)
903 fn fptosi(&mut self, value: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> {
904 self.gcc_float_to_int_cast(value, dest_ty)
907 fn uitofp(&mut self, value: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> {
908 self.gcc_uint_to_float_cast(value, dest_ty)
911 fn sitofp(&mut self, value: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> {
912 self.gcc_int_to_float_cast(value, dest_ty)
915 fn fptrunc(&mut self, value: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> {
916 // TODO(antoyo): make sure it truncates.
917 self.context.new_cast(None, value, dest_ty)
920 fn fpext(&mut self, value: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> {
921 self.context.new_cast(None, value, dest_ty)
924 fn ptrtoint(&mut self, value: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> {
925 let usize_value = self.cx.const_bitcast(value, self.cx.type_isize());
926 self.intcast(usize_value, dest_ty, false)
929 fn inttoptr(&mut self, value: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> {
930 let usize_value = self.intcast(value, self.cx.type_isize(), false);
931 self.cx.const_bitcast(usize_value, dest_ty)
934 fn bitcast(&mut self, value: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> {
935 self.cx.const_bitcast(value, dest_ty)
938 fn intcast(&mut self, value: RValue<'gcc>, dest_typ: Type<'gcc>, _is_signed: bool) -> RValue<'gcc> {
939 // NOTE: is_signed is for value, not dest_typ.
940 self.gcc_int_cast(value, dest_typ)
943 fn pointercast(&mut self, value: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> {
944 let val_type = value.get_type();
945 match (type_is_pointer(val_type), type_is_pointer(dest_ty)) {
947 // NOTE: Projecting a field of a pointer type will attempt a cast from a signed char to
948 // a pointer, which is not supported by gccjit.
949 return self.cx.context.new_cast(None, self.inttoptr(value, val_type.make_pointer()), dest_ty);
952 // When they are not pointers, we want a transmute (or reinterpret_cast).
953 self.bitcast(value, dest_ty)
955 (true, true) => self.cx.context.new_cast(None, value, dest_ty),
956 (true, false) => unimplemented!(),
961 fn icmp(&mut self, op: IntPredicate, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> {
962 self.gcc_icmp(op, lhs, rhs)
965 fn fcmp(&mut self, op: RealPredicate, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> {
966 self.context.new_comparison(None, op.to_gcc_comparison(), lhs, rhs)
969 /* Miscellaneous instructions */
970 fn memcpy(&mut self, dst: RValue<'gcc>, _dst_align: Align, src: RValue<'gcc>, _src_align: Align, size: RValue<'gcc>, flags: MemFlags) {
971 assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported");
972 let size = self.intcast(size, self.type_size_t(), false);
973 let _is_volatile = flags.contains(MemFlags::VOLATILE);
974 let dst = self.pointercast(dst, self.type_i8p());
975 let src = self.pointercast(src, self.type_ptr_to(self.type_void()));
976 let memcpy = self.context.get_builtin_function("memcpy");
977 // TODO(antoyo): handle aligns and is_volatile.
978 self.block.add_eval(None, self.context.new_call(None, memcpy, &[dst, src, size]));
981 fn memmove(&mut self, dst: RValue<'gcc>, dst_align: Align, src: RValue<'gcc>, src_align: Align, size: RValue<'gcc>, flags: MemFlags) {
982 if flags.contains(MemFlags::NONTEMPORAL) {
983 // HACK(nox): This is inefficient but there is no nontemporal memmove.
984 let val = self.load(src.get_type().get_pointee().expect("get_pointee"), src, src_align);
985 let ptr = self.pointercast(dst, self.type_ptr_to(self.val_ty(val)));
986 self.store_with_flags(val, ptr, dst_align, flags);
989 let size = self.intcast(size, self.type_size_t(), false);
990 let _is_volatile = flags.contains(MemFlags::VOLATILE);
991 let dst = self.pointercast(dst, self.type_i8p());
992 let src = self.pointercast(src, self.type_ptr_to(self.type_void()));
994 let memmove = self.context.get_builtin_function("memmove");
995 // TODO(antoyo): handle is_volatile.
996 self.block.add_eval(None, self.context.new_call(None, memmove, &[dst, src, size]));
999 fn memset(&mut self, ptr: RValue<'gcc>, fill_byte: RValue<'gcc>, size: RValue<'gcc>, _align: Align, flags: MemFlags) {
1000 let _is_volatile = flags.contains(MemFlags::VOLATILE);
1001 let ptr = self.pointercast(ptr, self.type_i8p());
1002 let memset = self.context.get_builtin_function("memset");
1003 // TODO(antoyo): handle align and is_volatile.
1004 let fill_byte = self.context.new_cast(None, fill_byte, self.i32_type);
1005 let size = self.intcast(size, self.type_size_t(), false);
1006 self.block.add_eval(None, self.context.new_call(None, memset, &[ptr, fill_byte, size]));
1009 fn select(&mut self, cond: RValue<'gcc>, then_val: RValue<'gcc>, mut else_val: RValue<'gcc>) -> RValue<'gcc> {
1010 let func = self.current_func();
1011 let variable = func.new_local(None, then_val.get_type(), "selectVar");
1012 let then_block = func.new_block("then");
1013 let else_block = func.new_block("else");
1014 let after_block = func.new_block("after");
1015 self.llbb().end_with_conditional(None, cond, then_block, else_block);
1017 then_block.add_assignment(None, variable, then_val);
1018 then_block.end_with_jump(None, after_block);
1020 if !then_val.get_type().is_compatible_with(else_val.get_type()) {
1021 else_val = self.context.new_cast(None, else_val, then_val.get_type());
1023 else_block.add_assignment(None, variable, else_val);
1024 else_block.end_with_jump(None, after_block);
1026 // NOTE: since jumps were added in a place rustc does not expect, the current block in the
1027 // state need to be updated.
1028 self.switch_to_block(after_block);
1030 variable.to_rvalue()
1034 fn va_arg(&mut self, _list: RValue<'gcc>, _ty: Type<'gcc>) -> RValue<'gcc> {
1038 fn extract_element(&mut self, _vec: RValue<'gcc>, _idx: RValue<'gcc>) -> RValue<'gcc> {
1042 fn vector_splat(&mut self, _num_elts: usize, _elt: RValue<'gcc>) -> RValue<'gcc> {
1046 fn extract_value(&mut self, aggregate_value: RValue<'gcc>, idx: u64) -> RValue<'gcc> {
1047 // FIXME(antoyo): it would be better if the API only called this on struct, not on arrays.
1048 assert_eq!(idx as usize as u64, idx);
1049 let value_type = aggregate_value.get_type();
1051 if value_type.dyncast_array().is_some() {
1052 let index = self.context.new_rvalue_from_long(self.u64_type, i64::try_from(idx).expect("i64::try_from"));
1053 let element = self.context.new_array_access(None, aggregate_value, index);
1054 element.get_address(None)
1056 else if value_type.dyncast_vector().is_some() {
1059 else if let Some(pointer_type) = value_type.get_pointee() {
1060 if let Some(struct_type) = pointer_type.is_struct() {
1061 // NOTE: hack to workaround a limitation of the rustc API: see comment on
1062 // CodegenCx.structs_as_pointer
1063 aggregate_value.dereference_field(None, struct_type.get_field(idx as i32)).to_rvalue()
1066 panic!("Unexpected type {:?}", value_type);
1069 else if let Some(struct_type) = value_type.is_struct() {
1070 aggregate_value.access_field(None, struct_type.get_field(idx as i32)).to_rvalue()
1073 panic!("Unexpected type {:?}", value_type);
1077 fn insert_value(&mut self, aggregate_value: RValue<'gcc>, value: RValue<'gcc>, idx: u64) -> RValue<'gcc> {
1078 // FIXME(antoyo): it would be better if the API only called this on struct, not on arrays.
1079 assert_eq!(idx as usize as u64, idx);
1080 let value_type = aggregate_value.get_type();
1083 if value_type.dyncast_array().is_some() {
1084 let index = self.context.new_rvalue_from_long(self.u64_type, i64::try_from(idx).expect("i64::try_from"));
1085 self.context.new_array_access(None, aggregate_value, index)
1087 else if value_type.dyncast_vector().is_some() {
1090 else if let Some(pointer_type) = value_type.get_pointee() {
1091 if let Some(struct_type) = pointer_type.is_struct() {
1092 // NOTE: hack to workaround a limitation of the rustc API: see comment on
1093 // CodegenCx.structs_as_pointer
1094 aggregate_value.dereference_field(None, struct_type.get_field(idx as i32))
1097 panic!("Unexpected type {:?}", value_type);
1101 panic!("Unexpected type {:?}", value_type);
1104 let lvalue_type = lvalue.to_rvalue().get_type();
1106 // NOTE: sometimes, rustc will create a value with the wrong type.
1107 if lvalue_type != value.get_type() {
1108 self.context.new_cast(None, value, lvalue_type)
1114 self.llbb().add_assignment(None, lvalue, value);
1119 fn set_personality_fn(&mut self, _personality: RValue<'gcc>) {
1123 fn cleanup_landing_pad(&mut self, _ty: Type<'gcc>, _pers_fn: RValue<'gcc>) -> RValue<'gcc> {
1124 let field1 = self.context.new_field(None, self.u8_type.make_pointer(), "landing_pad_field_1");
1125 let field2 = self.context.new_field(None, self.i32_type, "landing_pad_field_1");
1126 let struct_type = self.context.new_struct_type(None, "landing_pad", &[field1, field2]);
1127 self.current_func().new_local(None, struct_type.as_type(), "landing_pad")
1129 // TODO(antoyo): Properly implement unwinding.
1130 // the above is just to make the compilation work as it seems
1131 // rustc_codegen_ssa now calls the unwinding builder methods even on panic=abort.
1134 fn resume(&mut self, _exn: RValue<'gcc>) {
1135 // TODO(bjorn3): Properly implement unwinding.
1139 fn cleanup_pad(&mut self, _parent: Option<RValue<'gcc>>, _args: &[RValue<'gcc>]) -> Funclet {
1143 fn cleanup_ret(&mut self, _funclet: &Funclet, _unwind: Option<Block<'gcc>>) {
1147 fn catch_pad(&mut self, _parent: RValue<'gcc>, _args: &[RValue<'gcc>]) -> Funclet {
1153 _parent: Option<RValue<'gcc>>,
1154 _unwind: Option<Block<'gcc>>,
1155 _handlers: &[Block<'gcc>],
1160 // Atomic Operations
1161 fn atomic_cmpxchg(&mut self, dst: RValue<'gcc>, cmp: RValue<'gcc>, src: RValue<'gcc>, order: AtomicOrdering, failure_order: AtomicOrdering, weak: bool) -> RValue<'gcc> {
1162 let expected = self.current_func().new_local(None, cmp.get_type(), "expected");
1163 self.llbb().add_assignment(None, expected, cmp);
1164 let success = self.compare_exchange(dst, expected, src, order, failure_order, weak);
1166 let pair_type = self.cx.type_struct(&[src.get_type(), self.bool_type], false);
1167 let result = self.current_func().new_local(None, pair_type, "atomic_cmpxchg_result");
1168 let align = Align::from_bits(64).expect("align"); // TODO(antoyo): use good align.
1170 let value_type = result.to_rvalue().get_type();
1171 if let Some(struct_type) = value_type.is_struct() {
1172 self.store(success, result.access_field(None, struct_type.get_field(1)).get_address(None), align);
1173 // NOTE: since success contains the call to the intrinsic, it must be stored before
1174 // expected so that we store expected after the call.
1175 self.store(expected.to_rvalue(), result.access_field(None, struct_type.get_field(0)).get_address(None), align);
1177 // TODO(antoyo): handle when value is not a struct.
1182 fn atomic_rmw(&mut self, op: AtomicRmwBinOp, dst: RValue<'gcc>, src: RValue<'gcc>, order: AtomicOrdering) -> RValue<'gcc> {
1183 let size = src.get_type().get_size();
1186 AtomicRmwBinOp::AtomicXchg => format!("__atomic_exchange_{}", size),
1187 AtomicRmwBinOp::AtomicAdd => format!("__atomic_fetch_add_{}", size),
1188 AtomicRmwBinOp::AtomicSub => format!("__atomic_fetch_sub_{}", size),
1189 AtomicRmwBinOp::AtomicAnd => format!("__atomic_fetch_and_{}", size),
1190 AtomicRmwBinOp::AtomicNand => format!("__atomic_fetch_nand_{}", size),
1191 AtomicRmwBinOp::AtomicOr => format!("__atomic_fetch_or_{}", size),
1192 AtomicRmwBinOp::AtomicXor => format!("__atomic_fetch_xor_{}", size),
1193 AtomicRmwBinOp::AtomicMax => return self.atomic_extremum(ExtremumOperation::Max, dst, src, order),
1194 AtomicRmwBinOp::AtomicMin => return self.atomic_extremum(ExtremumOperation::Min, dst, src, order),
1195 AtomicRmwBinOp::AtomicUMax => return self.atomic_extremum(ExtremumOperation::Max, dst, src, order),
1196 AtomicRmwBinOp::AtomicUMin => return self.atomic_extremum(ExtremumOperation::Min, dst, src, order),
1200 let atomic_function = self.context.get_builtin_function(name);
1201 let order = self.context.new_rvalue_from_int(self.i32_type, order.to_gcc());
1203 let void_ptr_type = self.context.new_type::<*mut ()>();
1204 let volatile_void_ptr_type = void_ptr_type.make_volatile();
1205 let dst = self.context.new_cast(None, dst, volatile_void_ptr_type);
1206 // FIXME(antoyo): not sure why, but we have the wrong type here.
1207 let new_src_type = atomic_function.get_param(1).to_rvalue().get_type();
1208 let src = self.context.new_cast(None, src, new_src_type);
1209 let res = self.context.new_call(None, atomic_function, &[dst, src, order]);
1210 self.context.new_cast(None, res, src.get_type())
1213 fn atomic_fence(&mut self, order: AtomicOrdering, scope: SynchronizationScope) {
1216 SynchronizationScope::SingleThread => "__atomic_signal_fence",
1217 SynchronizationScope::CrossThread => "__atomic_thread_fence",
1219 let thread_fence = self.context.get_builtin_function(name);
1220 let order = self.context.new_rvalue_from_int(self.i32_type, order.to_gcc());
1221 self.llbb().add_eval(None, self.context.new_call(None, thread_fence, &[order]));
1224 fn set_invariant_load(&mut self, load: RValue<'gcc>) {
1225 // NOTE: Hack to consider vtable function pointer as non-global-variable function pointer.
1226 self.normal_function_addresses.borrow_mut().insert(load);
1230 fn lifetime_start(&mut self, _ptr: RValue<'gcc>, _size: Size) {
1234 fn lifetime_end(&mut self, _ptr: RValue<'gcc>, _size: Size) {
1241 fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1243 args: &[RValue<'gcc>],
1244 funclet: Option<&Funclet>,
1246 // FIXME(antoyo): remove when having a proper API.
1247 let gcc_func = unsafe { std::mem::transmute(func) };
1248 let call = if self.functions.borrow().values().find(|value| **value == gcc_func).is_some() {
1249 self.function_call(func, args, funclet)
1252 // If it's a not function that was defined, it's a function pointer.
1253 self.function_ptr_call(func, args, funclet)
1255 if let Some(_fn_abi) = fn_abi {
1256 // TODO(bjorn3): Apply function attributes
1261 fn zext(&mut self, value: RValue<'gcc>, dest_typ: Type<'gcc>) -> RValue<'gcc> {
1262 // FIXME(antoyo): this does not zero-extend.
1263 if value.get_type().is_bool() && dest_typ.is_i8(&self.cx) {
1264 // FIXME(antoyo): hack because base::from_immediate converts i1 to i8.
1265 // Fix the code in codegen_ssa::base::from_immediate.
1268 self.gcc_int_cast(value, dest_typ)
1271 fn cx(&self) -> &CodegenCx<'gcc, 'tcx> {
1275 fn do_not_inline(&mut self, _llret: RValue<'gcc>) {
1276 // FIXME(bjorn3): implement
1279 fn set_span(&mut self, _span: Span) {}
1281 fn from_immediate(&mut self, val: Self::Value) -> Self::Value {
1282 if self.cx().val_ty(val) == self.cx().type_i1() {
1283 self.zext(val, self.cx().type_i8())
1290 fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value {
1291 if scalar.is_bool() {
1292 return self.trunc(val, self.cx().type_i1());
1297 fn fptoui_sat(&mut self, val: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> {
1298 self.fptoint_sat(false, val, dest_ty)
1301 fn fptosi_sat(&mut self, val: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> {
1302 self.fptoint_sat(true, val, dest_ty)
1305 fn instrprof_increment(&mut self, _fn_name: RValue<'gcc>, _hash: RValue<'gcc>, _num_counters: RValue<'gcc>, _index: RValue<'gcc>) {
1310 impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
1311 fn fptoint_sat(&mut self, signed: bool, val: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> {
1312 let src_ty = self.cx.val_ty(val);
1313 let (float_ty, int_ty) = if self.cx.type_kind(src_ty) == TypeKind::Vector {
1314 assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty));
1315 (self.cx.element_type(src_ty), self.cx.element_type(dest_ty))
1320 // FIXME(jistone): the following was originally the fallback SSA implementation, before LLVM 13
1321 // added native `fptosi.sat` and `fptoui.sat` conversions, but it was used by GCC as well.
1322 // Now that LLVM always relies on its own, the code has been moved to GCC, but the comments are
1323 // still LLVM-specific. This should be updated, and use better GCC specifics if possible.
1325 let int_width = self.cx.int_width(int_ty);
1326 let float_width = self.cx.float_width(float_ty);
1327 // LLVM's fpto[su]i returns undef when the input val is infinite, NaN, or does not fit into the
1328 // destination integer type after rounding towards zero. This `undef` value can cause UB in
1329 // safe code (see issue #10184), so we implement a saturating conversion on top of it:
1330 // Semantically, the mathematical value of the input is rounded towards zero to the next
1331 // mathematical integer, and then the result is clamped into the range of the destination
1332 // integer type. Positive and negative infinity are mapped to the maximum and minimum value of
1333 // the destination integer type. NaN is mapped to 0.
1335 // Define f_min and f_max as the largest and smallest (finite) floats that are exactly equal to
1336 // a value representable in int_ty.
1337 // They are exactly equal to int_ty::{MIN,MAX} if float_ty has enough significand bits.
1338 // Otherwise, int_ty::MAX must be rounded towards zero, as it is one less than a power of two.
1339 // int_ty::MIN, however, is either zero or a negative power of two and is thus exactly
1340 // representable. Note that this only works if float_ty's exponent range is sufficiently large.
1341 // f16 or 256 bit integers would break this property. Right now the smallest float type is f32
1342 // with exponents ranging up to 127, which is barely enough for i128::MIN = -2^127.
1343 // On the other hand, f_max works even if int_ty::MAX is greater than float_ty::MAX. Because
1344 // we're rounding towards zero, we just get float_ty::MAX (which is always an integer).
1345 // This already happens today with u128::MAX = 2^128 - 1 > f32::MAX.
1346 let int_max = |signed: bool, int_width: u64| -> u128 {
1347 let shift_amount = 128 - int_width;
1348 if signed { i128::MAX as u128 >> shift_amount } else { u128::MAX >> shift_amount }
1350 let int_min = |signed: bool, int_width: u64| -> i128 {
1351 if signed { i128::MIN >> (128 - int_width) } else { 0 }
1354 let compute_clamp_bounds_single = |signed: bool, int_width: u64| -> (u128, u128) {
1356 ieee::Single::from_i128_r(int_min(signed, int_width), Round::TowardZero);
1357 assert_eq!(rounded_min.status, Status::OK);
1359 ieee::Single::from_u128_r(int_max(signed, int_width), Round::TowardZero);
1360 assert!(rounded_max.value.is_finite());
1361 (rounded_min.value.to_bits(), rounded_max.value.to_bits())
1363 let compute_clamp_bounds_double = |signed: bool, int_width: u64| -> (u128, u128) {
1365 ieee::Double::from_i128_r(int_min(signed, int_width), Round::TowardZero);
1366 assert_eq!(rounded_min.status, Status::OK);
1368 ieee::Double::from_u128_r(int_max(signed, int_width), Round::TowardZero);
1369 assert!(rounded_max.value.is_finite());
1370 (rounded_min.value.to_bits(), rounded_max.value.to_bits())
1372 // To implement saturation, we perform the following steps:
1374 // 1. Cast val to an integer with fpto[su]i. This may result in undef.
1375 // 2. Compare val to f_min and f_max, and use the comparison results to select:
1376 // a) int_ty::MIN if val < f_min or val is NaN
1377 // b) int_ty::MAX if val > f_max
1378 // c) the result of fpto[su]i otherwise
1379 // 3. If val is NaN, return 0.0, otherwise return the result of step 2.
1381 // This avoids resulting undef because values in range [f_min, f_max] by definition fit into the
1382 // destination type. It creates an undef temporary, but *producing* undef is not UB. Our use of
1383 // undef does not introduce any non-determinism either.
1384 // More importantly, the above procedure correctly implements saturating conversion.
1386 // If val is NaN, 0 is returned by definition.
1387 // Otherwise, val is finite or infinite and thus can be compared with f_min and f_max.
1388 // This yields three cases to consider:
1389 // (1) if val in [f_min, f_max], the result of fpto[su]i is returned, which agrees with
1390 // saturating conversion for inputs in that range.
1391 // (2) if val > f_max, then val is larger than int_ty::MAX. This holds even if f_max is rounded
1392 // (i.e., if f_max < int_ty::MAX) because in those cases, nextUp(f_max) is already larger
1393 // than int_ty::MAX. Because val is larger than int_ty::MAX, the return value of int_ty::MAX
1395 // (3) if val < f_min, then val is smaller than int_ty::MIN. As shown earlier, f_min exactly equals
1396 // int_ty::MIN and therefore the return value of int_ty::MIN is correct.
1399 let float_bits_to_llval = |bx: &mut Self, bits| {
1400 let bits_llval = match float_width {
1401 32 => bx.cx().const_u32(bits as u32),
1402 64 => bx.cx().const_u64(bits as u64),
1403 n => bug!("unsupported float width {}", n),
1405 bx.bitcast(bits_llval, float_ty)
1407 let (f_min, f_max) = match float_width {
1408 32 => compute_clamp_bounds_single(signed, int_width),
1409 64 => compute_clamp_bounds_double(signed, int_width),
1410 n => bug!("unsupported float width {}", n),
1412 let f_min = float_bits_to_llval(self, f_min);
1413 let f_max = float_bits_to_llval(self, f_max);
1414 let int_max = self.cx.const_uint_big(int_ty, int_max(signed, int_width));
1415 let int_min = self.cx.const_uint_big(int_ty, int_min(signed, int_width) as u128);
1416 let zero = self.cx.const_uint(int_ty, 0);
1418 // If we're working with vectors, constants must be "splatted": the constant is duplicated
1419 // into each lane of the vector. The algorithm stays the same, we are just using the
1420 // same constant across all lanes.
1421 let maybe_splat = |bx: &mut Self, val| {
1422 if bx.cx().type_kind(dest_ty) == TypeKind::Vector {
1423 bx.vector_splat(bx.vector_length(dest_ty), val)
1428 let f_min = maybe_splat(self, f_min);
1429 let f_max = maybe_splat(self, f_max);
1430 let int_max = maybe_splat(self, int_max);
1431 let int_min = maybe_splat(self, int_min);
1432 let zero = maybe_splat(self, zero);
1435 let fptosui_result = if signed { self.fptosi(val, dest_ty) } else { self.fptoui(val, dest_ty) };
1436 let less_or_nan = self.fcmp(RealPredicate::RealULT, val, f_min);
1437 let greater = self.fcmp(RealPredicate::RealOGT, val, f_max);
1439 // Step 2: We use two comparisons and two selects, with %s1 being the
1441 // %less_or_nan = fcmp ult %val, %f_min
1442 // %greater = fcmp olt %val, %f_max
1443 // %s0 = select %less_or_nan, int_ty::MIN, %fptosi_result
1444 // %s1 = select %greater, int_ty::MAX, %s0
1445 // Note that %less_or_nan uses an *unordered* comparison. This
1446 // comparison is true if the operands are not comparable (i.e., if val is
1447 // NaN). The unordered comparison ensures that s1 becomes int_ty::MIN if
1450 // Performance note: Unordered comparison can be lowered to a "flipped"
1451 // comparison and a negation, and the negation can be merged into the
1452 // select. Therefore, it not necessarily any more expensive than an
1453 // ordered ("normal") comparison. Whether these optimizations will be
1454 // performed is ultimately up to the backend, but at least x86 does
1456 let s0 = self.select(less_or_nan, int_min, fptosui_result);
1457 let s1 = self.select(greater, int_max, s0);
1459 // Step 3: NaN replacement.
1460 // For unsigned types, the above step already yielded int_ty::MIN == 0 if val is NaN.
1461 // Therefore we only need to execute this step for signed integer types.
1463 // LLVM has no isNaN predicate, so we use (val == val) instead
1464 let cmp = self.fcmp(RealPredicate::RealOEQ, val, val);
1465 self.select(cmp, s1, zero)
1471 #[cfg(feature="master")]
1472 pub fn shuffle_vector(&mut self, v1: RValue<'gcc>, v2: RValue<'gcc>, mask: RValue<'gcc>) -> RValue<'gcc> {
1473 let struct_type = mask.get_type().is_struct().expect("mask of struct type");
1475 // TODO(antoyo): use a recursive unqualified() here.
1476 let vector_type = v1.get_type().unqualified().dyncast_vector().expect("vector type");
1477 let element_type = vector_type.get_element_type();
1478 let vec_num_units = vector_type.get_num_units();
1480 let mask_num_units = struct_type.get_field_count();
1481 let mut vector_elements = vec![];
1482 let mask_element_type =
1483 if element_type.is_integral() {
1487 #[cfg(feature="master")]
1489 self.cx.type_ix(element_type.get_size() as u64 * 8)
1491 #[cfg(not(feature="master"))]
1494 for i in 0..mask_num_units {
1495 let field = struct_type.get_field(i as i32);
1496 vector_elements.push(self.context.new_cast(None, mask.access_field(None, field).to_rvalue(), mask_element_type));
1499 // NOTE: the mask needs to be the same length as the input vectors, so add the missing
1500 // elements in the mask if needed.
1501 for _ in mask_num_units..vec_num_units {
1502 vector_elements.push(self.context.new_rvalue_zero(mask_element_type));
1505 let array_type = self.context.new_array_type(None, element_type, vec_num_units as i32);
1506 let result_type = self.context.new_vector_type(element_type, mask_num_units as u64);
1508 if vec_num_units < mask_num_units {
1509 // NOTE: the mask needs to be the same length as the input vectors, so join the 2
1510 // vectors and create a dummy second vector.
1511 // TODO(antoyo): switch to using new_vector_access.
1512 let array = self.context.new_bitcast(None, v1, array_type);
1513 let mut elements = vec![];
1514 for i in 0..vec_num_units {
1515 elements.push(self.context.new_array_access(None, array, self.context.new_rvalue_from_int(self.int_type, i as i32)).to_rvalue());
1517 // TODO(antoyo): switch to using new_vector_access.
1518 let array = self.context.new_bitcast(None, v2, array_type);
1519 for i in 0..(mask_num_units - vec_num_units) {
1520 elements.push(self.context.new_array_access(None, array, self.context.new_rvalue_from_int(self.int_type, i as i32)).to_rvalue());
1522 let v1 = self.context.new_rvalue_from_vector(None, result_type, &elements);
1523 let zero = self.context.new_rvalue_zero(element_type);
1524 let v2 = self.context.new_rvalue_from_vector(None, result_type, &vec![zero; mask_num_units]);
1531 let new_mask_num_units = std::cmp::max(mask_num_units, vec_num_units);
1532 let mask_type = self.context.new_vector_type(mask_element_type, new_mask_num_units as u64);
1533 let mask = self.context.new_rvalue_from_vector(None, mask_type, &vector_elements);
1534 let result = self.context.new_rvalue_vector_perm(None, v1, v2, mask);
1536 if vec_num_units != mask_num_units {
1537 // NOTE: if padding was added, only select the number of elements of the masks to
1538 // remove that padding in the result.
1539 let mut elements = vec![];
1540 // TODO(antoyo): switch to using new_vector_access.
1541 let array = self.context.new_bitcast(None, result, array_type);
1542 for i in 0..mask_num_units {
1543 elements.push(self.context.new_array_access(None, array, self.context.new_rvalue_from_int(self.int_type, i as i32)).to_rvalue());
1545 self.context.new_rvalue_from_vector(None, result_type, &elements)
1552 #[cfg(not(feature="master"))]
1553 pub fn shuffle_vector(&mut self, _v1: RValue<'gcc>, _v2: RValue<'gcc>, _mask: RValue<'gcc>) -> RValue<'gcc> {
1557 #[cfg(feature="master")]
1558 pub fn vector_reduce<F>(&mut self, src: RValue<'gcc>, op: F) -> RValue<'gcc>
1559 where F: Fn(RValue<'gcc>, RValue<'gcc>, &'gcc Context<'gcc>) -> RValue<'gcc>
1561 let vector_type = src.get_type().unqualified().dyncast_vector().expect("vector type");
1562 let element_count = vector_type.get_num_units();
1563 let mut vector_elements = vec![];
1564 for i in 0..element_count {
1565 vector_elements.push(i);
1567 let mask_type = self.context.new_vector_type(self.int_type, element_count as u64);
1570 while shift < element_count {
1571 let vector_elements: Vec<_> =
1572 vector_elements.iter()
1573 .map(|i| self.context.new_rvalue_from_int(self.int_type, ((i + shift) % element_count) as i32))
1575 let mask = self.context.new_rvalue_from_vector(None, mask_type, &vector_elements);
1576 let shifted = self.context.new_rvalue_vector_perm(None, res, res, mask);
1578 res = op(res, shifted, &self.context);
1580 self.context.new_vector_access(None, res, self.context.new_rvalue_zero(self.int_type))
1584 #[cfg(not(feature="master"))]
1585 pub fn vector_reduce<F>(&mut self, src: RValue<'gcc>, op: F) -> RValue<'gcc>
1586 where F: Fn(RValue<'gcc>, RValue<'gcc>, &'gcc Context<'gcc>) -> RValue<'gcc>
1591 pub fn vector_reduce_op(&mut self, src: RValue<'gcc>, op: BinaryOp) -> RValue<'gcc> {
1592 self.vector_reduce(src, |a, b, context| context.new_binary_op(None, op, a.get_type(), a, b))
1595 pub fn vector_reduce_fadd_fast(&mut self, _acc: RValue<'gcc>, _src: RValue<'gcc>) -> RValue<'gcc> {
1599 pub fn vector_reduce_fmul_fast(&mut self, _acc: RValue<'gcc>, _src: RValue<'gcc>) -> RValue<'gcc> {
1603 // Inspired by Hacker's Delight min implementation.
1604 pub fn vector_reduce_min(&mut self, src: RValue<'gcc>) -> RValue<'gcc> {
1605 self.vector_reduce(src, |a, b, context| {
1606 let differences_or_zeros = difference_or_zero(a, b, context);
1607 context.new_binary_op(None, BinaryOp::Minus, a.get_type(), a, differences_or_zeros)
1611 // Inspired by Hacker's Delight max implementation.
1612 pub fn vector_reduce_max(&mut self, src: RValue<'gcc>) -> RValue<'gcc> {
1613 self.vector_reduce(src, |a, b, context| {
1614 let differences_or_zeros = difference_or_zero(a, b, context);
1615 context.new_binary_op(None, BinaryOp::Plus, b.get_type(), b, differences_or_zeros)
1619 pub fn vector_select(&mut self, cond: RValue<'gcc>, then_val: RValue<'gcc>, else_val: RValue<'gcc>) -> RValue<'gcc> {
1620 // cond is a vector of integers, not of bools.
1621 let cond_type = cond.get_type();
1622 let vector_type = cond_type.unqualified().dyncast_vector().expect("vector type");
1623 let num_units = vector_type.get_num_units();
1624 let element_type = vector_type.get_element_type();
1625 let zeros = vec![self.context.new_rvalue_zero(element_type); num_units];
1626 let zeros = self.context.new_rvalue_from_vector(None, cond_type, &zeros);
1628 let masks = self.context.new_comparison(None, ComparisonOp::NotEquals, cond, zeros);
1629 let then_vals = masks & then_val;
1631 let ones = vec![self.context.new_rvalue_one(element_type); num_units];
1632 let ones = self.context.new_rvalue_from_vector(None, cond_type, &ones);
1633 let inverted_masks = masks + ones;
1634 // NOTE: sometimes, the type of else_val can be different than the type of then_val in
1635 // libgccjit (vector of int vs vector of int32_t), but they should be the same for the AND
1636 // operation to work.
1637 let else_val = self.context.new_bitcast(None, else_val, then_val.get_type());
1638 let else_vals = inverted_masks & else_val;
1640 then_vals | else_vals
1644 fn difference_or_zero<'gcc>(a: RValue<'gcc>, b: RValue<'gcc>, context: &'gcc Context<'gcc>) -> RValue<'gcc> {
1645 let difference = a - b;
1646 let masks = context.new_comparison(None, ComparisonOp::GreaterThanEquals, b, a);
1650 impl<'a, 'gcc, 'tcx> StaticBuilderMethods for Builder<'a, 'gcc, 'tcx> {
1651 fn get_static(&mut self, def_id: DefId) -> RValue<'gcc> {
1652 // Forward to the `get_static` method of `CodegenCx`
1653 self.cx().get_static(def_id).get_address(None)
1657 impl<'tcx> HasParamEnv<'tcx> for Builder<'_, '_, 'tcx> {
1658 fn param_env(&self) -> ParamEnv<'tcx> {
1663 impl<'tcx> HasTargetSpec for Builder<'_, '_, 'tcx> {
1664 fn target_spec(&self) -> &Target {
1665 &self.cx.target_spec()
1669 pub trait ToGccComp {
1670 fn to_gcc_comparison(&self) -> ComparisonOp;
1673 impl ToGccComp for IntPredicate {
1674 fn to_gcc_comparison(&self) -> ComparisonOp {
1676 IntPredicate::IntEQ => ComparisonOp::Equals,
1677 IntPredicate::IntNE => ComparisonOp::NotEquals,
1678 IntPredicate::IntUGT => ComparisonOp::GreaterThan,
1679 IntPredicate::IntUGE => ComparisonOp::GreaterThanEquals,
1680 IntPredicate::IntULT => ComparisonOp::LessThan,
1681 IntPredicate::IntULE => ComparisonOp::LessThanEquals,
1682 IntPredicate::IntSGT => ComparisonOp::GreaterThan,
1683 IntPredicate::IntSGE => ComparisonOp::GreaterThanEquals,
1684 IntPredicate::IntSLT => ComparisonOp::LessThan,
1685 IntPredicate::IntSLE => ComparisonOp::LessThanEquals,
1690 impl ToGccComp for RealPredicate {
1691 fn to_gcc_comparison(&self) -> ComparisonOp {
1692 // TODO(antoyo): check that ordered vs non-ordered is respected.
1694 RealPredicate::RealPredicateFalse => unreachable!(),
1695 RealPredicate::RealOEQ => ComparisonOp::Equals,
1696 RealPredicate::RealOGT => ComparisonOp::GreaterThan,
1697 RealPredicate::RealOGE => ComparisonOp::GreaterThanEquals,
1698 RealPredicate::RealOLT => ComparisonOp::LessThan,
1699 RealPredicate::RealOLE => ComparisonOp::LessThanEquals,
1700 RealPredicate::RealONE => ComparisonOp::NotEquals,
1701 RealPredicate::RealORD => unreachable!(),
1702 RealPredicate::RealUNO => unreachable!(),
1703 RealPredicate::RealUEQ => ComparisonOp::Equals,
1704 RealPredicate::RealUGT => ComparisonOp::GreaterThan,
1705 RealPredicate::RealUGE => ComparisonOp::GreaterThan,
1706 RealPredicate::RealULT => ComparisonOp::LessThan,
1707 RealPredicate::RealULE => ComparisonOp::LessThan,
1708 RealPredicate::RealUNE => ComparisonOp::NotEquals,
1709 RealPredicate::RealPredicateTrue => unreachable!(),
1715 #[allow(non_camel_case_types)]
1725 trait ToGccOrdering {
1726 fn to_gcc(self) -> i32;
1729 impl ToGccOrdering for AtomicOrdering {
1730 fn to_gcc(self) -> i32 {
1735 AtomicOrdering::Unordered => __ATOMIC_RELAXED,
1736 AtomicOrdering::Relaxed => __ATOMIC_RELAXED, // TODO(antoyo): check if that's the same.
1737 AtomicOrdering::Acquire => __ATOMIC_ACQUIRE,
1738 AtomicOrdering::Release => __ATOMIC_RELEASE,
1739 AtomicOrdering::AcquireRelease => __ATOMIC_ACQ_REL,
1740 AtomicOrdering::SequentiallyConsistent => __ATOMIC_SEQ_CST,