use rustc_target::abi::{VariantIdx, Variants};
use super::{
- from_known_layout, mir_assign_valid_types, ConstValue, GlobalId, InterpCx, InterpResult,
- MPlaceTy, Machine, MemPlace, Place, PlaceTy, Pointer, Scalar, ScalarMaybeUninit,
+ alloc_range, from_known_layout, mir_assign_valid_types, AllocId, ConstValue, GlobalId,
+ InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, Place, PlaceTy, Pointer, Provenance,
+ Scalar, ScalarMaybeUninit,
};
/// An `Immediate` represents a single immediate self-contained Rust value.
/// operations and wide pointers. This idea was taken from rustc's codegen.
/// In particular, thanks to `ScalarPair`, arithmetic operations and casts can be entirely
/// defined on `Immediate`, and do not have to work with a `Place`.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, HashStable, Hash)]
-pub enum Immediate<Tag = ()> {
+#[derive(Copy, Clone, PartialEq, Eq, HashStable, Hash, Debug)]
+pub enum Immediate<Tag: Provenance = AllocId> {
Scalar(ScalarMaybeUninit<Tag>),
ScalarPair(ScalarMaybeUninit<Tag>, ScalarMaybeUninit<Tag>),
}
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
rustc_data_structures::static_assert_size!(Immediate, 56);
-impl<Tag> From<ScalarMaybeUninit<Tag>> for Immediate<Tag> {
+impl<Tag: Provenance> From<ScalarMaybeUninit<Tag>> for Immediate<Tag> {
#[inline(always)]
fn from(val: ScalarMaybeUninit<Tag>) -> Self {
Immediate::Scalar(val)
}
}
-impl<Tag> From<Scalar<Tag>> for Immediate<Tag> {
+impl<Tag: Provenance> From<Scalar<Tag>> for Immediate<Tag> {
#[inline(always)]
fn from(val: Scalar<Tag>) -> Self {
Immediate::Scalar(val.into())
}
}
-impl<Tag> From<Pointer<Tag>> for Immediate<Tag> {
- #[inline(always)]
- fn from(val: Pointer<Tag>) -> Self {
- Immediate::Scalar(Scalar::from(val).into())
+impl<'tcx, Tag: Provenance> Immediate<Tag> {
+ pub fn from_pointer(p: Pointer<Tag>, cx: &impl HasDataLayout) -> Self {
+ Immediate::Scalar(ScalarMaybeUninit::from_pointer(p, cx))
+ }
+
+ pub fn from_maybe_pointer(p: Pointer<Option<Tag>>, cx: &impl HasDataLayout) -> Self {
+ Immediate::Scalar(ScalarMaybeUninit::from_maybe_pointer(p, cx))
}
-}
-impl<'tcx, Tag> Immediate<Tag> {
pub fn new_slice(val: Scalar<Tag>, len: u64, cx: &impl HasDataLayout) -> Self {
Immediate::ScalarPair(val.into(), Scalar::from_machine_usize(len, cx).into())
}
- pub fn new_dyn_trait(val: Scalar<Tag>, vtable: Pointer<Tag>) -> Self {
- Immediate::ScalarPair(val.into(), vtable.into())
+ pub fn new_dyn_trait(val: Scalar<Tag>, vtable: Pointer<Tag>, cx: &impl HasDataLayout) -> Self {
+ Immediate::ScalarPair(val.into(), ScalarMaybeUninit::from_pointer(vtable, cx))
}
#[inline]
// ScalarPair needs a type to interpret, so we often have an immediate and a type together
// as input for binary and cast operations.
#[derive(Copy, Clone, Debug)]
-pub struct ImmTy<'tcx, Tag = ()> {
+pub struct ImmTy<'tcx, Tag: Provenance = AllocId> {
imm: Immediate<Tag>,
pub layout: TyAndLayout<'tcx>,
}
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
rustc_data_structures::static_assert_size!(ImmTy<'_>, 72);
-impl<Tag: Copy> std::fmt::Display for ImmTy<'tcx, Tag> {
+impl<Tag: Provenance> std::fmt::Display for ImmTy<'tcx, Tag> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
/// Helper function for printing a scalar to a FmtPrinter
- fn p<'a, 'tcx, F: std::fmt::Write, Tag>(
+ fn p<'a, 'tcx, F: std::fmt::Write, Tag: Provenance>(
cx: FmtPrinter<'a, 'tcx, F>,
s: ScalarMaybeUninit<Tag>,
ty: Ty<'tcx>,
) -> Result<FmtPrinter<'a, 'tcx, F>, std::fmt::Error> {
match s {
- ScalarMaybeUninit::Scalar(s) => {
- cx.pretty_print_const_scalar(s.erase_tag(), ty, true)
+ ScalarMaybeUninit::Scalar(Scalar::Int(int)) => {
+ cx.pretty_print_const_scalar_int(int, ty, true)
+ }
+ ScalarMaybeUninit::Scalar(Scalar::Ptr(ptr, _sz)) => {
+ // Just print the ptr value. `pretty_print_const_scalar_ptr` would also try to
+ // print what is points to, which would fail since it has no access to the local
+ // memory.
+ cx.pretty_print_const_pointer(ptr, ty, true)
}
ScalarMaybeUninit::Uninit => cx.typed_value(
|mut this| {
p(cx, s, ty)?;
return Ok(());
}
- write!(f, "{}: {}", s.erase_tag(), self.layout.ty)
+ write!(f, "{}: {}", s, self.layout.ty)
}
Immediate::ScalarPair(a, b) => {
// FIXME(oli-obk): at least print tuples and slices nicely
- write!(f, "({}, {}): {}", a.erase_tag(), b.erase_tag(), self.layout.ty,)
+ write!(f, "({}, {}): {}", a, b, self.layout.ty,)
}
}
})
}
}
-impl<'tcx, Tag> std::ops::Deref for ImmTy<'tcx, Tag> {
+impl<'tcx, Tag: Provenance> std::ops::Deref for ImmTy<'tcx, Tag> {
type Target = Immediate<Tag>;
#[inline(always)]
fn deref(&self) -> &Immediate<Tag> {
/// An `Operand` is the result of computing a `mir::Operand`. It can be immediate,
/// or still in memory. The latter is an optimization, to delay reading that chunk of
/// memory and to avoid having to store arbitrary-sized data here.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, HashStable, Hash)]
-pub enum Operand<Tag = ()> {
+#[derive(Copy, Clone, PartialEq, Eq, HashStable, Hash, Debug)]
+pub enum Operand<Tag: Provenance = AllocId> {
Immediate(Immediate<Tag>),
Indirect(MemPlace<Tag>),
}
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
-pub struct OpTy<'tcx, Tag = ()> {
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
+pub struct OpTy<'tcx, Tag: Provenance = AllocId> {
op: Operand<Tag>, // Keep this private; it helps enforce invariants.
pub layout: TyAndLayout<'tcx>,
}
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-rustc_data_structures::static_assert_size!(OpTy<'_, ()>, 80);
+rustc_data_structures::static_assert_size!(OpTy<'_>, 80);
-impl<'tcx, Tag> std::ops::Deref for OpTy<'tcx, Tag> {
+impl<'tcx, Tag: Provenance> std::ops::Deref for OpTy<'tcx, Tag> {
type Target = Operand<Tag>;
#[inline(always)]
fn deref(&self) -> &Operand<Tag> {
}
}
-impl<'tcx, Tag: Copy> From<MPlaceTy<'tcx, Tag>> for OpTy<'tcx, Tag> {
+impl<'tcx, Tag: Provenance> From<MPlaceTy<'tcx, Tag>> for OpTy<'tcx, Tag> {
#[inline(always)]
fn from(mplace: MPlaceTy<'tcx, Tag>) -> Self {
OpTy { op: Operand::Indirect(*mplace), layout: mplace.layout }
}
}
-impl<'tcx, Tag: Copy> From<&'_ MPlaceTy<'tcx, Tag>> for OpTy<'tcx, Tag> {
+impl<'tcx, Tag: Provenance> From<&'_ MPlaceTy<'tcx, Tag>> for OpTy<'tcx, Tag> {
#[inline(always)]
fn from(mplace: &MPlaceTy<'tcx, Tag>) -> Self {
OpTy { op: Operand::Indirect(**mplace), layout: mplace.layout }
}
}
-impl<'tcx, Tag> From<ImmTy<'tcx, Tag>> for OpTy<'tcx, Tag> {
+impl<'tcx, Tag: Provenance> From<ImmTy<'tcx, Tag>> for OpTy<'tcx, Tag> {
#[inline(always)]
fn from(val: ImmTy<'tcx, Tag>) -> Self {
OpTy { op: Operand::Immediate(val.imm), layout: val.layout }
}
}
-impl<'tcx, Tag: Copy> ImmTy<'tcx, Tag> {
+impl<'tcx, Tag: Provenance> ImmTy<'tcx, Tag> {
#[inline]
pub fn from_scalar(val: Scalar<Tag>, layout: TyAndLayout<'tcx>) -> Self {
ImmTy { imm: val.into(), layout }
}
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
- /// Normalize `place.ptr` to a `Pointer` if this is a place and not a ZST.
- /// Can be helpful to avoid lots of `force_ptr` calls later, if this place is used a lot.
- #[inline]
- pub fn force_op_ptr(
- &self,
- op: &OpTy<'tcx, M::PointerTag>,
- ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> {
- match op.try_as_mplace(self) {
- Ok(mplace) => Ok(self.force_mplace_ptr(mplace)?.into()),
- Err(imm) => Ok(imm.into()), // Nothing to cast/force
- }
- }
-
/// Try reading an immediate in memory; this is interesting particularly for `ScalarPair`.
/// Returns `None` if the layout does not permit loading this as a value.
fn try_read_immediate_from_mplace(
return Ok(None);
}
- let ptr = match self
- .check_mplace_access(mplace, None)
- .expect("places should be checked on creation")
- {
+ let alloc = match self.get_alloc(mplace)? {
Some(ptr) => ptr,
None => {
- if let Scalar::Ptr(ptr) = mplace.ptr {
- // We may be reading from a static.
- // In order to ensure that `static FOO: Type = FOO;` causes a cycle error
- // instead of magically pulling *any* ZST value from the ether, we need to
- // actually access the referenced allocation.
- self.memory.get_raw(ptr.alloc_id)?;
- }
return Ok(Some(ImmTy {
// zero-sized type
imm: Scalar::ZST.into(),
}
};
- let alloc = self.memory.get_raw(ptr.alloc_id)?;
-
match mplace.layout.abi {
Abi::Scalar(..) => {
- let scalar = alloc.read_scalar(self, ptr, mplace.layout.size)?;
+ let scalar = alloc.read_scalar(alloc_range(Size::ZERO, mplace.layout.size))?;
Ok(Some(ImmTy { imm: scalar.into(), layout: mplace.layout }))
}
Abi::ScalarPair(ref a, ref b) => {
// which `ptr.offset(b_offset)` cannot possibly fail to satisfy.
let (a, b) = (&a.value, &b.value);
let (a_size, b_size) = (a.size(self), b.size(self));
- let a_ptr = ptr;
let b_offset = a_size.align_to(b.align(self).abi);
assert!(b_offset.bytes() > 0); // we later use the offset to tell apart the fields
- let b_ptr = ptr.offset(b_offset, self)?;
- let a_val = alloc.read_scalar(self, a_ptr, a_size)?;
- let b_val = alloc.read_scalar(self, b_ptr, b_size)?;
+ let a_val = alloc.read_scalar(alloc_range(Size::ZERO, a_size))?;
+ let b_val = alloc.read_scalar(alloc_range(b_offset, b_size))?;
Ok(Some(ImmTy { imm: Immediate::ScalarPair(a_val, b_val), layout: mplace.layout }))
}
_ => Ok(None),
&self,
src: &OpTy<'tcx, M::PointerTag>,
) -> InterpResult<'tcx, Result<ImmTy<'tcx, M::PointerTag>, MPlaceTy<'tcx, M::PointerTag>>> {
- Ok(match src.try_as_mplace(self) {
+ Ok(match src.try_as_mplace() {
Ok(ref mplace) => {
if let Some(val) = self.try_read_immediate_from_mplace(mplace)? {
Ok(val)
Ok(self.read_immediate(op)?.to_scalar_or_uninit())
}
+ /// Read a pointer from a place.
+ pub fn read_pointer(
+ &self,
+ op: &OpTy<'tcx, M::PointerTag>,
+ ) -> InterpResult<'tcx, Pointer<Option<M::PointerTag>>> {
+ Ok(self.scalar_to_ptr(self.read_scalar(op)?.check_init()?))
+ }
+
// Turn the wide MPlace into a string (must already be dereferenced!)
pub fn read_str(&self, mplace: &MPlaceTy<'tcx, M::PointerTag>) -> InterpResult<'tcx, &str> {
let len = mplace.len(self)?;
op: &OpTy<'tcx, M::PointerTag>,
field: usize,
) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> {
- let base = match op.try_as_mplace(self) {
+ let base = match op.try_as_mplace() {
Ok(ref mplace) => {
// We can reuse the mplace field computation logic for indirect operands.
let field = self.mplace_field(mplace, field)?;
self.operand_field(op, index)
} else {
// Indexing into a big array. This must be an mplace.
- let mplace = op.assert_mem_place(self);
+ let mplace = op.assert_mem_place();
Ok(self.mplace_index(&mplace, index)?.into())
}
}
variant: VariantIdx,
) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> {
// Downcasts only change the layout
- Ok(match op.try_as_mplace(self) {
+ Ok(match op.try_as_mplace() {
Ok(ref mplace) => self.mplace_downcast(mplace, variant)?.into(),
Err(..) => {
let layout = op.layout.for_variant(self, variant);
Subslice { .. } | ConstantIndex { .. } | Index(_) => {
// The rest should only occur as mplace, we do not use Immediates for types
// allowing such operations. This matches place_projection forcing an allocation.
- let mplace = base.assert_mem_place(self);
+ let mplace = base.assert_mem_place();
self.mplace_projection(&mplace, proj_elem)?.into()
}
})
// Other cases need layout.
let tag_scalar = |scalar| -> InterpResult<'tcx, _> {
Ok(match scalar {
- Scalar::Ptr(ptr) => Scalar::Ptr(self.global_base_pointer(ptr)?),
+ Scalar::Ptr(ptr, size) => Scalar::Ptr(self.global_base_pointer(ptr)?, size),
Scalar::Int(int) => Scalar::Int(int),
})
};
// We rely on mutability being set correctly in that allocation to prevent writes
// where none should happen.
let ptr = self.global_base_pointer(Pointer::new(id, offset))?;
- Operand::Indirect(MemPlace::from_ptr(ptr, layout.align.abi))
+ Operand::Indirect(MemPlace::from_ptr(ptr.into(), layout.align.abi))
}
- ConstValue::Scalar(x) => Operand::Immediate(tag_scalar(x)?.into()),
+ ConstValue::Scalar(x) => Operand::Immediate(tag_scalar(x.into())?.into()),
ConstValue::Slice { data, start, end } => {
// We rely on mutability being set correctly in `data` to prevent writes
// where none should happen.
Size::from_bytes(start), // offset: `start`
);
Operand::Immediate(Immediate::new_slice(
- self.global_base_pointer(ptr)?.into(),
+ Scalar::from_pointer(self.global_base_pointer(ptr)?, &*self.tcx),
u64::try_from(end.checked_sub(start).unwrap()).unwrap(), // len: `end - start`
self,
))
// Figure out which discriminant and variant this corresponds to.
Ok(match *tag_encoding {
TagEncoding::Direct => {
- let tag_bits = self
- .force_bits(tag_val, tag_layout.size)
- .map_err(|_| err_ub!(InvalidTag(tag_val.erase_tag())))?;
+ let tag_bits = tag_val
+ .try_to_int()
+ .map_err(|dbg_val| err_ub!(InvalidTag(dbg_val)))?
+ .assert_bits(tag_layout.size);
// Cast bits from tag layout to discriminant layout.
let discr_val = self.cast_from_scalar(tag_bits, tag_layout, discr_layout.ty);
let discr_bits = discr_val.assert_bits(discr_layout.size);
}
_ => span_bug!(self.cur_span(), "tagged layout for non-adt non-generator"),
}
- .ok_or_else(|| err_ub!(InvalidTag(tag_val.erase_tag())))?;
+ .ok_or_else(|| err_ub!(InvalidTag(Scalar::from_uint(tag_bits, tag_layout.size))))?;
// Return the cast value, and the index.
(discr_val, index.0)
}
// discriminant (encoded in niche/tag) and variant index are the same.
let variants_start = niche_variants.start().as_u32();
let variants_end = niche_variants.end().as_u32();
- let variant = match tag_val.to_bits_or_ptr(tag_layout.size, self) {
- Err(ptr) => {
- // The niche must be just 0 (which an inbounds pointer value never is)
+ let variant = match tag_val.try_to_int() {
+ Err(dbg_val) => {
+ // So this is a pointer then, and casting to an int failed.
+ // Can only happen during CTFE.
+ let ptr = self.scalar_to_ptr(tag_val);
+ // The niche must be just 0, and the ptr not null, then we know this is
+ // okay. Everything else, we conservatively reject.
let ptr_valid = niche_start == 0
&& variants_start == variants_end
&& !self.memory.ptr_may_be_null(ptr);
if !ptr_valid {
- throw_ub!(InvalidTag(tag_val.erase_tag()))
+ throw_ub!(InvalidTag(dbg_val))
}
dataful_variant
}
Ok(tag_bits) => {
+ let tag_bits = tag_bits.assert_bits(tag_layout.size);
// We need to use machine arithmetic to get the relative variant idx:
// variant_index_relative = tag_val - niche_start_val
let tag_val = ImmTy::from_uint(tag_bits, tag_layout);
projects / rust.git / blobdiff