diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index b0a1dedd646b0..88b48cc61e7ab 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -1279,6 +1279,17 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
) -> MergingSucc {
debug!("codegen_terminator: {:?}", terminator);
+ if bx.tcx().may_insert_niche_checks() {
+ if let mir::TerminatorKind::Return = terminator.kind {
+ let op = mir::Operand::Copy(mir::Place::return_place());
+ let ty = op.ty(self.mir, bx.tcx());
+ let ty = self.monomorphize(ty);
+ if let Some(niche) = bx.layout_of(ty).largest_niche {
+ self.codegen_niche_check(bx, op, niche, terminator.source_info);
+ }
+ }
+ }
+
let helper = TerminatorCodegenHelper { bb, terminator };
let mergeable_succ = || {
@@ -1583,7 +1594,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
tuple.layout.fields.count()
}
- fn get_caller_location(
+ pub fn get_caller_location(
&mut self,
bx: &mut Bx,
source_info: mir::SourceInfo,
diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs
index 62f69af3f2f75..3cc94f36ddfc3 100644
--- a/compiler/rustc_codegen_ssa/src/mir/mod.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs
@@ -21,6 +21,7 @@ pub mod debuginfo;
mod intrinsic;
mod locals;
mod naked_asm;
+mod niche_check;
pub mod operand;
pub mod place;
mod rvalue;
diff --git a/compiler/rustc_codegen_ssa/src/mir/niche_check.rs b/compiler/rustc_codegen_ssa/src/mir/niche_check.rs
new file mode 100644
index 0000000000000..9a7fc102abb2b
--- /dev/null
+++ b/compiler/rustc_codegen_ssa/src/mir/niche_check.rs
@@ -0,0 +1,286 @@
+use rustc_abi::BackendRepr;
+use rustc_hir::LangItem;
+use rustc_middle::mir;
+use rustc_middle::mir::visit::{NonMutatingUseContext, PlaceContext, Visitor};
+use rustc_middle::ty::{Mutability, Ty, TyCtxt};
+use rustc_span::Span;
+use rustc_span::def_id::LOCAL_CRATE;
+use rustc_target::abi::{Float, Integer, Niche, Primitive, Scalar, Size, WrappingRange};
+use tracing::instrument;
+
+use super::FunctionCx;
+use crate::mir::OperandValue;
+use crate::mir::place::PlaceValue;
+use crate::traits::*;
+use crate::{base, common};
+
+pub(super) struct NicheFinder<'s, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
+ pub(super) fx: &'s mut FunctionCx<'a, 'tcx, Bx>,
+ pub(super) bx: &'s mut Bx,
+ pub(super) places: Vec<(mir::Operand<'tcx>, Niche)>,
+}
+
+impl<'s, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx> for NicheFinder<'s, 'a, 'tcx, Bx> {
+ fn visit_rvalue(&mut self, rvalue: &mir::Rvalue<'tcx>, location: mir::Location) {
+ match rvalue {
+ mir::Rvalue::Cast(mir::CastKind::Transmute, op, ty) => {
+ let ty = self.fx.monomorphize(*ty);
+ if let Some(niche) = self.bx.layout_of(ty).largest_niche {
+ self.places.push((op.clone(), niche));
+ }
+ }
+ _ => self.super_rvalue(rvalue, location),
+ }
+ }
+
+ fn visit_terminator(&mut self, terminator: &mir::Terminator<'tcx>, _location: mir::Location) {
+ if let mir::TerminatorKind::Return = terminator.kind {
+ let op = mir::Operand::Copy(mir::Place::return_place());
+ let ty = op.ty(self.fx.mir, self.bx.tcx());
+ let ty = self.fx.monomorphize(ty);
+ if let Some(niche) = self.bx.layout_of(ty).largest_niche {
+ self.places.push((op, niche));
+ }
+ }
+ }
+
+ fn visit_place(
+ &mut self,
+ place: &mir::Place<'tcx>,
+ context: PlaceContext,
+ _location: mir::Location,
+ ) {
+ match context {
+ PlaceContext::NonMutatingUse(
+ NonMutatingUseContext::Copy | NonMutatingUseContext::Move,
+ ) => {}
+ _ => {
+ return;
+ }
+ }
+
+ let ty = place.ty(self.fx.mir, self.bx.tcx()).ty;
+ let ty = self.fx.monomorphize(ty);
+ if let Some(niche) = self.bx.layout_of(ty).largest_niche {
+ self.places.push((mir::Operand::Copy(*place), niche));
+ };
+ }
+}
+
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+ fn value_in_niche(
+ &mut self,
+ bx: &mut Bx,
+ op: crate::mir::OperandRef<'tcx, Bx::Value>,
+ niche: Niche,
+ ) -> Option<Bx::Value> {
+ let niche_ty = niche.ty(bx.tcx());
+ let niche_layout = bx.layout_of(niche_ty);
+
+ let (imm, from_scalar, from_backend_ty) = match op.val {
+ OperandValue::Immediate(imm) => {
+ let BackendRepr::Scalar(from_scalar) = op.layout.backend_repr else {
+ unreachable!()
+ };
+ let from_backend_ty = bx.backend_type(op.layout);
+ (imm, from_scalar, from_backend_ty)
+ }
+ OperandValue::Pair(first, second) => {
+ let BackendRepr::ScalarPair(first_scalar, second_scalar) = op.layout.backend_repr
+ else {
+ unreachable!()
+ };
+ if niche.offset == Size::ZERO {
+ (first, first_scalar, bx.scalar_pair_element_backend_type(op.layout, 0, true))
+ } else {
+ // yolo
+ (second, second_scalar, bx.scalar_pair_element_backend_type(op.layout, 1, true))
+ }
+ }
+ OperandValue::ZeroSized => unreachable!(),
+ OperandValue::Ref(PlaceValue { llval: ptr, .. }) => {
+ // General case: Load the niche primitive via pointer arithmetic.
+ let niche_ptr_ty = Ty::new_ptr(bx.tcx(), niche_ty, Mutability::Not);
+ let ptr = bx.pointercast(ptr, bx.backend_type(bx.layout_of(niche_ptr_ty)));
+
+ let offset = niche.offset.bytes() / niche_layout.size.bytes();
+ let niche_backend_ty = bx.backend_type(bx.layout_of(niche_ty));
+ let ptr = bx.inbounds_gep(niche_backend_ty, ptr, &[bx.const_usize(offset)]);
+ let value = bx.load(niche_backend_ty, ptr, rustc_target::abi::Align::ONE);
+ return Some(value);
+ }
+ };
+
+ // Any type whose ABI is a Scalar bool is turned into an i1, so it cannot contain a value
+ // outside of its niche.
+ if from_scalar.is_bool() {
+ return None;
+ }
+
+ let to_scalar = Scalar::Initialized {
+ value: niche.value,
+ valid_range: WrappingRange::full(niche.size(bx.tcx())),
+ };
+ let to_backend_ty = bx.backend_type(niche_layout);
+ if from_backend_ty == to_backend_ty {
+ return Some(imm);
+ }
+ let value = self.transmute_immediate(
+ bx,
+ imm,
+ from_scalar,
+ from_backend_ty,
+ to_scalar,
+ to_backend_ty,
+ );
+ Some(value)
+ }
+
+ #[instrument(level = "debug", skip(self, bx))]
+ pub fn codegen_niche_check(
+ &mut self,
+ bx: &mut Bx,
+ mir_op: mir::Operand<'tcx>,
+ niche: Niche,
+ source_info: mir::SourceInfo,
+ ) {
+ let tcx = bx.tcx();
+ let op_ty = self.monomorphize(mir_op.ty(self.mir, tcx));
+ if op_ty == tcx.types.bool {
+ return;
+ }
+
+ let op = self.codegen_operand(bx, &mir_op);
+
+ let Some(value_in_niche) = self.value_in_niche(bx, op, niche) else {
+ return;
+ };
+ let size = niche.size(tcx);
+
+ let start = niche.scalar(niche.valid_range.start, bx);
+ let end = niche.scalar(niche.valid_range.end, bx);
+
+ let binop_le = base::bin_op_to_icmp_predicate(mir::BinOp::Le, false);
+ let binop_ge = base::bin_op_to_icmp_predicate(mir::BinOp::Ge, false);
+ let is_valid = if niche.valid_range.start == 0 {
+ bx.icmp(binop_le, value_in_niche, end)
+ } else if niche.valid_range.end == (u128::MAX >> 128 - size.bits()) {
+ bx.icmp(binop_ge, value_in_niche, start)
+ } else {
+ // We need to check if the value is within a *wrapping* range. We could do this:
+ // (niche >= start) && (niche <= end)
+ // But what we're going to actually do is this:
+ // max = end - start
+ // (niche - start) <= max
+ // The latter is much more complicated conceptually, but is actually less operations
+ // because we can compute max in codegen.
+ let mut max = niche.valid_range.end.wrapping_sub(niche.valid_range.start);
+ let size = niche.size(tcx);
+ if size.bits() < 128 {
+ let mask = (1 << size.bits()) - 1;
+ max &= mask;
+ }
+ let max_adjusted_allowed_value = niche.scalar(max, bx);
+
+ let biased = bx.sub(value_in_niche, start);
+ bx.icmp(binop_le, biased, max_adjusted_allowed_value)
+ };
+
+ // Create destination blocks, branching on is_valid
+ let panic = bx.append_sibling_block("panic");
+ let success = bx.append_sibling_block("success");
+ bx.cond_br(is_valid, success, panic);
+
+ // Switch to the failure block and codegen a call to the panic intrinsic
+ bx.switch_to_block(panic);
+ self.set_debug_loc(bx, source_info);
+ let location = self.get_caller_location(bx, source_info).immediate();
+ self.codegen_panic(
+ bx,
+ niche.lang_item(),
+ &[value_in_niche, start, end, location],
+ source_info.span,
+ );
+
+ // Continue codegen in the success block.
+ bx.switch_to_block(success);
+ self.set_debug_loc(bx, source_info);
+ }
+
+ #[instrument(level = "debug", skip(self, bx))]
+ fn codegen_panic(&mut self, bx: &mut Bx, lang_item: LangItem, args: &[Bx::Value], span: Span) {
+ if bx.tcx().is_compiler_builtins(LOCAL_CRATE) {
+ bx.abort()
+ } else {
+ let (fn_abi, fn_ptr, instance) = common::build_langcall(bx, Some(span), lang_item);
+ let fn_ty = bx.fn_decl_backend_type(&fn_abi);
+ let fn_attrs = if bx.tcx().def_kind(self.instance.def_id()).has_codegen_attrs() {
+ Some(bx.tcx().codegen_fn_attrs(self.instance.def_id()))
+ } else {
+ None
+ };
+ bx.call(fn_ty, fn_attrs, Some(&fn_abi), fn_ptr, args, None, Some(instance));
+ }
+ bx.unreachable();
+ }
+}
+
+trait NicheExt {
+ fn ty<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx>;
+ fn size<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Size;
+ fn scalar<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(&self, val: u128, bx: &mut Bx) -> Bx::Value;
+ fn lang_item(&self) -> LangItem;
+}
+
+impl NicheExt for Niche {
+ fn lang_item(&self) -> LangItem {
+ match self.value {
+ Primitive::Int(Integer::I8, _) => LangItem::PanicOccupiedNicheU8,
+ Primitive::Int(Integer::I16, _) => LangItem::PanicOccupiedNicheU16,
+ Primitive::Int(Integer::I32, _) => LangItem::PanicOccupiedNicheU32,
+ Primitive::Int(Integer::I64, _) => LangItem::PanicOccupiedNicheU64,
+ Primitive::Int(Integer::I128, _) => LangItem::PanicOccupiedNicheU128,
+ Primitive::Pointer(_) => LangItem::PanicOccupiedNichePtr,
+ Primitive::Float(Float::F16) => LangItem::PanicOccupiedNicheU16,
+ Primitive::Float(Float::F32) => LangItem::PanicOccupiedNicheU32,
+ Primitive::Float(Float::F64) => LangItem::PanicOccupiedNicheU64,
+ Primitive::Float(Float::F128) => LangItem::PanicOccupiedNicheU128,
+ }
+ }
+
+ fn ty<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
+ let types = &tcx.types;
+ match self.value {
+ Primitive::Int(Integer::I8, _) => types.u8,
+ Primitive::Int(Integer::I16, _) => types.u16,
+ Primitive::Int(Integer::I32, _) => types.u32,
+ Primitive::Int(Integer::I64, _) => types.u64,
+ Primitive::Int(Integer::I128, _) => types.u128,
+ Primitive::Pointer(_) => Ty::new_ptr(tcx, types.unit, Mutability::Not),
+ Primitive::Float(Float::F16) => types.u16,
+ Primitive::Float(Float::F32) => types.u32,
+ Primitive::Float(Float::F64) => types.u64,
+ Primitive::Float(Float::F128) => types.u128,
+ }
+ }
+
+ fn size<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Size {
+ self.value.size(&tcx)
+ }
+
+ fn scalar<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(&self, val: u128, bx: &mut Bx) -> Bx::Value {
+ use rustc_middle::mir::interpret::{Pointer, Scalar};
+ let tcx = bx.tcx();
+ let niche_ty = self.ty(tcx);
+ let value = if niche_ty.is_any_ptr() {
+ Scalar::from_maybe_pointer(Pointer::from_addr_invalid(val as u64), &tcx)
+ } else {
+ Scalar::from_uint(val, self.size(tcx))
+ };
+ let layout = rustc_target::abi::Scalar::Initialized {
+ value: self.value,
+ valid_range: WrappingRange::full(self.size(tcx)),
+ };
+ bx.scalar_to_backend(value, layout, bx.backend_type(bx.layout_of(self.ty(tcx))))
+ }
+}
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
index cf53739223467..7af5a1edcad3e 100644
--- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
@@ -160,7 +160,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
}
}
- fn codegen_transmute(
+ pub fn codegen_transmute(
&mut self,
bx: &mut Bx,
src: OperandRef<'tcx, Bx::Value>,
@@ -195,7 +195,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
///
/// Returns `None` for cases that can't work in that framework, such as for
/// `Immediate`->`Ref` that needs an `alloc` to get the location.
- fn codegen_transmute_operand(
+ pub fn codegen_transmute_operand(
&mut self,
bx: &mut Bx,
operand: OperandRef<'tcx, Bx::Value>,
@@ -337,7 +337,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
///
/// `to_backend_ty` must be the *non*-immediate backend type (so it will be
/// `i8`, not `i1`, for `bool`-like types.)
- fn transmute_immediate(
+ pub fn transmute_immediate(
&self,
bx: &mut Bx,
mut imm: Bx::Value,
diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs
index cd55a838a7561..e796f32a19877 100644
--- a/compiler/rustc_codegen_ssa/src/mir/statement.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs
@@ -1,14 +1,33 @@
+use rustc_middle::mir::visit::Visitor;
use rustc_middle::mir::{self, NonDivergingIntrinsic};
use rustc_middle::span_bug;
use tracing::instrument;
use super::{FunctionCx, LocalRef};
+use crate::mir::niche_check::NicheFinder;
use crate::traits::*;
impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+ fn niches_to_check(
+ &mut self,
+ bx: &mut Bx,
+ statement: &mir::Statement<'tcx>,
+ ) -> Vec<(mir::Operand<'tcx>, rustc_target::abi::Niche)> {
+ let mut finder = NicheFinder { fx: self, bx, places: Vec::new() };
+ finder.visit_statement(statement, rustc_middle::mir::Location::START);
+ finder.places
+ }
+
#[instrument(level = "debug", skip(self, bx))]
pub(crate) fn codegen_statement(&mut self, bx: &mut Bx, statement: &mir::Statement<'tcx>) {
self.set_debug_loc(bx, statement.source_info);
+
+ if bx.tcx().may_insert_niche_checks() {
+ for (op, niche) in self.niches_to_check(bx, statement) {
+ self.codegen_niche_check(bx, op, niche, statement.source_info);
+ }
+ }
+
match statement.kind {
mir::StatementKind::Assign(box (ref place, ref rvalue)) => {
if let Some(index) = place.as_local() {
diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs
index 15cb331d07a5f..dfdee445d4a4a 100644
--- a/compiler/rustc_hir/src/lang_items.rs
+++ b/compiler/rustc_hir/src/lang_items.rs
@@ -293,6 +293,12 @@ language_item_table! {
ConstPanicFmt, sym::const_panic_fmt, const_panic_fmt, Target::Fn, GenericRequirement::None;
PanicBoundsCheck, sym::panic_bounds_check, panic_bounds_check_fn, Target::Fn, GenericRequirement::Exact(0);
PanicMisalignedPointerDereference, sym::panic_misaligned_pointer_dereference, panic_misaligned_pointer_dereference_fn, Target::Fn, GenericRequirement::Exact(0);
+ PanicOccupiedNicheU8, sym::panic_occupied_niche_u8, panic_occupied_niche_u8, Target::Fn, GenericRequirement::None;
+ PanicOccupiedNicheU16, sym::panic_occupied_niche_u16, panic_occupied_niche_u16, Target::Fn, GenericRequirement::None;
+ PanicOccupiedNicheU32, sym::panic_occupied_niche_u32, panic_occupied_niche_u32, Target::Fn, GenericRequirement::None;
+ PanicOccupiedNicheU64, sym::panic_occupied_niche_u64, panic_occupied_niche_u64, Target::Fn, GenericRequirement::None;
+ PanicOccupiedNicheU128, sym::panic_occupied_niche_u128, panic_occupied_niche_u128, Target::Fn, GenericRequirement::None;
+ PanicOccupiedNichePtr, sym::panic_occupied_niche_ptr, panic_occupied_niche_ptr, Target::Fn, GenericRequirement::None;
PanicInfo, sym::panic_info, panic_info, Target::Struct, GenericRequirement::None;
PanicLocation, sym::panic_location, panic_location, Target::Struct, GenericRequirement::None;
PanicImpl, sym::panic_impl, panic_impl, Target::Fn, GenericRequirement::None;
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 2841470d24878..e47315d409d0d 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -3210,6 +3210,14 @@ impl<'tcx> TyCtxt<'tcx> {
pub fn do_not_recommend_impl(self, def_id: DefId) -> bool {
self.get_diagnostic_attr(def_id, sym::do_not_recommend).is_some()
}
+
+ /// Whether a codegen backend may emit alignment checks for pointers when they are
+ /// read or written through. If this returns true, the backend is allowed to emit such checks.
+ /// If this returns false, the backend must not emit such checks.
+ pub fn may_insert_niche_checks(self) -> bool {
+ let has_panic_shim = self.lang_items().get(LangItem::PanicOccupiedNicheU8).is_some();
+ has_panic_shim && self.sess.ub_checks()
+ }
}
/// Parameter attributes that can only be determined by examining the body of a function instead
diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs
index 480d82c1a385b..0d0d5703dee2c 100644
--- a/compiler/rustc_monomorphize/src/collector.rs
+++ b/compiler/rustc_monomorphize/src/collector.rs
@@ -231,6 +231,7 @@ use rustc_middle::util::Providers;
use rustc_middle::{bug, span_bug};
use rustc_session::Limit;
use rustc_session::config::EntryFnType;
+use rustc_span::def_id::LOCAL_CRATE;
use rustc_span::source_map::{Spanned, dummy_spanned, respan};
use rustc_span::symbol::sym;
use rustc_span::{DUMMY_SP, Span};
@@ -1369,6 +1370,26 @@ fn collect_roots(tcx: TyCtxt<'_>, mode: MonoItemCollectionStrategy) -> Vec<MonoI
collector.process_impl_item(id);
}
+ if tcx.may_insert_niche_checks() && !tcx.is_compiler_builtins(LOCAL_CRATE) {
+ for item in [
+ LangItem::PanicOccupiedNicheU8,
+ LangItem::PanicOccupiedNicheU16,
+ LangItem::PanicOccupiedNicheU32,
+ LangItem::PanicOccupiedNicheU64,
+ LangItem::PanicOccupiedNicheU128,
+ LangItem::PanicOccupiedNichePtr,
+ ] {
+ let Some(def_id) = tcx.lang_items().get(item) else {
+ continue;
+ };
+ let instance = rustc_middle::ty::Instance::mono(tcx, def_id);
+ if tcx.should_codegen_locally(instance) {
+ let mono_item = create_fn_mono_item(tcx, instance, DUMMY_SP);
+ collector.output.push(mono_item)
+ }
+ }
+ }
+
collector.push_extra_entry_roots();
}
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 9b499c716039b..cf3a195b99d48 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -1455,6 +1455,12 @@ symbols! {
panic_location,
panic_misaligned_pointer_dereference,
panic_nounwind,
+ panic_occupied_niche_ptr,
+ panic_occupied_niche_u128,
+ panic_occupied_niche_u16,
+ panic_occupied_niche_u32,
+ panic_occupied_niche_u64,
+ panic_occupied_niche_u8,
panic_runtime,
panic_str_2015,
panic_unwind,
diff --git a/library/core/src/panicking.rs b/library/core/src/panicking.rs
index 53e2b238bae69..eef19162a0fd3 100644
--- a/library/core/src/panicking.rs
+++ b/library/core/src/panicking.rs
@@ -291,7 +291,34 @@ fn panic_misaligned_pointer_dereference(required: usize, found: usize) -> ! {
)
}
-/// Panics because we cannot unwind out of a function.
+macro_rules! panic_occupied_niche {
+ ($name:ident, $ty:ty) => {
+ #[cfg_attr(not(feature = "panic_immediate_abort"), inline(never), cold)]
+ #[cfg_attr(feature = "panic_immediate_abort", inline)]
+ #[track_caller]
+ #[cfg_attr(not(bootstrap), lang = stringify!($name))] // needed by codegen for panic on occupied niches
+ #[rustc_nounwind]
+ fn $name(found: $ty, min: $ty, max: $ty) -> ! {
+ if cfg!(feature = "panic_immediate_abort") {
+ super::intrinsics::abort()
+ }
+
+ panic_nounwind_fmt(
+ format_args!("occupied niche: found {found:?} but must be in {min:?}..={max:?}"),
+ /* force_no_backtrace */ false,
+ )
+ }
+ };
+}
+
+panic_occupied_niche!(panic_occupied_niche_u8, u8);
+panic_occupied_niche!(panic_occupied_niche_u16, u16);
+panic_occupied_niche!(panic_occupied_niche_u32, u32);
+panic_occupied_niche!(panic_occupied_niche_u64, u64);
+panic_occupied_niche!(panic_occupied_niche_u128, u128);
+panic_occupied_niche!(panic_occupied_niche_ptr, *const ());
+
+/// Panic because we cannot unwind out of a function.
///
/// This is a separate function to avoid the codesize impact of each crate containing the string to
/// pass to `panic_nounwind`.
diff --git a/tests/run-make/wasm-symbols-different-module/rmake.rs b/tests/run-make/wasm-symbols-different-module/rmake.rs
index 3cd459db02fae..2c5672a8fd30a 100644
--- a/tests/run-make/wasm-symbols-different-module/rmake.rs
+++ b/tests/run-make/wasm-symbols-different-module/rmake.rs
@@ -22,7 +22,7 @@ fn test_file(file: &str, expected_imports: &[(&str, &[&str])]) {
fn test(file: &str, args: &[&str], expected_imports: &[(&str, &[&str])]) {
println!("test {file:?} {args:?} for {expected_imports:?}");
- rustc().input(file).target("wasm32-wasip1").args(args).run();
+ rustc().input(file).target("wasm32-wasip1").args(args).arg("-Zub-checks=no").run();
let file = rfs::read(Path::new(file).with_extension("wasm"));
@@ -46,5 +46,5 @@ fn test(file: &str, args: &[&str], expected_imports: &[(&str, &[&str])]) {
assert!(names.contains(name));
}
}
- assert!(imports.is_empty());
+ assert!(imports.is_empty(), "unexpected imports: {:?}", imports);
}
diff --git a/tests/ui/async-await/future-sizes/async-awaiting-fut.rs b/tests/ui/async-await/future-sizes/async-awaiting-fut.rs
index a3f0bdc851481..38c85ba38e499 100644
--- a/tests/ui/async-await/future-sizes/async-awaiting-fut.rs
+++ b/tests/ui/async-await/future-sizes/async-awaiting-fut.rs
@@ -1,4 +1,4 @@
-//@ compile-flags: -Z print-type-sizes --crate-type lib
+//@ compile-flags: -Z print-type-sizes -Zub-checks=no --crate-type lib
//@ edition:2021
//@ build-pass
//@ ignore-pass
diff --git a/tests/ui/async-await/future-sizes/large-arg.rs b/tests/ui/async-await/future-sizes/large-arg.rs
index 5fbae22a7714d..0dafcd98478dd 100644
--- a/tests/ui/async-await/future-sizes/large-arg.rs
+++ b/tests/ui/async-await/future-sizes/large-arg.rs
@@ -1,4 +1,4 @@
-//@ compile-flags: -Z print-type-sizes --crate-type=lib
+//@ compile-flags: -Z print-type-sizes -Zub-checks=no --crate-type=lib
//@ edition: 2021
//@ build-pass
//@ ignore-pass
diff --git a/tests/ui/niche_checks/invalid_enums.rs b/tests/ui/niche_checks/invalid_enums.rs
new file mode 100644
index 0000000000000..cb3bc07793e5d
--- /dev/null
+++ b/tests/ui/niche_checks/invalid_enums.rs
@@ -0,0 +1,23 @@
+//@ run-fail
+//@ ignore-wasm32-bare: No panic messages
+//@ compile-flags: -Zmir-opt-level=0 -Cdebug-assertions=no -Zub-checks=yes
+
+#[repr(C)]
+struct Thing {
+ x: usize,
+ y: Contents,
+ z: usize,
+}
+
+#[repr(usize)]
+enum Contents {
+ A = 8usize,
+ B = 9usize,
+ C = 10usize,
+}
+
+fn main() {
+ unsafe {
+ let _thing = std::mem::transmute::<(usize, usize, usize), Thing>((0, 3, 0));
+ }
+}
diff --git a/tests/ui/niche_checks/invalid_nonnull_transmute.rs b/tests/ui/niche_checks/invalid_nonnull_transmute.rs
new file mode 100644
index 0000000000000..5cbce1aa6711c
--- /dev/null
+++ b/tests/ui/niche_checks/invalid_nonnull_transmute.rs
@@ -0,0 +1,10 @@
+//@ run-fail
+//@ ignore-wasm32-bare: No panic messages
+//@ compile-flags: -Zmir-opt-level=0 -Cdebug-assertions=no -Zub-checks=yes
+//@ error-pattern: occupied niche: found 0x0 but must be in 0x1..=0xffffffff
+
+fn main() {
+ unsafe {
+ std::mem::transmute::<*const u8, std::ptr::NonNull<u8>>(std::ptr::null());
+ }
+}
diff --git a/tests/ui/niche_checks/invalid_nonzero_argument.rs b/tests/ui/niche_checks/invalid_nonzero_argument.rs
new file mode 100644
index 0000000000000..6746aa9ad2486
--- /dev/null
+++ b/tests/ui/niche_checks/invalid_nonzero_argument.rs
@@ -0,0 +1,14 @@
+//@ run-fail
+//@ ignore-wasm32-bare: No panic messages
+//@ compile-flags: -Zmir-opt-level=0 -Cdebug-assertions=no -Zub-checks=yes
+//@ error-pattern: occupied niche: found 0 but must be in 1..=255
+
+fn main() {
+ let mut bad = std::num::NonZeroU8::new(1u8).unwrap();
+ unsafe {
+ std::ptr::write_bytes(&mut bad, 0u8, 1usize);
+ }
+ func(bad);
+}
+
+fn func<T>(_t: T) {}
diff --git a/tests/ui/niche_checks/invalid_option.rs b/tests/ui/niche_checks/invalid_option.rs
new file mode 100644
index 0000000000000..9698738d1e8d0
--- /dev/null
+++ b/tests/ui/niche_checks/invalid_option.rs
@@ -0,0 +1,10 @@
+//@ run-fail
+//@ ignore-wasm32-bare: No panic messages
+//@ compile-flags: -Zmir-opt-level=0 -Cdebug-assertions=no -Zub-checks=yes
+//@ error-pattern: occupied niche: found 2 but must be in 0..=1
+
+fn main() {
+ unsafe {
+ std::mem::transmute::<(u8, u8), Option<u8>>((2, 0));
+ }
+}
diff --git a/tests/ui/niche_checks/operand_pair.rs b/tests/ui/niche_checks/operand_pair.rs
new file mode 100644
index 0000000000000..97caf07239373
--- /dev/null
+++ b/tests/ui/niche_checks/operand_pair.rs
@@ -0,0 +1,12 @@
+//@ run-fail
+//@ ignore-wasm32-bare: No panic messages
+//@ compile-flags: -Zmir-opt-level=0 -Cdebug-assertions=no -Zub-checks=yes
+//@ error-pattern: occupied niche: found 0x0 but must be in 0x1..=0xffffffff
+
+use std::ptr::NonNull;
+
+fn main() {
+ unsafe {
+ std::mem::transmute::<(usize, *const u8), (usize, NonNull<u8>)>((0usize, std::ptr::null()));
+ }
+}
diff --git a/tests/ui/niche_checks/valid_nonzero_argument.rs b/tests/ui/niche_checks/valid_nonzero_argument.rs
new file mode 100644
index 0000000000000..2d6d83c4cb377
--- /dev/null
+++ b/tests/ui/niche_checks/valid_nonzero_argument.rs
@@ -0,0 +1,13 @@
+//@ run-pass
+//@ compile-flags: -Zmir-opt-level=0 -Cdebug-assertions=no -Zub-checks=yes
+
+fn main() {
+ for val in i8::MIN..=i8::MAX {
+ if val != 0 {
+ let x = std::num::NonZeroI8::new(val).unwrap();
+ if val != i8::MIN {
+ let _y = -x;
+ }
+ }
+ }
+}
diff --git a/tests/ui/numbers-arithmetic/saturating-float-casts-wasm.rs b/tests/ui/numbers-arithmetic/saturating-float-casts-wasm.rs
index 8f1fe59ddc59b..d78af52574203 100644
--- a/tests/ui/numbers-arithmetic/saturating-float-casts-wasm.rs
+++ b/tests/ui/numbers-arithmetic/saturating-float-casts-wasm.rs
@@ -1,6 +1,6 @@
//@ run-pass
//@ only-wasm32
-//@ compile-flags: -Zmir-opt-level=0 -C target-feature=+nontrapping-fptoint
+//@ compile-flags: -Zmir-opt-level=0 -Zub-checks=no -C target-feature=+nontrapping-fptoint
#![feature(test, stmt_expr_attributes)]
#![deny(overflowing_literals)]
diff --git a/tests/ui/numbers-arithmetic/saturating-float-casts.rs b/tests/ui/numbers-arithmetic/saturating-float-casts.rs
index 3a1af09d2d423..c9e35f5d17b13 100644
--- a/tests/ui/numbers-arithmetic/saturating-float-casts.rs
+++ b/tests/ui/numbers-arithmetic/saturating-float-casts.rs
@@ -1,5 +1,5 @@
//@ run-pass
-//@ compile-flags:-Zmir-opt-level=0
+//@ compile-flags: -Zmir-opt-level=0 -Zub-checks=no
#![feature(test, stmt_expr_attributes)]
#![deny(overflowing_literals)]
diff --git a/tests/ui/print_type_sizes/async.rs b/tests/ui/print_type_sizes/async.rs
index 805bccbcf6381..5918e04fb5d81 100644
--- a/tests/ui/print_type_sizes/async.rs
+++ b/tests/ui/print_type_sizes/async.rs
@@ -1,4 +1,4 @@
-//@ compile-flags: -Z print-type-sizes --crate-type lib
+//@ compile-flags: -Z print-type-sizes -Zub-checks=no --crate-type lib
//@ edition:2021
//@ build-pass
//@ ignore-pass
diff --git a/tests/ui/print_type_sizes/coroutine.rs b/tests/ui/print_type_sizes/coroutine.rs
index 1533578878944..f07e496c0a66e 100644
--- a/tests/ui/print_type_sizes/coroutine.rs
+++ b/tests/ui/print_type_sizes/coroutine.rs
@@ -1,4 +1,4 @@
-//@ compile-flags: -Z print-type-sizes --crate-type=lib
+//@ compile-flags: -Z print-type-sizes -Zub-checks=no --crate-type=lib
//@ build-pass
//@ ignore-pass
diff --git a/tests/ui/print_type_sizes/niche-filling.stdout b/tests/ui/print_type_sizes/niche-filling.stdout
index eeb5de5324121..22de9ffd74af0 100644
--- a/tests/ui/print_type_sizes/niche-filling.stdout
+++ b/tests/ui/print_type_sizes/niche-filling.stdout
@@ -1,3 +1,7 @@
+print-type-size type: `std::panic::Location<'_>`: 24 bytes, alignment: 8 bytes
+print-type-size field `.file`: 16 bytes
+print-type-size field `.line`: 4 bytes
+print-type-size field `.col`: 4 bytes
print-type-size type: `IndirectNonZero`: 12 bytes, alignment: 4 bytes
print-type-size field `.nested`: 8 bytes
print-type-size field `.post`: 2 bytes