diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index 1a916c8768243..d834fc7b008fa 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -14,6 +14,7 @@ use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_session::config::OutputFilenames; use rustc_span::Symbol; +use rustc_target::spec::PanicStrategy; use crate::constant::ConstantCx; use crate::debuginfo::{FunctionDebugContext, TypeDebugContext}; @@ -384,6 +385,11 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { fx.bcx.switch_to_block(failure); fx.bcx.ins().nop(); + if fx.tcx.sess.panic_strategy() == PanicStrategy::ImmediateAbort { + fx.bcx.ins().trap(TrapCode::user(1 /* unreachable */).unwrap()); + continue; + } + match &**msg { AssertKind::BoundsCheck { len, index } => { let len = codegen_operand(fx, len).load_scalar(fx); @@ -1052,6 +1058,10 @@ pub(crate) fn codegen_panic_nounwind<'tcx>( msg_str: &str, span: Span, ) { + if fx.tcx.sess.panic_strategy() == PanicStrategy::ImmediateAbort { + fx.bcx.ins().trap(TrapCode::user(1 /* unreachable */).unwrap()); + } + let msg_ptr = crate::constant::pointer_for_anonymous_str(fx, msg_str); let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap()); let args = [msg_ptr, msg_len]; diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index 4cb390e7d5699..c8ab7186bb03f 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -4,7 +4,7 @@ use rustc_abi::{Align, ExternAbi}; use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, DiffActivity, DiffMode}; use rustc_ast::{LitKind, MetaItem, MetaItemInner}; use rustc_hir::attrs::{ - AttributeKind, EiiImplResolution, InlineAttr, Linkage, RtsanSetting, UsedBy, + AttributeKind, EiiImplResolution, InlineAttr, Linkage, OptimizeAttr, RtsanSetting, UsedBy, }; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE, LocalDefId}; @@ -19,7 +19,7 @@ use rustc_middle::ty::{self as ty, TyCtxt}; use rustc_session::lint; use rustc_session::parse::feature_err; use rustc_span::{Span, sym}; -use rustc_target::spec::Os; +use rustc_target::spec::{Os, PanicStrategy}; use crate::errors; use crate::target_features::{ @@ -391,6 +391,24 @@ fn apply_overrides(tcx: TyCtxt<'_>, did: LocalDefId, codegen_fn_attrs: &mut Code codegen_fn_attrs.flags |= CodegenFnAttrFlags::NO_MANGLE; } } + + if tcx.is_panic_entrypoint(did) { + // Panic entrypoints are always cold. + // + // If we have immediate-abort enabled, we want them to be inlined. + // They shouldn't be called, but on the off-chance that they are, they should be inlined. + // + // When the panic strategies that support panic messages are enabled, we want panic + // entrypoints outlined and optimized for size. + // Most panic entrypoints want #[track_caller] but not all, so we do not add it. + codegen_fn_attrs.flags |= CodegenFnAttrFlags::COLD; + if tcx.sess.panic_strategy() == PanicStrategy::ImmediateAbort { + codegen_fn_attrs.inline = InlineAttr::Always; + } else { + codegen_fn_attrs.inline = InlineAttr::Never; + codegen_fn_attrs.optimize = OptimizeAttr::Size; + } + } } fn check_result( diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 35de8b5e1486b..9a878fd58b40c 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -15,6 +15,7 @@ use rustc_session::config::OptLevel; use rustc_span::Span; use rustc_span::source_map::Spanned; use rustc_target::callconv::{ArgAbi, ArgAttributes, CastTarget, FnAbi, PassMode}; +use rustc_target::spec::PanicStrategy; use tracing::{debug, info}; use super::operand::OperandRef; @@ -737,6 +738,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.switch_to_block(panic_block); self.set_debug_loc(bx, terminator.source_info); + if bx.tcx().sess.panic_strategy() == PanicStrategy::ImmediateAbort { + bx.abort(); + bx.unreachable(); + return MergingSucc::False; + } + // Get the location information. let location = self.get_caller_location(bx, terminator.source_info).immediate(); diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs index ded8a5a4ae51c..8f2448d6459bd 100644 --- a/compiler/rustc_feature/src/builtin_attrs.rs +++ b/compiler/rustc_feature/src/builtin_attrs.rs @@ -1329,6 +1329,10 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ "`#[rustc_has_incoherent_inherent_impls]` allows the addition of incoherent inherent impls for \ the given type by annotating all impl items with `#[rustc_allow_incoherent_impl]`" ), + rustc_attr!( + rustc_panic_entrypoint, Normal, template!(Word), WarnFollowing, + EncodeCrossCrate::Yes, "`#[rustc_panic_entrypoint]` makes this function patchable by panic=immediate-abort", + ), BuiltinAttribute { name: sym::rustc_diagnostic_item, diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs index 557f76208bfe6..c1f2ac17e2296 100644 --- a/compiler/rustc_hir/src/lang_items.rs +++ b/compiler/rustc_hir/src/lang_items.rs @@ -247,6 +247,7 @@ language_item_table! { FnMut, sym::fn_mut, fn_mut_trait, Target::Trait, GenericRequirement::Exact(1); FnOnce, sym::fn_once, fn_once_trait, Target::Trait, GenericRequirement::Exact(1); + AbortIntrinsic, sym::abort_intrinsic, abort_intrinsic, Target::Fn, GenericRequirement::Exact(0); AsyncFn, sym::async_fn, async_fn_trait, Target::Trait, GenericRequirement::Exact(1); AsyncFnMut, sym::async_fn_mut, async_fn_mut_trait, Target::Trait, GenericRequirement::Exact(1); AsyncFnOnce, sym::async_fn_once, async_fn_once_trait, Target::Trait, GenericRequirement::Exact(1); diff --git a/compiler/rustc_lint/messages.ftl b/compiler/rustc_lint/messages.ftl index 867b937d4090d..ce2e12ac2c788 100644 --- a/compiler/rustc_lint/messages.ftl +++ b/compiler/rustc_lint/messages.ftl @@ -614,6 +614,8 @@ lint_missing_gpu_kernel_export_name = function with the "gpu-kernel" ABI has a m .note = mangled names make it hard to find the kernel, this is usually not intended .help = use `unsafe(no_mangle)` or `unsafe(export_name = "")` +lint_missing_panic_entrypoint = "diverging functions should usually have #[inline] or #[rustc_panic_entrypoint]" + lint_mixed_script_confusables = the usage of Script Group `{$set}` in this crate consists solely of mixed script confusables .includes_note = the usage includes {$includes} diff --git a/compiler/rustc_lint/src/internal.rs b/compiler/rustc_lint/src/internal.rs index 37eb375c7a6c2..1712d2a80da88 100644 --- a/compiler/rustc_lint/src/internal.rs +++ b/compiler/rustc_lint/src/internal.rs @@ -1,12 +1,14 @@ //! Some lints that are only useful in the compiler or crates that use compiler internals, such as //! Clippy. -use rustc_hir::attrs::AttributeKind; +use rustc_hir::attrs::{AttributeKind, InlineAttr}; use rustc_hir::def::Res; use rustc_hir::def_id::DefId; -use rustc_hir::{Expr, ExprKind, HirId, find_attr}; +use rustc_hir::intravisit::FnKind; +use rustc_hir::{AttrPath, Body, Expr, ExprKind, FnDecl, HirId, find_attr}; use rustc_middle::ty::{self, GenericArgsRef, PredicatePolarity}; use rustc_session::{declare_lint_pass, declare_tool_lint}; +use rustc_span::def_id::LocalDefId; use rustc_span::hygiene::{ExpnKind, MacroKind}; use rustc_span::{Span, sym}; use {rustc_ast as ast, rustc_hir as hir}; @@ -627,3 +629,84 @@ impl EarlyLintPass for ImplicitSysrootCrateImport { } } } + +declare_tool_lint! { + /// The `missing_panic_entrypoint` lint detects forgotten use of #[rustc_panic_entrypoint]. + /// + /// This lint is intended to ensure that panic=immediate-abort can function as designed, + /// because it uses #[rustc_panic_entrypoint] to locate functions that should be outlined + /// for other panic modes, and be deleted entirely when immediate-abort is enabled. + pub rustc::MISSING_PANIC_ENTRYPOINT, + Allow, + "detects missing #[rustc_panic_entrypoint]", + report_in_external_macro: true +} + +declare_lint_pass!(MissingPanicEntrypoint => [MISSING_PANIC_ENTRYPOINT]); + +fn has_panic_entrypoint(attrs: &[hir::Attribute]) -> bool { + attrs.iter().any(|attr| { + if let hir::Attribute::Unparsed(box hir::AttrItem { + path: AttrPath { segments, .. }, .. + }) = attr + { + if segments[0] == sym::rustc_panic_entrypoint { + return true; + } + } + false + }) +} + +fn has_inline_encouragement(attrs: &[hir::Attribute]) -> bool { + attrs.iter().any(|attr| { + matches!( + attr, + hir::Attribute::Parsed(hir::attrs::AttributeKind::Inline( + InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. }, + _ + )) + ) + }) +} + +fn has_rustc_intrinsic(attrs: &[hir::Attribute]) -> bool { + attrs.iter().any(|attr| { + if let hir::Attribute::Unparsed(box hir::AttrItem { + path: AttrPath { segments, .. }, .. + }) = attr + { + if segments[0] == sym::rustc_intrinsic { + return true; + } + } + false + }) +} + +impl<'tcx> LateLintPass<'tcx> for MissingPanicEntrypoint { + fn check_fn( + &mut self, + cx: &LateContext<'tcx>, + _: FnKind<'tcx>, + fn_decl: &'tcx FnDecl<'tcx>, + _: &'tcx Body<'tcx>, + span: Span, + def_id: LocalDefId, + ) { + if matches!(fn_decl.output, hir::FnRetTy::Return(hir::Ty { kind: hir::TyKind::Never, .. })) + { + let attrs = cx.tcx.hir_attrs(cx.tcx.local_def_id_to_hir_id(def_id)); + if has_rustc_intrinsic(attrs) { + return; + } + if !has_inline_encouragement(attrs) && !has_panic_entrypoint(attrs) { + cx.emit_span_lint( + MISSING_PANIC_ENTRYPOINT, + span, + crate::lints::MissingPanicEntrypoint, + ); + } + } + } +} diff --git a/compiler/rustc_lint/src/lib.rs b/compiler/rustc_lint/src/lib.rs index 80b32645a8953..92078a8937260 100644 --- a/compiler/rustc_lint/src/lib.rs +++ b/compiler/rustc_lint/src/lib.rs @@ -240,6 +240,7 @@ late_lint_methods!( MapUnitFn: MapUnitFn, MissingDebugImplementations: MissingDebugImplementations, MissingDoc: MissingDoc, + MissingPanicEntrypoint: MissingPanicEntrypoint, AsyncClosureUsage: AsyncClosureUsage, AsyncFnInTrait: AsyncFnInTrait, NonLocalDefinitions: NonLocalDefinitions::default(), @@ -664,6 +665,7 @@ fn register_internals(store: &mut LintStore) { store.register_late_mod_pass(|_| Box::new(SymbolInternStringLiteral)); store.register_lints(&ImplicitSysrootCrateImport::lint_vec()); store.register_early_pass(|| Box::new(ImplicitSysrootCrateImport)); + store.register_late_mod_pass(|_| Box::new(MissingPanicEntrypoint)); store.register_group( false, "rustc::internal", @@ -683,6 +685,7 @@ fn register_internals(store: &mut LintStore) { LintId::of(SPAN_USE_EQ_CTXT), LintId::of(DIRECT_USE_OF_RUSTC_TYPE_IR), LintId::of(IMPLICIT_SYSROOT_CRATE_IMPORT), + LintId::of(MISSING_PANIC_ENTRYPOINT), ], ); } diff --git a/compiler/rustc_lint/src/lints.rs b/compiler/rustc_lint/src/lints.rs index a20d90e1227e9..590b0e579834e 100644 --- a/compiler/rustc_lint/src/lints.rs +++ b/compiler/rustc_lint/src/lints.rs @@ -3325,3 +3325,7 @@ pub(crate) struct UnknownCrateTypesSuggestion { pub span: Span, pub snippet: Symbol, } + +#[derive(LintDiagnostic)] +#[diag(lint_missing_panic_entrypoint)] +pub(crate) struct MissingPanicEntrypoint; diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index f015d0edc56c8..cfc73cfbdf31c 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -49,7 +49,7 @@ use rustc_session::config::CrateType; use rustc_session::cstore::{CrateStoreDyn, Untracked}; use rustc_session::lint::Lint; use rustc_span::def_id::{CRATE_DEF_ID, DefPathHash, StableCrateId}; -use rustc_span::{DUMMY_SP, Ident, Span, Symbol, kw}; +use rustc_span::{DUMMY_SP, Ident, Span, Symbol, kw, sym}; use rustc_type_ir::TyKind::*; use rustc_type_ir::lang_items::{SolverAdtLangItem, SolverLangItem, SolverTraitLangItem}; pub use rustc_type_ir::lift::Lift; @@ -3556,6 +3556,10 @@ impl<'tcx> TyCtxt<'tcx> { } false } + + pub fn is_panic_entrypoint(self, def_id: impl Into) -> bool { + self.has_attr(def_id, sym::rustc_panic_entrypoint) + } } pub fn provide(providers: &mut Providers) { diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 2f8c6b8763987..f6d1c27bb8e01 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -167,6 +167,7 @@ declare_passes! { mod remove_unneeded_drops : RemoveUnneededDrops; mod remove_zsts : RemoveZsts; mod required_consts : RequiredConstsVisitor; + mod panic_entrypoints : PanicEntrypoints; mod post_analysis_normalize : PostAnalysisNormalize; mod sanity_check : SanityCheck; // This pass is public to allow external drivers to perform MIR cleanup @@ -698,7 +699,7 @@ pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<' &check_null::CheckNull, &check_enums::CheckEnums, // Before inlining: trim down MIR with passes to reduce inlining work. - + &panic_entrypoints::PanicEntrypoints, // Has to be done before inlining, otherwise actual call will be almost always inlined. // Also simple, so can just do first. &lower_slice_len::LowerSliceLenCalls, diff --git a/compiler/rustc_mir_transform/src/panic_entrypoints.rs b/compiler/rustc_mir_transform/src/panic_entrypoints.rs new file mode 100644 index 0000000000000..d7d7fc01a1d4f --- /dev/null +++ b/compiler/rustc_mir_transform/src/panic_entrypoints.rs @@ -0,0 +1,68 @@ +use rustc_hir::LangItem; +use rustc_middle::mir::{Const, ConstValue, UnwindAction, *}; +use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_span::Span; +use rustc_target::spec::PanicStrategy; +use tracing::{debug, instrument}; + +pub(super) struct PanicEntrypoints; + +impl<'tcx> crate::MirPass<'tcx> for PanicEntrypoints { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.panic_strategy() == PanicStrategy::ImmediateAbort + } + + #[instrument(level = "trace", skip(self, tcx, body))] + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + if tcx.is_panic_entrypoint(body.source.def_id()) { + debug!("Replacing body of {:?}", body.source); + let blocks = body.basic_blocks.as_mut(); + blocks.raw.clear(); + blocks.push(BasicBlockData::new(Some(abort_terminator(tcx, body.span)), false)); + return; + } + + for bb in body.basic_blocks.as_mut().iter_mut() { + let terminator = bb.terminator.as_mut().expect("invalid terminator"); + let TerminatorKind::Call { func, .. } = &mut terminator.kind else { + continue; + }; + let Operand::Constant(box ConstOperand { const_, .. }) = &func else { + continue; + }; + let ty::FnDef(def_id, _) = *const_.ty().kind() else { + continue; + }; + if tcx.is_panic_entrypoint(def_id) { + debug!("Remapping call from {:?} to {:?}", body.source, def_id); + *terminator = abort_terminator(tcx, terminator.source_info.span); + } + } + } + + fn is_required(&self) -> bool { + true + } +} + +fn abort_terminator<'tcx>(tcx: TyCtxt<'tcx>, span: Span) -> Terminator<'tcx> { + let abort_intrin = tcx.require_lang_item(LangItem::AbortIntrinsic, span); + let no_args: [ty::GenericArg<'_>; 0] = []; + let func = Operand::Constant(Box::new(ConstOperand { + span, + user_ty: None, + const_: Const::Val(ConstValue::ZeroSized, Ty::new_fn_def(tcx, abort_intrin, no_args)), + })); + Terminator { + source_info: SourceInfo::outermost(span), + kind: TerminatorKind::Call { + func, + args: Box::new([]), + destination: Place::return_place(), + target: None, + unwind: UnwindAction::Unreachable, + call_source: CallSource::Misc, + fn_span: span, + }, + } +} diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs index cdd141f9233e8..c3a7eaf15822b 100644 --- a/compiler/rustc_passes/src/check_attr.rs +++ b/compiler/rustc_passes/src/check_attr.rs @@ -400,6 +400,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> { | sym::rustc_partition_reused | sym::rustc_partition_codegened | sym::rustc_expected_cgu_reuse + | sym::rustc_panic_entrypoint // crate-level attrs, are checked below | sym::feature | sym::register_tool diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 8fd228211f3c3..7e51c02c78ae8 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -434,6 +434,7 @@ symbols! { abi_vectorcall, abi_x86_interrupt, abort, + abort_intrinsic, add, add_assign, add_with_overflow, @@ -1999,6 +2000,7 @@ symbols! { rustc_offload_kernel, rustc_on_unimplemented, rustc_outlives, + rustc_panic_entrypoint, rustc_paren_sugar, rustc_partition_codegened, rustc_partition_reused, diff --git a/library/alloc/src/alloc.rs b/library/alloc/src/alloc.rs index cd1c2ea8fcd1e..1c8956fcb10bd 100644 --- a/library/alloc/src/alloc.rs +++ b/library/alloc/src/alloc.rs @@ -531,9 +531,9 @@ unsafe extern "Rust" { #[stable(feature = "global_alloc", since = "1.28.0")] #[rustc_const_unstable(feature = "const_alloc_error", issue = "92523")] #[cfg(not(no_global_oom_handling))] -#[cold] -#[optimize(size)] +#[rustc_panic_entrypoint] pub const fn handle_alloc_error(layout: Layout) -> ! { + #[inline] const fn ct_error(_: Layout) -> ! { panic!("allocation failed"); } @@ -545,13 +545,7 @@ pub const fn handle_alloc_error(layout: Layout) -> ! { } } - #[cfg(not(panic = "immediate-abort"))] - { - core::intrinsics::const_eval_select((layout,), ct_error, rt_error) - } - - #[cfg(panic = "immediate-abort")] - ct_error(layout) + core::intrinsics::const_eval_select((layout,), ct_error, rt_error) } #[cfg(not(no_global_oom_handling))] @@ -562,6 +556,7 @@ pub mod __alloc_error_handler { // called via generated `__rust_alloc_error_handler` if there is no // `#[alloc_error_handler]`. #[rustc_std_internal_symbol] + #[allow(rustc::missing_panic_entrypoint)] pub unsafe fn __rdl_alloc_error_handler(size: usize, _align: usize) -> ! { core::panicking::panic_nounwind_fmt( format_args!("memory allocation of {size} bytes failed"), diff --git a/library/alloc/src/lib.rs b/library/alloc/src/lib.rs index f7167650635d3..4e76ffffb2da9 100644 --- a/library/alloc/src/lib.rs +++ b/library/alloc/src/lib.rs @@ -82,6 +82,7 @@ #![warn(rustdoc::unescaped_backticks)] #![deny(ffi_unwind_calls)] #![warn(unreachable_pub)] +#![warn(rustc::missing_panic_entrypoint)] // // Library features: // tidy-alphabetical-start diff --git a/library/alloc/src/raw_vec/mod.rs b/library/alloc/src/raw_vec/mod.rs index ff996ba93cd7f..c71450c1dea04 100644 --- a/library/alloc/src/raw_vec/mod.rs +++ b/library/alloc/src/raw_vec/mod.rs @@ -23,7 +23,7 @@ mod tests; // ensure that the code generation related to these panics is minimal as there's // only one location which panics rather than a bunch throughout the module. #[cfg(not(no_global_oom_handling))] -#[cfg_attr(not(panic = "immediate-abort"), inline(never))] +#[rustc_panic_entrypoint] const fn capacity_overflow() -> ! { panic!("capacity overflow"); } @@ -852,8 +852,7 @@ impl RawVecInner { // Central function for reserve error handling. #[cfg(not(no_global_oom_handling))] -#[cold] -#[optimize(size)] +#[rustc_panic_entrypoint] #[rustc_const_unstable(feature = "const_heap", issue = "79597")] const fn handle_error(e: TryReserveError) -> ! { match e.kind() { diff --git a/library/alloc/src/vec/mod.rs b/library/alloc/src/vec/mod.rs index 93432f3e049e1..a42538e2a421a 100644 --- a/library/alloc/src/vec/mod.rs +++ b/library/alloc/src/vec/mod.rs @@ -2133,9 +2133,8 @@ impl Vec { #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub fn swap_remove(&mut self, index: usize) -> T { - #[cold] - #[cfg_attr(not(panic = "immediate-abort"), inline(never))] - #[optimize(size)] + #[track_caller] + #[rustc_panic_entrypoint] fn assert_failed(index: usize, len: usize) -> ! { panic!("swap_remove index (is {index}) should be < len (is {len})"); } @@ -2214,10 +2213,8 @@ impl Vec { #[track_caller] #[must_use = "if you don't need a reference to the value, use `Vec::insert` instead"] pub fn insert_mut(&mut self, index: usize, element: T) -> &mut T { - #[cold] - #[cfg_attr(not(panic = "immediate-abort"), inline(never))] #[track_caller] - #[optimize(size)] + #[rustc_panic_entrypoint] fn assert_failed(index: usize, len: usize) -> ! { panic!("insertion index (is {index}) should be <= len (is {len})"); } @@ -2278,10 +2275,8 @@ impl Vec { #[track_caller] #[rustc_confusables("delete", "take")] pub fn remove(&mut self, index: usize) -> T { - #[cold] - #[cfg_attr(not(panic = "immediate-abort"), inline(never))] #[track_caller] - #[optimize(size)] + #[rustc_panic_entrypoint] fn assert_failed(index: usize, len: usize) -> ! { panic!("removal index (is {index}) should be < len (is {len})"); } @@ -2991,10 +2986,8 @@ impl Vec { where A: Clone, { - #[cold] - #[cfg_attr(not(panic = "immediate-abort"), inline(never))] #[track_caller] - #[optimize(size)] + #[rustc_panic_entrypoint] fn assert_failed(at: usize, len: usize) -> ! { panic!("`at` split index (is {at}) should be <= len (is {len})"); } diff --git a/library/core/src/cell.rs b/library/core/src/cell.rs index 661ea4ab6a27f..857865de640a0 100644 --- a/library/core/src/cell.rs +++ b/library/core/src/cell.rs @@ -883,9 +883,8 @@ impl Display for BorrowMutError { } // This ensures the panicking code is outlined from `borrow_mut` for `RefCell`. -#[cfg_attr(not(panic = "immediate-abort"), inline(never))] #[track_caller] -#[cold] +#[rustc_panic_entrypoint] const fn panic_already_borrowed(err: BorrowMutError) -> ! { const_panic!( "RefCell already borrowed", @@ -895,9 +894,8 @@ const fn panic_already_borrowed(err: BorrowMutError) -> ! { } // This ensures the panicking code is outlined from `borrow` for `RefCell`. -#[cfg_attr(not(panic = "immediate-abort"), inline(never))] #[track_caller] -#[cold] +#[rustc_panic_entrypoint] const fn panic_already_mutably_borrowed(err: BorrowError) -> ! { const_panic!( "RefCell already mutably borrowed", diff --git a/library/core/src/cell/lazy.rs b/library/core/src/cell/lazy.rs index 28a76569c1d03..9aebbac5c61f9 100644 --- a/library/core/src/cell/lazy.rs +++ b/library/core/src/cell/lazy.rs @@ -366,8 +366,7 @@ impl fmt::Debug for LazyCell { } } -#[cold] -#[inline(never)] +#[rustc_panic_entrypoint] const fn panic_poisoned() -> ! { panic!("LazyCell instance has previously been poisoned") } diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index 051dda731881f..c3ce6f56e2811 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -366,6 +366,7 @@ pub fn rustc_peek(_: T) -> T; /// `SIGBUS`. The precise behavior is not guaranteed and not stable. #[rustc_nounwind] #[rustc_intrinsic] +#[lang = "abort_intrinsic"] pub fn abort() -> !; /// Informs the optimizer that this point in the code is not reachable, diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs index c4d16ba633b81..3b6045584949f 100644 --- a/library/core/src/lib.rs +++ b/library/core/src/lib.rs @@ -88,6 +88,7 @@ #![warn(multiple_supertrait_upcastable)] #![allow(internal_features)] #![deny(ffi_unwind_calls)] +#![warn(rustc::missing_panic_entrypoint)] #![warn(unreachable_pub)] // Do not check link redundancy on bootstrapping phase #![allow(rustdoc::redundant_explicit_links)] diff --git a/library/core/src/num/int_log10.rs b/library/core/src/num/int_log10.rs index af8e1f90968d6..e8d6402f8a4ba 100644 --- a/library/core/src/num/int_log10.rs +++ b/library/core/src/num/int_log10.rs @@ -164,8 +164,8 @@ define_signed_ilog10! { /// Instantiate this panic logic once, rather than for all the ilog methods /// on every single primitive type. -#[cold] #[track_caller] +#[rustc_panic_entrypoint] pub(super) const fn panic_for_nonpositive_argument() -> ! { panic!("argument of integer logarithm must be positive") } diff --git a/library/core/src/num/int_sqrt.rs b/library/core/src/num/int_sqrt.rs index c7a322c08c139..a3d93e1f8be51 100644 --- a/library/core/src/num/int_sqrt.rs +++ b/library/core/src/num/int_sqrt.rs @@ -309,8 +309,8 @@ unsigned_fn!(u128, u64, u128_stages); /// Instantiate this panic logic once, rather than for all the isqrt methods /// on every single primitive type. -#[cold] #[track_caller] +#[rustc_panic_entrypoint] pub(super) const fn panic_for_negative_argument() -> ! { panic!("argument of integer square root cannot be negative") } diff --git a/library/core/src/num/mod.rs b/library/core/src/num/mod.rs index 558426c94e5dc..44bb776f53025 100644 --- a/library/core/src/num/mod.rs +++ b/library/core/src/num/mod.rs @@ -1388,10 +1388,8 @@ pub const fn can_not_overflow(radix: u32, is_signed_ty: bool, digits: &[u8]) radix <= 16 && digits.len() <= size_of::() * 2 - is_signed_ty as usize } -#[cfg_attr(not(panic = "immediate-abort"), inline(never))] -#[cfg_attr(panic = "immediate-abort", inline)] -#[cold] #[track_caller] +#[rustc_panic_entrypoint] const fn from_ascii_radix_panic(radix: u32) -> ! { const_panic!( "from_ascii_radix: radix must lie in the range `[2, 36]`", diff --git a/library/core/src/num/overflow_panic.rs b/library/core/src/num/overflow_panic.rs index e30573dd3f392..a72ffec2bdaab 100644 --- a/library/core/src/num/overflow_panic.rs +++ b/library/core/src/num/overflow_panic.rs @@ -2,50 +2,50 @@ //! //! In particular, these are used by the `strict_` methods on integers. -#[cold] #[track_caller] +#[rustc_panic_entrypoint] pub(super) const fn add() -> ! { panic!("attempt to add with overflow") } -#[cold] #[track_caller] +#[rustc_panic_entrypoint] pub(super) const fn sub() -> ! { panic!("attempt to subtract with overflow") } -#[cold] #[track_caller] +#[rustc_panic_entrypoint] pub(super) const fn mul() -> ! { panic!("attempt to multiply with overflow") } -#[cold] #[track_caller] +#[rustc_panic_entrypoint] pub(super) const fn div() -> ! { panic!("attempt to divide with overflow") } -#[cold] #[track_caller] +#[rustc_panic_entrypoint] pub(super) const fn rem() -> ! { panic!("attempt to calculate the remainder with overflow") } -#[cold] #[track_caller] +#[rustc_panic_entrypoint] pub(super) const fn neg() -> ! { panic!("attempt to negate with overflow") } -#[cold] #[track_caller] +#[rustc_panic_entrypoint] pub(super) const fn shr() -> ! { panic!("attempt to shift right with overflow") } -#[cold] #[track_caller] +#[rustc_panic_entrypoint] pub(super) const fn shl() -> ! { panic!("attempt to shift left with overflow") } diff --git a/library/core/src/option.rs b/library/core/src/option.rs index eb4f978b7c19d..e950d9d7182b1 100644 --- a/library/core/src/option.rs +++ b/library/core/src/option.rs @@ -2185,19 +2185,15 @@ impl Option> { } } -#[cfg_attr(not(panic = "immediate-abort"), inline(never))] -#[cfg_attr(panic = "immediate-abort", inline)] -#[cold] #[track_caller] +#[rustc_panic_entrypoint] const fn unwrap_failed() -> ! { panic("called `Option::unwrap()` on a `None` value") } // This is a separate function to reduce the code size of .expect() itself. -#[cfg_attr(not(panic = "immediate-abort"), inline(never))] -#[cfg_attr(panic = "immediate-abort", inline)] -#[cold] #[track_caller] +#[rustc_panic_entrypoint] const fn expect_failed(msg: &str) -> ! { panic_display(&msg) } diff --git a/library/core/src/panicking.rs b/library/core/src/panicking.rs index 3609dd1fe2e02..a8445635b6b42 100644 --- a/library/core/src/panicking.rs +++ b/library/core/src/panicking.rs @@ -49,19 +49,12 @@ compile_error!( /// site as much as possible (so that `panic!()` has as low an impact /// on (e.g.) the inlining of other functions as possible), by moving /// the actual formatting into this shared place. -// If panic=immediate-abort, inline the abort call, -// otherwise avoid inlining because of it is cold path. -#[cfg_attr(not(panic = "immediate-abort"), inline(never), cold)] -#[cfg_attr(panic = "immediate-abort", inline)] #[track_caller] #[lang = "panic_fmt"] // needed for const-evaluated panics #[rustc_do_not_const_check] // hooked by const-eval #[rustc_const_stable_indirect] // must follow stable const rules since it is exposed to stable +#[rustc_panic_entrypoint] pub const fn panic_fmt(fmt: fmt::Arguments<'_>) -> ! { - if cfg!(panic = "immediate-abort") { - super::intrinsics::abort() - } - // NOTE This function never crosses the FFI boundary; it's a Rust-to-Rust call // that gets resolved to the `#[panic_handler]` function. unsafe extern "Rust" { @@ -83,8 +76,6 @@ pub const fn panic_fmt(fmt: fmt::Arguments<'_>) -> ! { /// Like `panic_fmt`, but for non-unwinding panics. /// /// Has to be a separate function so that it can carry the `rustc_nounwind` attribute. -#[cfg_attr(not(panic = "immediate-abort"), inline(never), cold)] -#[cfg_attr(panic = "immediate-abort", inline)] #[track_caller] // This attribute has the key side-effect that if the panic handler ignores `can_unwind` // and unwinds anyway, we will hit the "unwinding out of nounwind function" guard, @@ -92,6 +83,7 @@ pub const fn panic_fmt(fmt: fmt::Arguments<'_>) -> ! { #[rustc_nounwind] #[rustc_const_stable_indirect] // must follow stable const rules since it is exposed to stable #[rustc_allow_const_fn_unstable(const_eval_select)] +#[rustc_panic_entrypoint] pub const fn panic_nounwind_fmt(fmt: fmt::Arguments<'_>, force_no_backtrace: bool) -> ! { const_eval_select!( @capture { fmt: fmt::Arguments<'_>, force_no_backtrace: bool } -> !: @@ -99,10 +91,6 @@ pub const fn panic_nounwind_fmt(fmt: fmt::Arguments<'_>, force_no_backtrace: boo // We don't unwind anyway at compile-time so we can call the regular `panic_fmt`. panic_fmt(fmt) } else #[track_caller] { - if cfg!(panic = "immediate-abort") { - super::intrinsics::abort() - } - // NOTE This function never crosses the FFI boundary; it's a Rust-to-Rust call // that gets resolved to the `#[panic_handler]` function. unsafe extern "Rust" { @@ -128,13 +116,10 @@ pub const fn panic_nounwind_fmt(fmt: fmt::Arguments<'_>, force_no_backtrace: boo // above. /// The underlying implementation of core's `panic!` macro when no formatting is used. -// Never inline unless panic=immediate-abort to avoid code -// bloat at the call sites as much as possible. -#[cfg_attr(not(panic = "immediate-abort"), inline(never), cold)] -#[cfg_attr(panic = "immediate-abort", inline)] #[track_caller] #[rustc_const_stable_indirect] // must follow stable const rules since it is exposed to stable #[lang = "panic"] // used by lints and miri for panics +#[rustc_panic_entrypoint] pub const fn panic(expr: &'static str) -> ! { // Use Arguments::from_str instead of format_args!("{expr}") to potentially // reduce size overhead. The format_args! macro uses str's Display trait to @@ -162,14 +147,10 @@ macro_rules! panic_const { ($($lang:ident = $message:expr,)+) => { $( /// This is a panic called with a message that's a result of a MIR-produced Assert. - // - // never inline unless panic=immediate-abort to avoid code - // bloat at the call sites as much as possible - #[cfg_attr(not(panic = "immediate-abort"), inline(never), cold)] - #[cfg_attr(panic = "immediate-abort", inline)] #[track_caller] #[rustc_const_stable_indirect] // must follow stable const rules since it is exposed to stable #[lang = stringify!($lang)] + #[rustc_panic_entrypoint] pub const fn $lang() -> ! { // See the comment in `panic(&'static str)` for why we use `Arguments::from_str` here. panic_fmt(fmt::Arguments::from_str($message)); @@ -216,19 +197,17 @@ pub mod panic_const { /// Like `panic`, but without unwinding and track_caller to reduce the impact on codesize on the caller. /// If you want `#[track_caller]` for nicer errors, call `panic_nounwind_fmt` directly. -#[cfg_attr(not(panic = "immediate-abort"), inline(never), cold)] -#[cfg_attr(panic = "immediate-abort", inline)] #[lang = "panic_nounwind"] // needed by codegen for non-unwinding panics #[rustc_nounwind] #[rustc_const_stable_indirect] // must follow stable const rules since it is exposed to stable +#[rustc_panic_entrypoint] pub const fn panic_nounwind(expr: &'static str) -> ! { panic_nounwind_fmt(fmt::Arguments::from_str(expr), /* force_no_backtrace */ false); } /// Like `panic_nounwind`, but also inhibits showing a backtrace. -#[cfg_attr(not(panic = "immediate-abort"), inline(never), cold)] -#[cfg_attr(panic = "immediate-abort", inline)] #[rustc_nounwind] +#[rustc_panic_entrypoint] pub fn panic_nounwind_nobacktrace(expr: &'static str) -> ! { panic_nounwind_fmt(fmt::Arguments::from_str(expr), /* force_no_backtrace */ true); } @@ -259,28 +238,18 @@ pub const fn panic_display(x: &T) -> ! { panic_fmt(format_args!("{}", *x)); } -#[cfg_attr(not(panic = "immediate-abort"), inline(never), cold, optimize(size))] -#[cfg_attr(panic = "immediate-abort", inline)] +#[rustc_panic_entrypoint] #[track_caller] #[lang = "panic_bounds_check"] // needed by codegen for panic on OOB array/slice access fn panic_bounds_check(index: usize, len: usize) -> ! { - if cfg!(panic = "immediate-abort") { - super::intrinsics::abort() - } - panic!("index out of bounds: the len is {len} but the index is {index}") } -#[cfg_attr(not(panic = "immediate-abort"), inline(never), cold, optimize(size))] -#[cfg_attr(panic = "immediate-abort", inline)] #[track_caller] #[lang = "panic_misaligned_pointer_dereference"] // needed by codegen for panic on misaligned pointer deref #[rustc_nounwind] // `CheckAlignment` MIR pass requires this function to never unwind +#[rustc_panic_entrypoint] fn panic_misaligned_pointer_dereference(required: usize, found: usize) -> ! { - if cfg!(panic = "immediate-abort") { - super::intrinsics::abort() - } - panic_nounwind_fmt( format_args!( "misaligned pointer dereference: address must be a multiple of {required:#x} but is {found:#x}" @@ -289,32 +258,22 @@ fn panic_misaligned_pointer_dereference(required: usize, found: usize) -> ! { ) } -#[cfg_attr(not(panic = "immediate-abort"), inline(never), cold, optimize(size))] -#[cfg_attr(panic = "immediate-abort", inline)] +#[rustc_panic_entrypoint] #[track_caller] #[lang = "panic_null_pointer_dereference"] // needed by codegen for panic on null pointer deref #[rustc_nounwind] // `CheckNull` MIR pass requires this function to never unwind fn panic_null_pointer_dereference() -> ! { - if cfg!(panic = "immediate-abort") { - super::intrinsics::abort() - } - panic_nounwind_fmt( format_args!("null pointer dereference occurred"), /* force_no_backtrace */ false, ) } -#[cfg_attr(not(panic = "immediate-abort"), inline(never), cold, optimize(size))] -#[cfg_attr(panic = "immediate-abort", inline)] #[track_caller] #[lang = "panic_invalid_enum_construction"] // needed by codegen for panic on invalid enum construction. #[rustc_nounwind] // `CheckEnums` MIR pass requires this function to never unwind +#[rustc_panic_entrypoint] fn panic_invalid_enum_construction(source: u128) -> ! { - if cfg!(panic = "immediate-abort") { - super::intrinsics::abort() - } - panic_nounwind_fmt( format_args!("trying to construct an enum from an invalid value {source:#x}"), /* force_no_backtrace */ false, @@ -328,10 +287,9 @@ fn panic_invalid_enum_construction(source: u128) -> ! { /// /// This function is called directly by the codegen backend, and must not have /// any extra arguments (including those synthesized by track_caller). -#[cfg_attr(not(panic = "immediate-abort"), inline(never), cold, optimize(size))] -#[cfg_attr(panic = "immediate-abort", inline)] #[lang = "panic_cannot_unwind"] // needed by codegen for panic in nounwind function #[rustc_nounwind] +#[rustc_panic_entrypoint] fn panic_cannot_unwind() -> ! { // Keep the text in sync with `UnwindTerminateReason::as_str` in `rustc_middle`. panic_nounwind("panic in a function that cannot unwind") @@ -344,10 +302,9 @@ fn panic_cannot_unwind() -> ! { /// /// This function is called directly by the codegen backend, and must not have /// any extra arguments (including those synthesized by track_caller). -#[cfg_attr(not(panic = "immediate-abort"), inline(never), cold, optimize(size))] -#[cfg_attr(panic = "immediate-abort", inline)] #[lang = "panic_in_cleanup"] // needed by codegen for panic in nounwind function #[rustc_nounwind] +#[rustc_panic_entrypoint] fn panic_in_cleanup() -> ! { // Keep the text in sync with `UnwindTerminateReason::as_str` in `rustc_middle`. panic_nounwind_nobacktrace("panic in a destructor during cleanup") @@ -356,6 +313,7 @@ fn panic_in_cleanup() -> ! { /// This function is used instead of panic_fmt in const eval. #[lang = "const_panic_fmt"] // needed by const-eval machine to replace calls to `panic_fmt` lang item #[rustc_const_stable_indirect] // must follow stable const rules since it is exposed to stable +#[inline] // No need to monomorphize this, it's only called by const-eval. pub const fn const_panic_fmt(fmt: fmt::Arguments<'_>) -> ! { if let Some(msg) = fmt.as_str() { // The panic_display function is hooked by const eval. @@ -377,10 +335,9 @@ pub enum AssertKind { } /// Internal function for `assert_eq!` and `assert_ne!` macros -#[cfg_attr(not(panic = "immediate-abort"), inline(never), cold, optimize(size))] -#[cfg_attr(panic = "immediate-abort", inline)] #[track_caller] #[doc(hidden)] +#[rustc_panic_entrypoint] pub fn assert_failed( kind: AssertKind, left: &T, @@ -395,10 +352,9 @@ where } /// Internal function for `assert_match!` -#[cfg_attr(not(panic = "immediate-abort"), inline(never), cold, optimize(size))] -#[cfg_attr(panic = "immediate-abort", inline)] #[track_caller] #[doc(hidden)] +#[rustc_panic_entrypoint] pub fn assert_matches_failed( left: &T, right: &str, @@ -415,9 +371,8 @@ pub fn assert_matches_failed( } /// Non-generic version of the above functions, to avoid code bloat. -#[cfg_attr(not(panic = "immediate-abort"), inline(never), cold, optimize(size))] -#[cfg_attr(panic = "immediate-abort", inline)] #[track_caller] +#[rustc_panic_entrypoint] fn assert_failed_inner( kind: AssertKind, left: &dyn fmt::Debug, diff --git a/library/core/src/result.rs b/library/core/src/result.rs index 9afa71ec0f117..5fdedbfc1cc2a 100644 --- a/library/core/src/result.rs +++ b/library/core/src/result.rs @@ -1860,9 +1860,8 @@ impl Result, E> { // This is a separate function to reduce the code size of the methods #[cfg(not(panic = "immediate-abort"))] -#[inline(never)] -#[cold] #[track_caller] +#[rustc_panic_entrypoint] fn unwrap_failed(msg: &str, error: &dyn fmt::Debug) -> ! { panic!("{msg}: {error:?}"); } @@ -1872,9 +1871,8 @@ fn unwrap_failed(msg: &str, error: &dyn fmt::Debug) -> ! { // by dead code elimination if a trait object is constructed even if it goes // unused #[cfg(panic = "immediate-abort")] -#[inline] -#[cold] #[track_caller] +#[rustc_panic_entrypoint] const fn unwrap_failed(_msg: &str, _error: &T) -> ! { panic!() } diff --git a/library/core/src/slice/index.rs b/library/core/src/slice/index.rs index d8ed521f44353..8c3156ccf008c 100644 --- a/library/core/src/slice/index.rs +++ b/library/core/src/slice/index.rs @@ -32,9 +32,8 @@ where } } -#[cfg_attr(not(panic = "immediate-abort"), inline(never), cold)] -#[cfg_attr(panic = "immediate-abort", inline)] #[track_caller] +#[rustc_panic_entrypoint] const fn slice_index_fail(start: usize, end: usize, len: usize) -> ! { if start > len { const_panic!( diff --git a/library/core/src/slice/mod.rs b/library/core/src/slice/mod.rs index 3e1eeba4e92e6..9839869a311d0 100644 --- a/library/core/src/slice/mod.rs +++ b/library/core/src/slice/mod.rs @@ -5345,9 +5345,8 @@ impl [f64] { const unsafe fn copy_from_slice_impl(dest: &mut [T], src: &[T]) { // The panic code path was put into a cold function to not bloat the // call site. - #[cfg_attr(not(panic = "immediate-abort"), inline(never), cold)] - #[cfg_attr(panic = "immediate-abort", inline)] #[track_caller] + #[rustc_panic_entrypoint] const fn len_mismatch_fail(dst_len: usize, src_len: usize) -> ! { const_panic!( "copy_from_slice: source slice length does not match destination slice length", diff --git a/library/core/src/slice/sort/shared/smallsort.rs b/library/core/src/slice/sort/shared/smallsort.rs index e555fce440872..f1df15368d9bb 100644 --- a/library/core/src/slice/sort/shared/smallsort.rs +++ b/library/core/src/slice/sort/shared/smallsort.rs @@ -840,8 +840,8 @@ unsafe fn bidirectional_merge bool>( } } -#[cfg_attr(not(panic = "immediate-abort"), inline(never), cold)] -#[cfg_attr(panic = "immediate-abort", inline)] +#[track_caller] +#[rustc_panic_entrypoint] fn panic_on_ord_violation() -> ! { // This is indicative of a logic bug in the user-provided comparison function or Ord // implementation. They are expected to implement a total order as explained in the Ord diff --git a/library/core/src/str/mod.rs b/library/core/src/str/mod.rs index ab7389a1300c5..2a362b3a5382f 100644 --- a/library/core/src/str/mod.rs +++ b/library/core/src/str/mod.rs @@ -60,26 +60,21 @@ pub use traits::FromStr; #[unstable(feature = "str_internals", issue = "none")] pub use validations::{next_code_point, utf8_char_width}; -#[inline(never)] -#[cold] #[track_caller] #[rustc_allow_const_fn_unstable(const_eval_select)] -#[cfg(not(panic = "immediate-abort"))] +#[rustc_panic_entrypoint] const fn slice_error_fail(s: &str, begin: usize, end: usize) -> ! { crate::intrinsics::const_eval_select((s, begin, end), slice_error_fail_ct, slice_error_fail_rt) } -#[cfg(panic = "immediate-abort")] -const fn slice_error_fail(s: &str, begin: usize, end: usize) -> ! { - slice_error_fail_ct(s, begin, end) -} - #[track_caller] +#[inline] const fn slice_error_fail_ct(_: &str, _: usize, _: usize) -> ! { panic!("failed to slice string"); } #[track_caller] +#[inline] fn slice_error_fail_rt(s: &str, begin: usize, end: usize) -> ! { const MAX_DISPLAY_LENGTH: usize = 256; let trunc_len = s.floor_char_boundary(MAX_DISPLAY_LENGTH); diff --git a/library/core/src/str/traits.rs b/library/core/src/str/traits.rs index a7cc943994c53..b0d4e3ff98064 100644 --- a/library/core/src/str/traits.rs +++ b/library/core/src/str/traits.rs @@ -76,9 +76,8 @@ where } } -#[inline(never)] -#[cold] #[track_caller] +#[rustc_panic_entrypoint] const fn str_index_overflow_fail() -> ! { panic!("attempted to index str up to maximum usize"); } diff --git a/library/core/src/wtf8.rs b/library/core/src/wtf8.rs index 7214918db6c39..f26cf9b8a7e31 100644 --- a/library/core/src/wtf8.rs +++ b/library/core/src/wtf8.rs @@ -485,7 +485,8 @@ unsafe fn slice_unchecked(s: &Wtf8, begin: usize, end: usize) -> &Wtf8 { } /// Copied from core::str::raw::slice_error_fail -#[inline(never)] +#[track_caller] +#[rustc_panic_entrypoint] fn slice_error_fail(s: &Wtf8, begin: usize, end: usize) -> ! { assert!(begin <= end); panic!("index {begin} and/or {end} in `{s:?}` do not lie on character boundary"); diff --git a/library/std/src/panicking.rs b/library/std/src/panicking.rs index a4a974d0447b8..3988dc0aa22e8 100644 --- a/library/std/src/panicking.rs +++ b/library/std/src/panicking.rs @@ -711,17 +711,11 @@ pub fn panic_handler(info: &core::panic::PanicInfo<'_>) -> ! { #[unstable(feature = "libstd_sys_internals", reason = "used by the panic! macro", issue = "none")] #[cfg_attr(not(any(test, doctest)), lang = "begin_panic")] // lang item for CTFE panic support -// never inline unless panic=immediate-abort to avoid code -// bloat at the call sites as much as possible -#[cfg_attr(not(panic = "immediate-abort"), inline(never), cold, optimize(size))] -#[cfg_attr(panic = "immediate-abort", inline)] +// never inline to avoid code bloat at the call sites as much as possible #[track_caller] #[rustc_do_not_const_check] // hooked by const-eval +#[rustc_panic_entrypoint] pub const fn begin_panic(msg: M) -> ! { - if cfg!(panic = "immediate-abort") { - intrinsics::abort() - } - struct Payload { inner: Option, } @@ -879,16 +873,9 @@ pub fn resume_unwind(payload: Box) -> ! { /// A function with a fixed suffix (through `rustc_std_internal_symbol`) /// on which to slap yer breakpoints. -#[inline(never)] #[cfg_attr(not(test), rustc_std_internal_symbol)] -#[cfg(not(panic = "immediate-abort"))] +#[rustc_panic_entrypoint] fn rust_panic(msg: &mut dyn PanicPayload) -> ! { let code = unsafe { __rust_start_panic(msg) }; rtabort!("failed to initiate panic, error {code}") } - -#[cfg_attr(not(test), rustc_std_internal_symbol)] -#[cfg(panic = "immediate-abort")] -fn rust_panic(_: &mut dyn PanicPayload) -> ! { - crate::intrinsics::abort(); -} diff --git a/library/std/src/sync/lazy_lock.rs b/library/std/src/sync/lazy_lock.rs index 7274b5d10c75b..9897268e548a7 100644 --- a/library/std/src/sync/lazy_lock.rs +++ b/library/std/src/sync/lazy_lock.rs @@ -404,8 +404,7 @@ impl fmt::Debug for LazyLock { } } -#[cold] -#[inline(never)] +#[rustc_panic_entrypoint] fn panic_poisoned() -> ! { panic!("LazyLock instance has previously been poisoned") } diff --git a/library/std/src/thread/id.rs b/library/std/src/thread/id.rs index 3da0825db604d..320397e5a18a8 100644 --- a/library/std/src/thread/id.rs +++ b/library/std/src/thread/id.rs @@ -33,7 +33,7 @@ pub struct ThreadId(NonZero); impl ThreadId { // Generate a new unique thread ID. pub(crate) fn new() -> ThreadId { - #[cold] + #[rustc_panic_entrypoint] fn exhausted() -> ! { panic!("failed to generate unique thread ID: bitspace exhausted") } diff --git a/library/std/src/thread/local.rs b/library/std/src/thread/local.rs index 1318d8dc27809..dab593e483b5b 100644 --- a/library/std/src/thread/local.rs +++ b/library/std/src/thread/local.rs @@ -428,9 +428,8 @@ impl fmt::Display for AccessError { impl Error for AccessError {} // This ensures the panicking code is outlined from `with` for `LocalKey`. -#[cfg_attr(not(panic = "immediate-abort"), inline(never))] #[track_caller] -#[cold] +#[rustc_panic_entrypoint] fn panic_access_error(err: AccessError) -> ! { panic!("cannot access a Thread Local Storage value during or after destruction: {err:?}") }