mir_transform: implement #[rustc_force_inline]

Cargo.lock

@@ -4158,6 +4158,7 @@ dependencies = [
  "rustc_apfloat",
  "rustc_arena",
  "rustc_ast",
+ "rustc_attr_parsing",
  "rustc_data_structures",
  "rustc_errors",
  "rustc_fluent_macro",

compiler/rustc_attr_data_structures/src/attributes.rs

@@ -11,6 +11,22 @@ pub enum InlineAttr {
     Hint,
     Always,
     Never,
+    /// `#[rustc_force_inline]` forces inlining to happen in the MIR inliner - it reports an error
+    /// if the inlining cannot happen. It is limited to only free functions so that the calls
+    /// can always be resolved.
+    Force {
+        attr_span: Span,
+        reason: Option<Symbol>,
+    },
+}
+
+impl InlineAttr {
+    pub fn always(&self) -> bool {
+        match self {
+            InlineAttr::Always | InlineAttr::Force { .. } => true,
+            InlineAttr::None | InlineAttr::Hint | InlineAttr::Never => false,
+        }
+    }
 }
 
 #[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq, HashStable_Generic)]

compiler/rustc_codegen_gcc/src/attributes.rs

@@ -20,7 +20,7 @@ fn inline_attr<'gcc, 'tcx>(
 ) -> Option<FnAttribute<'gcc>> {
     match inline {
         InlineAttr::Hint => Some(FnAttribute::Inline),
-        InlineAttr::Always => Some(FnAttribute::AlwaysInline),
+        InlineAttr::Always | InlineAttr::Force { .. } => Some(FnAttribute::AlwaysInline),
         InlineAttr::Never => {
             if cx.sess().target.arch != "amdgpu" {
                 Some(FnAttribute::NoInline)

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -37,7 +37,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll
     }
     match inline {
         InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)),
-        InlineAttr::Always => Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)),
+        InlineAttr::Always | InlineAttr::Force { .. } => {
+            Some(AttributeKind::AlwaysInline.create_attr(cx.llcx))
+        }
         InlineAttr::Never => {
             if cx.sess().target.arch != "amdgpu" {
                 Some(AttributeKind::NoInline.create_attr(cx.llcx))

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

@@ -18,6 +18,7 @@ use rustc_session::parse::feature_err;
 use rustc_session::{Session, lint};
 use rustc_span::{Ident, Span, sym};
 use rustc_target::spec::{SanitizerSet, abi};
+use tracing::debug;
 
 use crate::errors;
 use crate::target_features::{check_target_feature_trait_unsafe, from_target_feature_attr};
@@ -525,6 +526,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
         if !attr.has_name(sym::inline) {
             return ia;
         }
+
         if attr.is_word() {
             InlineAttr::Hint
         } else if let Some(ref items) = attr.meta_item_list() {
@@ -547,6 +549,20 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
             ia
         }
     });
+    codegen_fn_attrs.inline = attrs.iter().fold(codegen_fn_attrs.inline, |ia, attr| {
+        if !attr.has_name(sym::rustc_force_inline) || !tcx.features().rustc_attrs() {
+            return ia;
+        }
+
+        if attr.is_word() {
+            InlineAttr::Force { attr_span: attr.span, reason: None }
+        } else if let Some(val) = attr.value_str() {
+            InlineAttr::Force { attr_span: attr.span, reason: Some(val) }
+        } else {
+            debug!("`rustc_force_inline` not checked by attribute validation");
+            ia
+        }
+    });
 
     // naked function MUST NOT be inlined! This attribute is required for the rust compiler itself,
     // but not for the code generation backend because at that point the naked function will just be
@@ -596,7 +612,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
     // is probably a poor usage of `#[inline(always)]` and easily avoided by not using the attribute.
     if tcx.features().target_feature_11()
         && tcx.is_closure_like(did.to_def_id())
-        && codegen_fn_attrs.inline != InlineAttr::Always
+        && !codegen_fn_attrs.inline.always()
     {
         let owner_id = tcx.parent(did.to_def_id());
         if tcx.def_kind(owner_id).has_codegen_attrs() {
@@ -606,22 +622,28 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
         }
     }
 
-    // If a function uses #[target_feature] it can't be inlined into general
+    // If a function uses `#[target_feature]` it can't be inlined into general
     // purpose functions as they wouldn't have the right target features
-    // enabled. For that reason we also forbid #[inline(always)] as it can't be
+    // enabled. For that reason we also forbid `#[inline(always)]` as it can't be
     // respected.
-    if !codegen_fn_attrs.target_features.is_empty() && codegen_fn_attrs.inline == InlineAttr::Always
+    //
+    // `#[rustc_force_inline]` doesn't need to be prohibited here, only
+    // `#[inline(always)]`, as forced inlining is implemented entirely within
+    // rustc (and so the MIR inliner can do any necessary checks for compatible target
+    // features).
+    //
+    // This sidesteps the LLVM blockers in enabling `target_features` +
+    // `inline(always)` to be used together (see rust-lang/rust#116573 and
+    // llvm/llvm-project#70563).
+    if !codegen_fn_attrs.target_features.is_empty()
+        && matches!(codegen_fn_attrs.inline, InlineAttr::Always)
     {
         if let Some(span) = inline_span {
-            tcx.dcx().span_err(
-                span,
-                "cannot use `#[inline(always)]` with \
-                     `#[target_feature]`",
-            );
+            tcx.dcx().span_err(span, "cannot use `#[inline(always)]` with `#[target_feature]`");
         }
     }
 
-    if !codegen_fn_attrs.no_sanitize.is_empty() && codegen_fn_attrs.inline == InlineAttr::Always {
+    if !codegen_fn_attrs.no_sanitize.is_empty() && codegen_fn_attrs.inline.always() {
         if let (Some(no_sanitize_span), Some(inline_span)) = (no_sanitize_span, inline_span) {
             let hir_id = tcx.local_def_id_to_hir_id(did);
             tcx.node_span_lint(

compiler/rustc_feature/src/builtin_attrs.rs

@@ -1019,6 +1019,10 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         rustc_no_mir_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes,
         "#[rustc_no_mir_inline] prevents the MIR inliner from inlining a function while not affecting codegen"
     ),
+    rustc_attr!(
+        rustc_force_inline, Normal, template!(Word, NameValueStr: "reason"), WarnFollowing, EncodeCrossCrate::Yes,
+        "#![rustc_force_inline] forces a free function to be inlined"
+    ),
 
     // ==========================================================================
     // Internal attributes, Testing:

compiler/rustc_hir_typeck/src/coercion.rs

@@ -38,6 +38,7 @@
 use std::ops::Deref;
 
 use rustc_abi::ExternAbi;
+use rustc_attr_parsing::InlineAttr;
 use rustc_errors::codes::*;
 use rustc_errors::{Applicability, Diag, struct_span_code_err};
 use rustc_hir as hir;
@@ -926,8 +927,13 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
                         return Err(TypeError::IntrinsicCast);
                     }
 
-                    // Safe `#[target_feature]` functions are not assignable to safe fn pointers (RFC 2396).
+                    let fn_attrs = self.tcx.codegen_fn_attrs(def_id);
+                    if matches!(fn_attrs.inline, InlineAttr::Force { .. }) {
+                        return Err(TypeError::ForceInlineCast);
+                    }
 
+                    // Safe `#[target_feature]` functions are not assignable to safe fn pointers
+                    // (RFC 2396).
                     if b_hdr.safety.is_safe()
                         && !self.tcx.codegen_fn_attrs(def_id).target_features.is_empty()
                     {
@@ -1197,6 +1203,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             return Ok(prev_ty);
         }
 
+        let is_force_inline = |ty: Ty<'tcx>| {
+            if let ty::FnDef(did, _) = ty.kind() {
+                matches!(self.tcx.codegen_fn_attrs(did).inline, InlineAttr::Force { .. })
+            } else {
+                false
+            }
+        };
+        if is_force_inline(prev_ty) || is_force_inline(new_ty) {
+            return Err(TypeError::ForceInlineCast);
+        }
+
         // Special-case that coercion alone cannot handle:
         // Function items or non-capturing closures of differing IDs or GenericArgs.
         let (a_sig, b_sig) = {

compiler/rustc_middle/src/mir/mono.rs

@@ -132,9 +132,10 @@ impl<'tcx> MonoItem<'tcx> {
                 // creating one copy of this `#[inline]` function which may
                 // conflict with upstream crates as it could be an exported
                 // symbol.
-                match tcx.codegen_fn_attrs(instance.def_id()).inline {
-                    InlineAttr::Always => InstantiationMode::LocalCopy,
-                    _ => InstantiationMode::GloballyShared { may_conflict: true },
+                if tcx.codegen_fn_attrs(instance.def_id()).inline.always() {
+                    InstantiationMode::LocalCopy
+                } else {
+                    InstantiationMode::GloballyShared { may_conflict: true }
                 }
             }
             MonoItem::Static(..) | MonoItem::GlobalAsm(..) => {

compiler/rustc_middle/src/ty/error.rs

@@ -109,6 +109,9 @@ impl<'tcx> TypeError<'tcx> {
             TypeError::ConstMismatch(ref values) => {
                 format!("expected `{}`, found `{}`", values.expected, values.found).into()
             }
+            TypeError::ForceInlineCast => {
+                "cannot coerce functions which must be inlined to function pointers".into()
+            }
             TypeError::IntrinsicCast => "cannot coerce intrinsics to function pointers".into(),
             TypeError::TargetFeatureCast(_) => {
                 "cannot coerce functions with `#[target_feature]` to safe function pointers".into()

compiler/rustc_mir_build/Cargo.toml

@@ -12,6 +12,7 @@ rustc_abi = { path = "../rustc_abi" }
 rustc_apfloat = "0.2.0"
 rustc_arena = { path = "../rustc_arena" }
 rustc_ast = { path = "../rustc_ast" }
+rustc_attr_parsing = { path = "../rustc_attr_parsing" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }

compiler/rustc_mir_build/messages.ftl

@@ -118,6 +118,12 @@ mir_build_extern_static_requires_unsafe_unsafe_op_in_unsafe_fn_allowed =
     .note = extern statics are not controlled by the Rust type system: invalid data, aliasing violations or data races will cause undefined behavior
     .label = use of extern static
 
+mir_build_force_inline =
+    `{$callee}` is incompatible with `#[rustc_force_inline]`
+    .attr = annotation here
+    .callee = `{$callee}` defined here
+    .note = incompatible due to: {$reason}
+
 mir_build_inform_irrefutable = `let` bindings require an "irrefutable pattern", like a `struct` or an `enum` with only one variant
 
 mir_build_initializing_type_with_requires_unsafe =

compiler/rustc_mir_build/src/builder/mod.rs

@@ -29,6 +29,7 @@ use rustc_span::{Span, Symbol, sym};
 use super::lints;
 use crate::builder::expr::as_place::PlaceBuilder;
 use crate::builder::scope::DropKind;
+use crate::check_inline;
 
 pub(crate) fn closure_saved_names_of_captured_variables<'tcx>(
     tcx: TyCtxt<'tcx>,
@@ -80,6 +81,7 @@ pub(crate) fn mir_build<'tcx>(tcx: TyCtxtAt<'tcx>, def: LocalDefId) -> Body<'tcx
     };
 
     lints::check(tcx, &body);
+    check_inline::check_force_inline(tcx, &body);
 
     // The borrow checker will replace all the regions here with its own
     // inference variables. There's no point having non-erased regions here.

compiler/rustc_mir_build/src/check_inline.rs

@@ -0,0 +1,81 @@
+use rustc_attr_parsing::InlineAttr;
+use rustc_hir::def_id::DefId;
+use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
+use rustc_middle::mir::{Body, TerminatorKind};
+use rustc_middle::ty;
+use rustc_middle::ty::TyCtxt;
+use rustc_span::sym;
+
+/// Check that a body annotated with `#[rustc_force_inline]` will not fail to inline based on its
+/// definition alone (irrespective of any specific caller).
+pub(crate) fn check_force_inline<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) {
+    let def_id = body.source.def_id();
+    if !tcx.hir().body_owner_kind(def_id).is_fn_or_closure() || !def_id.is_local() {
+        return;
+    }
+    let InlineAttr::Force { attr_span, .. } = tcx.codegen_fn_attrs(def_id).inline else {
+        return;
+    };
+
+    if let Err(reason) =
+        is_inline_valid_on_fn(tcx, def_id).and_then(|_| is_inline_valid_on_body(tcx, body))
+    {
+        tcx.dcx().emit_err(crate::errors::InvalidForceInline {
+            attr_span,
+            callee_span: tcx.def_span(def_id),
+            callee: tcx.def_path_str(def_id),
+            reason,
+        });
+    }
+}
+
+pub fn is_inline_valid_on_fn<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> Result<(), &'static str> {
+    let codegen_attrs = tcx.codegen_fn_attrs(def_id);
+    if tcx.has_attr(def_id, sym::rustc_no_mir_inline) {
+        return Err("#[rustc_no_mir_inline]");
+    }
+
+    // FIXME(#127234): Coverage instrumentation currently doesn't handle inlined
+    // MIR correctly when Modified Condition/Decision Coverage is enabled.
+    if tcx.sess.instrument_coverage_mcdc() {
+        return Err("incompatible with MC/DC coverage");
+    }
+
+    let ty = tcx.type_of(def_id);
+    if match ty.instantiate_identity().kind() {
+        ty::FnDef(..) => tcx.fn_sig(def_id).instantiate_identity().c_variadic(),
+        ty::Closure(_, args) => args.as_closure().sig().c_variadic(),
+        _ => false,
+    } {
+        return Err("C variadic");
+    }
+
+    if codegen_attrs.flags.contains(CodegenFnAttrFlags::COLD) {
+        return Err("cold");
+    }
+
+    // Intrinsic fallback bodies are automatically made cross-crate inlineable,
+    // but at this stage we don't know whether codegen knows the intrinsic,
+    // so just conservatively don't inline it. This also ensures that we do not
+    // accidentally inline the body of an intrinsic that *must* be overridden.
+    if tcx.has_attr(def_id, sym::rustc_intrinsic) {
+        return Err("callee is an intrinsic");
+    }
+
+    Ok(())
+}
+
+pub fn is_inline_valid_on_body<'tcx>(
+    _: TyCtxt<'tcx>,
+    body: &Body<'tcx>,
+) -> Result<(), &'static str> {
+    if body
+        .basic_blocks
+        .iter()
+        .any(|bb| matches!(bb.terminator().kind, TerminatorKind::TailCall { .. }))
+    {
+        return Err("can't inline functions with tail calls");
+    }
+
+    Ok(())
+}

compiler/rustc_mir_build/src/errors.rs

@@ -1107,3 +1107,15 @@ impl<'a> Subdiagnostic for Rust2024IncompatiblePatSugg<'a> {
         );
     }
 }
+
+#[derive(Diagnostic)]
+#[diag(mir_build_force_inline)]
+#[note]
+pub(crate) struct InvalidForceInline {
+    #[primary_span]
+    pub attr_span: Span,
+    #[label(mir_build_callee)]
+    pub callee_span: Span,
+    pub callee: String,
+    pub reason: &'static str,
+}

compiler/rustc_mir_build/src/lib.rs

@@ -15,6 +15,7 @@
 // "Go to file" feature to silently ignore all files in the module, probably
 // because it assumes that "build" is a build-output directory. See #134365.
 mod builder;
+pub mod check_inline;
 mod check_tail_calls;
 mod check_unsafety;
 mod errors;

compiler/rustc_mir_transform/messages.ftl

@@ -19,6 +19,17 @@ mir_transform_ffi_unwind_call = call to {$foreign ->
 mir_transform_fn_item_ref = taking a reference to a function item does not give a function pointer
     .suggestion = cast `{$ident}` to obtain a function pointer
 
+mir_transform_force_inline =
+    `{$callee}` could not be inlined into `{$caller}` but is required to be inlined
+    .call = ...`{$callee}` called here
+    .attr = inlining due to this annotation
+    .caller = within `{$caller}`...
+    .callee = `{$callee}` defined here
+    .note = could not be inlined due to: {$reason}
+
+mir_transform_force_inline_justification =
+    `{$callee}` is required to be inlined to: {$sym}
+
 mir_transform_must_not_suspend = {$pre}`{$def_path}`{$post} held across a suspend point, but should not be
     .label = the value is held across this suspend point
     .note = {$reason}

compiler/rustc_mir_transform/src/cross_crate_inline.rs

@@ -46,7 +46,7 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     // #[inline(never)] to force code generation.
     match codegen_fn_attrs.inline {
         InlineAttr::Never => return false,
-        InlineAttr::Hint | InlineAttr::Always => return true,
+        InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. } => return true,
         _ => {}
     }
 
@@ -69,8 +69,9 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     // Don't do any inference if codegen optimizations are disabled and also MIR inlining is not
     // enabled. This ensures that we do inference even if someone only passes -Zinline-mir,
     // which is less confusing than having to also enable -Copt-level=1.
-    if matches!(tcx.sess.opts.optimize, OptLevel::No) && !pm::should_run_pass(tcx, &inline::Inline)
-    {
+    let inliner_will_run = pm::should_run_pass(tcx, &inline::Inline)
+        || inline::ForceInline::should_run_pass_for_callee(tcx, def_id.to_def_id());
+    if matches!(tcx.sess.opts.optimize, OptLevel::No) && !inliner_will_run {
         return false;
     }
 

compiler/rustc_mir_transform/src/errors.rs

@@ -4,8 +4,8 @@ use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
 use rustc_middle::mir::AssertKind;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::lint::{self, Lint};
-use rustc_span::Span;
 use rustc_span::def_id::DefId;
+use rustc_span::{Span, Symbol};
 
 use crate::fluent_generated as fluent;
 
@@ -142,3 +142,29 @@ pub(crate) struct MustNotSuspendReason {
 #[note(mir_transform_note2)]
 #[help]
 pub(crate) struct UndefinedTransmute;
+
+#[derive(Diagnostic)]
+#[diag(mir_transform_force_inline)]
+#[note]
+pub(crate) struct ForceInlineFailure {
+    #[label(mir_transform_caller)]
+    pub caller_span: Span,
+    #[label(mir_transform_callee)]
+    pub callee_span: Span,
+    #[label(mir_transform_attr)]
+    pub attr_span: Span,
+    #[primary_span]
+    #[label(mir_transform_call)]
+    pub call_span: Span,
+    pub callee: String,
+    pub caller: String,
+    pub reason: &'static str,
+    #[subdiagnostic]
+    pub justification: Option<ForceInlineJustification>,
+}
+
+#[derive(Subdiagnostic)]
+#[note(mir_transform_force_inline_justification)]
+pub(crate) struct ForceInlineJustification {
+    pub sym: Symbol,
+}

compiler/rustc_mir_transform/src/lib.rs

@@ -141,7 +141,7 @@ declare_passes! {
     mod gvn : GVN;
     // Made public so that `mir_drops_elaborated_and_const_checked` can be overridden
     // by custom rustc drivers, running all the steps by themselves. See #114628.
-    pub mod inline : Inline;
+    pub mod inline : Inline, ForceInline;
     mod instsimplify : InstSimplify { BeforeInline, AfterSimplifyCfg };
     mod jump_threading : JumpThreading;
     mod known_panics_lint : KnownPanicsLint;
@@ -488,7 +488,9 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> &
     let is_fn_like = tcx.def_kind(def).is_fn_like();
     if is_fn_like {
         // Do not compute the mir call graph without said call graph actually being used.
-        if pm::should_run_pass(tcx, &inline::Inline) {
+        if pm::should_run_pass(tcx, &inline::Inline)
+            || inline::ForceInline::should_run_pass_for_callee(tcx, def.to_def_id())
+        {
             tcx.ensure_with_value().mir_inliner_callees(ty::InstanceKind::Item(def.to_def_id()));
         }
     }
@@ -664,6 +666,8 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
             // Perform instsimplify before inline to eliminate some trivial calls (like clone
             // shims).
             &instsimplify::InstSimplify::BeforeInline,
+            // Perform inlining of `#[rustc_force_inline]`-annotated callees.
+            &inline::ForceInline,
             // Perform inlining, which may add a lot of code.
             &inline::Inline,
             // Code from other crates may have storage markers, so this needs to happen after

compiler/rustc_mir_transform/src/pass_manager.rs

@@ -79,6 +79,12 @@ pub(super) trait MirPass<'tcx> {
         true
     }
 
+    /// Returns `true` if this pass can be overridden by `-Zenable-mir-passes`. This should be
+    /// true for basically every pass other than those that are necessary for correctness.
+    fn can_be_overridden(&self) -> bool {
+        true
+    }
+
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>);
 
     fn is_mir_dump_enabled(&self) -> bool {
@@ -176,6 +182,10 @@ where
 {
     let name = pass.name();
 
+    if !pass.can_be_overridden() {
+        return pass.is_enabled(tcx.sess);
+    }
+
     let overridden_passes = &tcx.sess.opts.unstable_opts.mir_enable_passes;
     let overridden =
         overridden_passes.iter().rev().find(|(s, _)| s == &*name).map(|(_name, polarity)| {

compiler/rustc_mir_transform/src/shim.rs

@@ -20,7 +20,7 @@ use rustc_span::{DUMMY_SP, Span};
 use tracing::{debug, instrument};
 
 use crate::{
-    abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, deref_separator,
+    abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, deref_separator, inline,
     instsimplify, mentioned_items, pass_manager as pm, remove_noop_landing_pads, simplify,
 };
 
@@ -155,6 +155,8 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body<
             &remove_noop_landing_pads::RemoveNoopLandingPads,
             &simplify::SimplifyCfg::MakeShim,
             &instsimplify::InstSimplify::BeforeInline,
+            // Perform inlining of `#[rustc_force_inline]`-annotated callees.
+            &inline::ForceInline,
             &abort_unwinding_calls::AbortUnwindingCalls,
             &add_call_guards::CriticalCallEdges,
         ],

compiler/rustc_mir_transform/src/validate.rs

@@ -1,6 +1,7 @@
 //! Validates the MIR to ensure that invariants are upheld.
 
 use rustc_abi::{ExternAbi, FIRST_VARIANT, Size};
+use rustc_attr_parsing::InlineAttr;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_hir::LangItem;
 use rustc_index::IndexVec;
@@ -366,7 +367,8 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> {
                 self.check_edge(location, *target, EdgeKind::Normal);
                 self.check_unwind_edge(location, *unwind);
             }
-            TerminatorKind::Call { args, .. } | TerminatorKind::TailCall { args, .. } => {
+            TerminatorKind::Call { func, args, .. }
+            | TerminatorKind::TailCall { func, args, .. } => {
                 // FIXME(explicit_tail_calls): refactor this & add tail-call specific checks
                 if let TerminatorKind::Call { target, unwind, destination, .. } = terminator.kind {
                     if let Some(target) = target {
@@ -419,6 +421,13 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> {
                         }
                     }
                 }
+
+                if let ty::FnDef(did, ..) = func.ty(&self.body.local_decls, self.tcx).kind()
+                    && self.body.phase >= MirPhase::Runtime(RuntimePhase::Optimized)
+                    && matches!(self.tcx.codegen_fn_attrs(did).inline, InlineAttr::Force { .. })
+                {
+                    self.fail(location, "`#[rustc_force_inline]`-annotated function not inlined");
+                }
             }
             TerminatorKind::Assert { target, unwind, .. } => {
                 self.check_edge(location, *target, EdgeKind::Normal);

compiler/rustc_monomorphize/src/collector.rs

@@ -207,6 +207,7 @@
 
 use std::path::PathBuf;
 
+use rustc_attr_parsing::InlineAttr;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::sync::{LRef, MTLock, par_for_each_in};
 use rustc_data_structures::unord::{UnordMap, UnordSet};
@@ -224,8 +225,8 @@ use rustc_middle::ty::adjustment::{CustomCoerceUnsized, PointerCoercion};
 use rustc_middle::ty::layout::ValidityRequirement;
 use rustc_middle::ty::print::{shrunk_instance_name, with_no_trimmed_paths};
 use rustc_middle::ty::{
-    self, GenericArgs, GenericParamDefKind, Instance, InstanceKind, Ty, TyCtxt, TypeFoldable,
-    TypeVisitableExt, VtblEntry,
+    self, GenericArgs, GenericParamDefKind, Instance, InstanceKind, Interner, Ty, TyCtxt,
+    TypeFoldable, TypeVisitableExt, VtblEntry,
 };
 use rustc_middle::util::Providers;
 use rustc_middle::{bug, span_bug};
@@ -959,6 +960,14 @@ fn should_codegen_locally<'tcx>(tcx: TyCtxtAt<'tcx>, instance: Instance<'tcx>) -
         return false;
     }
 
+    if tcx.def_kind(def_id).has_codegen_attrs()
+        && matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. })
+    {
+        // `#[rustc_force_inline]` items should never be codegened. This should be caught by
+        // the MIR validator.
+        tcx.delay_bug("attempt to codegen `#[rustc_force_inline]` item");
+    }
+
     if def_id.is_local() {
         // Local items cannot be referred to locally without monomorphizing them locally.
         return true;
@@ -1453,7 +1462,9 @@ impl<'v> RootCollector<'_, 'v> {
     fn is_root(&self, def_id: LocalDefId) -> bool {
         !self.tcx.generics_of(def_id).requires_monomorphization(self.tcx)
             && match self.strategy {
-                MonoItemCollectionStrategy::Eager => true,
+                MonoItemCollectionStrategy::Eager => {
+                    !matches!(self.tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. })
+                }
                 MonoItemCollectionStrategy::Lazy => {
                     self.entry_fn.and_then(|(id, _)| id.as_local()) == Some(def_id)
                         || self.tcx.is_reachable_non_generic(def_id)

compiler/rustc_passes/messages.ftl

@@ -656,6 +656,14 @@ passes_rustc_allow_const_fn_unstable =
 passes_rustc_dirty_clean =
     attribute requires -Z query-dep-graph to be enabled
 
+passes_rustc_force_inline =
+    attribute should be applied to a function
+    .label = not a function definition
+
+passes_rustc_force_inline_coro =
+    attribute cannot be applied to a `async`, `gen` or `async gen` function
+    .label = `async`, `gen` or `async gen` function
+
 passes_rustc_layout_scalar_valid_range_arg =
     expected exactly one integer literal argument
 

compiler/rustc_passes/src/check_attr.rs

@@ -247,7 +247,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     self.check_coroutine(attr, target);
                 }
                 [sym::linkage, ..] => self.check_linkage(attr, span, target),
-                [sym::rustc_pub_transparent, ..] => self.check_rustc_pub_transparent( attr.span, span, attrs),
+                [sym::rustc_pub_transparent, ..] => self.check_rustc_pub_transparent(attr.span, span, attrs),
                 [
                     // ok
                     sym::allow
@@ -332,6 +332,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
 
         self.check_repr(attrs, span, target, item, hir_id);
         self.check_used(attrs, target, span);
+        self.check_rustc_force_inline(hir_id, attrs, span, target);
     }
 
     fn inline_attr_str_error_with_macro_def(&self, hir_id: HirId, attr: &Attribute, sym: &str) {
@@ -2480,6 +2481,45 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
         }
     }
 
+    fn check_rustc_force_inline(
+        &self,
+        hir_id: HirId,
+        attrs: &[Attribute],
+        span: Span,
+        target: Target,
+    ) {
+        let force_inline_attr = attrs.iter().find(|attr| attr.has_name(sym::rustc_force_inline));
+        match (target, force_inline_attr) {
+            (Target::Closure, None) => {
+                let is_coro = matches!(
+                    self.tcx.hir().expect_expr(hir_id).kind,
+                    hir::ExprKind::Closure(hir::Closure {
+                        kind: hir::ClosureKind::Coroutine(..)
+                            | hir::ClosureKind::CoroutineClosure(..),
+                        ..
+                    })
+                );
+                let parent_did = self.tcx.hir().get_parent_item(hir_id).to_def_id();
+                let parent_span = self.tcx.def_span(parent_did);
+                let parent_force_inline_attr =
+                    self.tcx.get_attr(parent_did, sym::rustc_force_inline);
+                if let Some(attr) = parent_force_inline_attr
+                    && is_coro
+                {
+                    self.dcx().emit_err(errors::RustcForceInlineCoro {
+                        attr_span: attr.span,
+                        span: parent_span,
+                    });
+                }
+            }
+            (Target::Fn, _) => (),
+            (_, Some(attr)) => {
+                self.dcx().emit_err(errors::RustcForceInline { attr_span: attr.span, span });
+            }
+            (_, None) => (),
+        }
+    }
+
     /// Checks if `#[autodiff]` is applied to an item other than a function item.
     fn check_autodiff(&self, _hir_id: HirId, _attr: &Attribute, span: Span, target: Target) {
         debug!("check_autodiff");

compiler/rustc_passes/src/errors.rs

@@ -688,6 +688,24 @@ pub(crate) struct RustcPubTransparent {
     pub span: Span,
 }
 
+#[derive(Diagnostic)]
+#[diag(passes_rustc_force_inline)]
+pub(crate) struct RustcForceInline {
+    #[primary_span]
+    pub attr_span: Span,
+    #[label]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(passes_rustc_force_inline_coro)]
+pub(crate) struct RustcForceInlineCoro {
+    #[primary_span]
+    pub attr_span: Span,
+    #[label]
+    pub span: Span,
+}
+
 #[derive(Diagnostic)]
 #[diag(passes_link_ordinal)]
 pub(crate) struct LinkOrdinal {

compiler/rustc_span/src/symbol.rs

@@ -1731,6 +1731,7 @@ symbols! {
         rustc_error,
         rustc_evaluate_where_clauses,
         rustc_expected_cgu_reuse,
+        rustc_force_inline,
         rustc_has_incoherent_inherent_impls,
         rustc_hidden_type_of_opaques,
         rustc_if_this_changed,

compiler/rustc_trait_selection/messages.ftl

@@ -251,7 +251,9 @@ trait_selection_no_value_in_rustc_on_unimplemented = this attribute must have a
 
 trait_selection_nothing = {""}
 
-trait_selection_oc_cant_coerce = cannot coerce intrinsics to function pointers
+trait_selection_oc_cant_coerce_force_inline =
+    cannot coerce functions which must be inlined to function pointers
+trait_selection_oc_cant_coerce_intrinsic = cannot coerce intrinsics to function pointers
 trait_selection_oc_closure_selfref = closure/coroutine type that references itself
 trait_selection_oc_const_compat = const not compatible with trait
 trait_selection_oc_fn_lang_correct_type = {$lang_item_name ->

compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs

@@ -2294,7 +2294,7 @@ impl<'tcx> ObligationCause<'tcx> {
                 {
                     FailureCode::Error0644
                 }
-                TypeError::IntrinsicCast => FailureCode::Error0308,
+                TypeError::IntrinsicCast | TypeError::ForceInlineCast => FailureCode::Error0308,
                 _ => FailureCode::Error0308,
             },
         }
@@ -2360,8 +2360,11 @@ impl<'tcx> ObligationCause<'tcx> {
                 {
                     ObligationCauseFailureCode::ClosureSelfref { span }
                 }
+                TypeError::ForceInlineCast => {
+                    ObligationCauseFailureCode::CantCoerceForceInline { span, subdiags }
+                }
                 TypeError::IntrinsicCast => {
-                    ObligationCauseFailureCode::CantCoerce { span, subdiags }
+                    ObligationCauseFailureCode::CantCoerceIntrinsic { span, subdiags }
                 }
                 _ => ObligationCauseFailureCode::Generic { span, subdiags },
             },

compiler/rustc_trait_selection/src/errors.rs

@@ -1729,8 +1729,15 @@ pub enum ObligationCauseFailureCode {
         #[primary_span]
         span: Span,
     },
-    #[diag(trait_selection_oc_cant_coerce, code = E0308)]
-    CantCoerce {
+    #[diag(trait_selection_oc_cant_coerce_force_inline, code = E0308)]
+    CantCoerceForceInline {
+        #[primary_span]
+        span: Span,
+        #[subdiagnostic]
+        subdiags: Vec<TypeErrorAdditionalDiags>,
+    },
+    #[diag(trait_selection_oc_cant_coerce_intrinsic, code = E0308)]
+    CantCoerceIntrinsic {
         #[primary_span]
         span: Span,
         #[subdiagnostic]

compiler/rustc_type_ir/src/error.rs

@@ -51,6 +51,9 @@ pub enum TypeError<I: Interner> {
     ConstMismatch(ExpectedFound<I::Const>),
 
     IntrinsicCast,
+    /// `#[rustc_force_inline]` functions must be inlined and must not be codegened independently,
+    /// so casting to a function pointer must be prohibited.
+    ForceInlineCast,
     /// Safe `#[target_feature]` functions are not assignable to safe function pointers.
     TargetFeatureCast(I::DefId),
 }
@@ -83,6 +86,7 @@ impl<I: Interner> TypeError<I> {
             | ProjectionMismatched(_)
             | ExistentialMismatch(_)
             | ConstMismatch(_)
+            | ForceInlineCast
             | IntrinsicCast => true,
         }
     }

tests/mir-opt/inline/forced.caller.ForceInline.panic-abort.diff

@@ -0,0 +1,21 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+
+  fn caller() -> () {
+      let mut _0: ();
+      let _1: ();
++     scope 1 (inlined callee_forced) {
++     }
+
+      bb0: {
+          StorageLive(_1);
+-         _1 = callee_forced() -> [return: bb1, unwind unreachable];
+-     }
+- 
+-     bb1: {
+          StorageDead(_1);
+          _0 = const ();
+          return;
+      }
+  }
+

tests/mir-opt/inline/forced.caller.ForceInline.panic-unwind.diff

@@ -0,0 +1,21 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+
+  fn caller() -> () {
+      let mut _0: ();
+      let _1: ();
++     scope 1 (inlined callee_forced) {
++     }
+
+      bb0: {
+          StorageLive(_1);
+-         _1 = callee_forced() -> [return: bb1, unwind continue];
+-     }
+- 
+-     bb1: {
+          StorageDead(_1);
+          _0 = const ();
+          return;
+      }
+  }
+

tests/mir-opt/inline/forced.rs

@@ -0,0 +1,13 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+//@ compile-flags: -Copt-level=0 --crate-type=lib
+#![feature(rustc_attrs)]
+
+#[rustc_force_inline]
+pub fn callee_forced() {}
+
+// EMIT_MIR forced.caller.ForceInline.diff
+pub fn caller() {
+    callee_forced();
+    // CHECK-LABEL: fn caller(
+    // CHECK: (inlined callee_forced)
+}

tests/mir-opt/inline/forced_async.caller.ForceInline.panic-abort.diff

@@ -0,0 +1,12 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+
+  fn caller() -> {async fn body of caller()} {
+      let mut _0: {async fn body of caller()};
+
+      bb0: {
+          _0 = {coroutine@$DIR/forced_async.rs:10:19: 14:2 (#0)};
+          return;
+      }
+  }
+

tests/mir-opt/inline/forced_async.caller.ForceInline.panic-unwind.diff

@@ -0,0 +1,12 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+
+  fn caller() -> {async fn body of caller()} {
+      let mut _0: {async fn body of caller()};
+
+      bb0: {
+          _0 = {coroutine@$DIR/forced_async.rs:10:19: 14:2 (#0)};
+          return;
+      }
+  }
+

tests/mir-opt/inline/forced_async.rs

@@ -0,0 +1,14 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+//@ compile-flags: -Copt-level=0 --crate-type=lib
+//@ edition: 2021
+#![feature(rustc_attrs)]
+
+#[rustc_force_inline]
+pub fn callee_forced() {}
+
+// EMIT_MIR forced_async.caller.ForceInline.diff
+async fn caller() {
+    callee_forced();
+    // CHECK-LABEL: fn caller(
+    // CHECK: (inlined callee_forced)
+}

tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.panic-abort.diff

@@ -0,0 +1,21 @@
+- // MIR for `caller::{closure#0}` before ForceInline
++ // MIR for `caller::{closure#0}` after ForceInline
+
+  fn caller::{closure#0}(_1: &{closure@$DIR/forced_closure.rs:10:6: 10:8}) -> () {
+      let mut _0: ();
+      let _2: ();
++     scope 1 (inlined callee_forced) {
++     }
+
+      bb0: {
+          StorageLive(_2);
+-         _2 = callee_forced() -> [return: bb1, unwind unreachable];
+-     }
+- 
+-     bb1: {
+          StorageDead(_2);
+          _0 = const ();
+          return;
+      }
+  }
+

tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.panic-unwind.diff

@@ -0,0 +1,21 @@
+- // MIR for `caller::{closure#0}` before ForceInline
++ // MIR for `caller::{closure#0}` after ForceInline
+
+  fn caller::{closure#0}(_1: &{closure@$DIR/forced_closure.rs:10:6: 10:8}) -> () {
+      let mut _0: ();
+      let _2: ();
++     scope 1 (inlined callee_forced) {
++     }
+
+      bb0: {
+          StorageLive(_2);
+-         _2 = callee_forced() -> [return: bb1, unwind continue];
+-     }
+- 
+-     bb1: {
+          StorageDead(_2);
+          _0 = const ();
+          return;
+      }
+  }
+

tests/mir-opt/inline/forced_closure.rs

@@ -0,0 +1,15 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+//@ compile-flags: -Copt-level=0 --crate-type=lib
+#![feature(rustc_attrs)]
+
+#[rustc_force_inline]
+pub fn callee_forced() {}
+
+// EMIT_MIR forced_closure.caller-{closure#0}.ForceInline.diff
+pub fn caller() {
+    (|| {
+        callee_forced();
+        // CHECK-LABEL: fn caller::{closure#0}(
+        // CHECK: (inlined callee_forced)
+    })();
+}

tests/mir-opt/inline/forced_dead_code.caller.ForceInline.panic-abort.diff

@@ -0,0 +1,21 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+
+  fn caller() -> () {
+      let mut _0: ();
+      let _1: ();
++     scope 1 (inlined callee_forced) {
++     }
+
+      bb0: {
+          StorageLive(_1);
+-         _1 = callee_forced() -> [return: bb1, unwind unreachable];
+-     }
+- 
+-     bb1: {
+          StorageDead(_1);
+          _0 = const ();
+          return;
+      }
+  }
+

tests/mir-opt/inline/forced_dead_code.caller.ForceInline.panic-unwind.diff

@@ -0,0 +1,21 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+
+  fn caller() -> () {
+      let mut _0: ();
+      let _1: ();
++     scope 1 (inlined callee_forced) {
++     }
+
+      bb0: {
+          StorageLive(_1);
+-         _1 = callee_forced() -> [return: bb1, unwind continue];
+-     }
+- 
+-     bb1: {
+          StorageDead(_1);
+          _0 = const ();
+          return;
+      }
+  }
+

tests/mir-opt/inline/forced_dead_code.rs

@@ -0,0 +1,17 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+//@ compile-flags: -Copt-level=0 -Clink-dead-code
+#![feature(rustc_attrs)]
+
+#[rustc_force_inline]
+pub fn callee_forced() {}
+
+// EMIT_MIR forced_dead_code.caller.ForceInline.diff
+pub fn caller() {
+    callee_forced();
+    // CHECK-LABEL: fn caller(
+    // CHECK: (inlined callee_forced)
+}
+
+fn main() {
+    caller();
+}

tests/ui/force-inlining/asm.rs

@@ -0,0 +1,68 @@
+//@ build-fail
+//@ compile-flags: --crate-type=lib --target thumbv4t-none-eabi
+//@ needs-llvm-components: arm
+
+// Checks that forced inlining won't mix asm with incompatible instruction sets.
+
+#![crate_type = "lib"]
+#![feature(rustc_attrs)]
+#![feature(no_core, lang_items)]
+#![no_core]
+
+#[lang = "sized"]
+pub trait Sized {}
+#[lang = "copy"]
+pub trait Copy {}
+#[lang = "freeze"]
+pub unsafe trait Freeze {}
+
+#[lang = "start"]
+fn start<T>(_main: fn() -> T, _argc: isize, _argv: *const *const u8, _sigpipe: u8) -> isize {
+    0
+}
+
+#[rustc_builtin_macro]
+#[macro_export]
+macro_rules! asm {
+    ("assembly template",
+        $(operands,)*
+        $(options($(option),*))?
+    ) => {
+        /* compiler built-in */
+    };
+}
+
+#[instruction_set(arm::a32)]
+#[rustc_force_inline]
+fn instruction_set_a32() {}
+
+#[instruction_set(arm::t32)]
+#[rustc_force_inline]
+fn instruction_set_t32() {}
+
+#[rustc_force_inline]
+fn instruction_set_default() {}
+
+#[rustc_force_inline]
+fn inline_always_and_using_inline_asm() {
+    unsafe { asm!("/* do nothing */") };
+}
+
+#[instruction_set(arm::t32)]
+pub fn t32() {
+    instruction_set_a32();
+//~^ ERROR `instruction_set_a32` could not be inlined into `t32` but is required to be inlined
+    instruction_set_t32();
+    instruction_set_default();
+    inline_always_and_using_inline_asm();
+//~^ ERROR `inline_always_and_using_inline_asm` could not be inlined into `t32` but is required to be inlined
+}
+
+pub fn default() {
+    instruction_set_a32();
+//~^ ERROR `instruction_set_a32` could not be inlined into `default` but is required to be inlined
+    instruction_set_t32();
+//~^ ERROR `instruction_set_t32` could not be inlined into `default` but is required to be inlined
+    instruction_set_default();
+    inline_always_and_using_inline_asm();
+}

tests/ui/force-inlining/asm.stderr

@@ -0,0 +1,34 @@
+error: `instruction_set_a32` could not be inlined into `t32` but is required to be inlined
+  --> $DIR/asm.rs:53:5
+   |
+LL |     instruction_set_a32();
+   |     ^^^^^^^^^^^^^^^^^^^^^ ...`instruction_set_a32` called here
+   |
+   = note: could not be inlined due to: incompatible instruction set
+
+error: `inline_always_and_using_inline_asm` could not be inlined into `t32` but is required to be inlined
+  --> $DIR/asm.rs:57:5
+   |
+LL |     inline_always_and_using_inline_asm();
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...`inline_always_and_using_inline_asm` called here
+   |
+   = note: could not be inlined due to: cannot move inline-asm across instruction sets
+
+error: `instruction_set_a32` could not be inlined into `default` but is required to be inlined
+  --> $DIR/asm.rs:62:5
+   |
+LL |     instruction_set_a32();
+   |     ^^^^^^^^^^^^^^^^^^^^^ ...`instruction_set_a32` called here
+   |
+   = note: could not be inlined due to: incompatible instruction set
+
+error: `instruction_set_t32` could not be inlined into `default` but is required to be inlined
+  --> $DIR/asm.rs:64:5
+   |
+LL |     instruction_set_t32();
+   |     ^^^^^^^^^^^^^^^^^^^^^ ...`instruction_set_t32` called here
+   |
+   = note: could not be inlined due to: incompatible instruction set
+
+error: aborting due to 4 previous errors
+

tests/ui/force-inlining/auxiliary/callees.rs

@@ -0,0 +1,10 @@
+//@ compile-flags: --crate-type=lib
+#![feature(rustc_attrs)]
+
+#[rustc_force_inline = "the test requires it"]
+pub fn forced_with_reason() {
+}
+
+#[rustc_force_inline]
+pub fn forced() {
+}

tests/ui/force-inlining/cast.rs

@@ -0,0 +1,25 @@
+//@ check-fail
+//@ compile-flags: --crate-type=lib
+#![allow(internal_features)]
+#![feature(rustc_attrs)]
+
+#[rustc_force_inline]
+pub fn callee(x: isize) -> usize { unimplemented!() }
+
+fn a() {
+    let _: fn(isize) -> usize = callee;
+//~^ ERROR cannot coerce functions which must be inlined to function pointers
+}
+
+fn b() {
+    let _ = callee as fn(isize) -> usize;
+//~^ ERROR non-primitive cast
+}
+
+fn c() {
+    let _: [fn(isize) -> usize; 2] = [
+        callee,
+//~^ ERROR cannot coerce functions which must be inlined to function pointers
+        callee,
+    ];
+}

tests/ui/force-inlining/cast.stderr

@@ -0,0 +1,40 @@
+error[E0308]: cannot coerce functions which must be inlined to function pointers
+  --> $DIR/cast.rs:10:33
+   |
+LL |     let _: fn(isize) -> usize = callee;
+   |            ------------------   ^^^^^^ cannot coerce functions which must be inlined to function pointers
+   |            |
+   |            expected due to this
+   |
+   = note: expected fn pointer `fn(_) -> _`
+                 found fn item `fn(_) -> _ {callee}`
+   = note: fn items are distinct from fn pointers
+help: consider casting to a fn pointer
+   |
+LL |     let _: fn(isize) -> usize = callee as fn(isize) -> usize;
+   |                                 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+error[E0605]: non-primitive cast: `fn(isize) -> usize {callee}` as `fn(isize) -> usize`
+  --> $DIR/cast.rs:15:13
+   |
+LL |     let _ = callee as fn(isize) -> usize;
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ invalid cast
+
+error[E0308]: cannot coerce functions which must be inlined to function pointers
+  --> $DIR/cast.rs:21:9
+   |
+LL |         callee,
+   |         ^^^^^^ cannot coerce functions which must be inlined to function pointers
+   |
+   = note: expected fn pointer `fn(_) -> _`
+                 found fn item `fn(_) -> _ {callee}`
+   = note: fn items are distinct from fn pointers
+help: consider casting to a fn pointer
+   |
+LL |         callee as fn(isize) -> usize,
+   |         ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+error: aborting due to 3 previous errors
+
+Some errors have detailed explanations: E0308, E0605.
+For more information about an error, try `rustc --explain E0308`.

tests/ui/force-inlining/cross-crate.rs

@@ -0,0 +1,13 @@
+//@ aux-build:callees.rs
+//@ build-pass
+//@ compile-flags: --crate-type=lib
+
+extern crate callees;
+
+// Test that forced inlining across crates works as expected.
+
+pub fn caller() {
+    callees::forced();
+
+    callees::forced_with_reason();
+}

tests/ui/force-inlining/deny-async.rs

@@ -0,0 +1,24 @@
+//@ check-fail
+//@ compile-flags: --crate-type=lib
+//@ edition: 2021
+#![allow(internal_features)]
+#![feature(rustc_attrs)]
+
+// Test that forced inlining into async functions w/ errors works as expected.
+
+#[rustc_no_mir_inline]
+#[rustc_force_inline]
+//~^ ERROR `callee` is incompatible with `#[rustc_force_inline]`
+pub fn callee() {
+}
+
+#[rustc_no_mir_inline]
+#[rustc_force_inline = "the test requires it"]
+//~^ ERROR `callee_justified` is incompatible with `#[rustc_force_inline]`
+pub fn callee_justified() {
+}
+
+async fn async_caller() {
+    callee();
+    callee_justified();
+}

tests/ui/force-inlining/deny-async.stderr

@@ -0,0 +1,24 @@
+error: `callee` is incompatible with `#[rustc_force_inline]`
+  --> $DIR/deny-async.rs:10:1
+   |
+LL | #[rustc_force_inline]
+   | ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | pub fn callee() {
+   | --------------- `callee` defined here
+   |
+   = note: incompatible due to: #[rustc_no_mir_inline]
+
+error: `callee_justified` is incompatible with `#[rustc_force_inline]`
+  --> $DIR/deny-async.rs:16:1
+   |
+LL | #[rustc_force_inline = "the test requires it"]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | pub fn callee_justified() {
+   | ------------------------- `callee_justified` defined here
+   |
+   = note: incompatible due to: #[rustc_no_mir_inline]
+
+error: aborting due to 2 previous errors
+

tests/ui/force-inlining/deny-closure.rs

@@ -0,0 +1,25 @@
+//@ check-fail
+//@ compile-flags: --crate-type=lib
+#![allow(internal_features)]
+#![feature(rustc_attrs)]
+
+// Test that forced inlining into closures w/ errors works as expected.
+
+#[rustc_no_mir_inline]
+#[rustc_force_inline]
+//~^ ERROR `callee` is incompatible with `#[rustc_force_inline]`
+pub fn callee() {
+}
+
+#[rustc_no_mir_inline]
+#[rustc_force_inline = "the test requires it"]
+//~^ ERROR `callee_justified` is incompatible with `#[rustc_force_inline]`
+pub fn callee_justified() {
+}
+
+pub fn caller() {
+    (|| {
+        callee();
+        callee_justified();
+    })();
+}

tests/ui/force-inlining/deny-closure.stderr

@@ -0,0 +1,24 @@
+error: `callee` is incompatible with `#[rustc_force_inline]`
+  --> $DIR/deny-closure.rs:9:1
+   |
+LL | #[rustc_force_inline]
+   | ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | pub fn callee() {
+   | --------------- `callee` defined here
+   |
+   = note: incompatible due to: #[rustc_no_mir_inline]
+
+error: `callee_justified` is incompatible with `#[rustc_force_inline]`
+  --> $DIR/deny-closure.rs:15:1
+   |
+LL | #[rustc_force_inline = "the test requires it"]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | pub fn callee_justified() {
+   | ------------------------- `callee_justified` defined here
+   |
+   = note: incompatible due to: #[rustc_no_mir_inline]
+
+error: aborting due to 2 previous errors
+

tests/ui/force-inlining/deny.rs

@@ -0,0 +1,23 @@
+//@ check-fail
+//@ compile-flags: --crate-type=lib
+#![allow(internal_features)]
+#![feature(rustc_attrs)]
+
+// Test that forced inlining w/ errors works as expected.
+
+#[rustc_no_mir_inline]
+#[rustc_force_inline]
+//~^ ERROR `callee` is incompatible with `#[rustc_force_inline]`
+pub fn callee() {
+}
+
+#[rustc_no_mir_inline]
+#[rustc_force_inline = "the test requires it"]
+//~^ ERROR `callee_justified` is incompatible with `#[rustc_force_inline]`
+pub fn callee_justified() {
+}
+
+pub fn caller() {
+    callee();
+    callee_justified();
+}

tests/ui/force-inlining/deny.stderr

@@ -0,0 +1,24 @@
+error: `callee` is incompatible with `#[rustc_force_inline]`
+  --> $DIR/deny.rs:9:1
+   |
+LL | #[rustc_force_inline]
+   | ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | pub fn callee() {
+   | --------------- `callee` defined here
+   |
+   = note: incompatible due to: #[rustc_no_mir_inline]
+
+error: `callee_justified` is incompatible with `#[rustc_force_inline]`
+  --> $DIR/deny.rs:15:1
+   |
+LL | #[rustc_force_inline = "the test requires it"]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | pub fn callee_justified() {
+   | ------------------------- `callee_justified` defined here
+   |
+   = note: incompatible due to: #[rustc_no_mir_inline]
+
+error: aborting due to 2 previous errors
+

tests/ui/force-inlining/early-deny.rs

@@ -0,0 +1,21 @@
+//@ check-fail
+//@ compile-flags: --crate-type=lib
+#![feature(c_variadic)]
+#![feature(rustc_attrs)]
+
+#[rustc_no_mir_inline]
+#[rustc_force_inline]
+//~^ ERROR `rustc_attr` is incompatible with `#[rustc_force_inline]`
+pub fn rustc_attr() {
+}
+
+#[cold]
+#[rustc_force_inline]
+//~^ ERROR `cold` is incompatible with `#[rustc_force_inline]`
+pub fn cold() {
+}
+
+#[rustc_force_inline]
+//~^ ERROR `variadic` is incompatible with `#[rustc_force_inline]`
+pub unsafe extern "C" fn variadic(args: ...) {
+}

tests/ui/force-inlining/early-deny.stderr

@@ -0,0 +1,35 @@
+error: `rustc_attr` is incompatible with `#[rustc_force_inline]`
+  --> $DIR/early-deny.rs:7:1
+   |
+LL | #[rustc_force_inline]
+   | ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | pub fn rustc_attr() {
+   | ------------------- `rustc_attr` defined here
+   |
+   = note: incompatible due to: #[rustc_no_mir_inline]
+
+error: `cold` is incompatible with `#[rustc_force_inline]`
+  --> $DIR/early-deny.rs:13:1
+   |
+LL | #[rustc_force_inline]
+   | ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | pub fn cold() {
+   | ------------- `cold` defined here
+   |
+   = note: incompatible due to: cold
+
+error: `variadic` is incompatible with `#[rustc_force_inline]`
+  --> $DIR/early-deny.rs:18:1
+   |
+LL | #[rustc_force_inline]
+   | ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | pub unsafe extern "C" fn variadic(args: ...) {
+   | -------------------------------------------- `variadic` defined here
+   |
+   = note: incompatible due to: C variadic
+
+error: aborting due to 3 previous errors
+

tests/ui/force-inlining/gate.rs

@@ -0,0 +1,12 @@
+//@ compile-flags: --crate-type=lib
+#![allow(internal_features)]
+
+#[rustc_force_inline]
+//~^ ERROR #![rustc_force_inline] forces a free function to be inlined
+pub fn bare() {
+}
+
+#[rustc_force_inline = "the test requires it"]
+//~^ ERROR #![rustc_force_inline] forces a free function to be inlined
+pub fn justified() {
+}

tests/ui/force-inlining/gate.stderr

@@ -0,0 +1,21 @@
+error[E0658]: #![rustc_force_inline] forces a free function to be inlined
+  --> $DIR/gate.rs:4:1
+   |
+LL | #[rustc_force_inline]
+   | ^^^^^^^^^^^^^^^^^^^^^
+   |
+   = help: add `#![feature(rustc_attrs)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error[E0658]: #![rustc_force_inline] forces a free function to be inlined
+  --> $DIR/gate.rs:9:1
+   |
+LL | #[rustc_force_inline = "the test requires it"]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = help: add `#![feature(rustc_attrs)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0658`.

tests/ui/force-inlining/invalid.rs

@@ -0,0 +1,164 @@
+//@ edition: 2024
+#![allow(internal_features, unused_imports, unused_macros)]
+#![feature(extern_types)]
+#![feature(gen_blocks)]
+#![feature(rustc_attrs)]
+#![feature(stmt_expr_attributes)]
+#![feature(trait_alias)]
+
+// Test that invalid force inlining attributes error as expected.
+
+#[rustc_force_inline("foo")]
+//~^ ERROR malformed `rustc_force_inline` attribute input
+pub fn forced1() {
+}
+
+#[rustc_force_inline(bar, baz)]
+//~^ ERROR malformed `rustc_force_inline` attribute input
+pub fn forced2() {
+}
+
+#[rustc_force_inline(2)]
+//~^ ERROR malformed `rustc_force_inline` attribute input
+pub fn forced3() {
+}
+
+#[rustc_force_inline = 2]
+//~^ ERROR malformed `rustc_force_inline` attribute input
+pub fn forced4() {
+}
+
+#[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+extern crate std as other_std;
+
+#[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+use std::collections::HashMap;
+
+#[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+static _FOO: &'static str = "FOO";
+
+#[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+const _BAR: u32 = 3;
+
+#[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+mod foo { }
+
+#[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+unsafe extern "C" {
+    #[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+    static X: &'static u32;
+
+    #[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+    type Y;
+
+    #[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+    fn foo();
+}
+
+#[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+type Foo = u32;
+
+#[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+enum Bar<#[rustc_force_inline] T> {
+//~^ ERROR attribute should be applied to a function
+    #[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+    Baz(std::marker::PhantomData<T>),
+}
+
+#[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+struct Qux {
+    #[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+    field: u32,
+}
+
+#[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+union FooBar {
+    x: u32,
+    y: u32,
+}
+
+#[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+trait FooBaz {
+    #[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+    type Foo;
+    #[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+    const Bar: i32;
+
+    #[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+    fn foo() {}
+}
+
+#[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+trait FooQux = FooBaz;
+
+#[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+impl<T> Bar<T> {
+    #[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+    fn foo() {}
+}
+
+#[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+impl<T> FooBaz for Bar<T> {
+    type Foo = u32;
+    const Bar: i32 = 3;
+}
+
+#[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+macro_rules! barqux { ($foo:tt) => { $foo }; }
+
+fn barqux(#[rustc_force_inline] _x: u32) {}
+//~^ ERROR allow, cfg, cfg_attr, deny, expect, forbid, and warn are the only allowed built-in attributes in function parameters
+//~^^ ERROR attribute should be applied to a function
+
+#[rustc_force_inline]
+//~^ ERROR attribute cannot be applied to a `async`, `gen` or `async gen` function
+async fn async_foo() {}
+
+#[rustc_force_inline]
+//~^ ERROR attribute cannot be applied to a `async`, `gen` or `async gen` function
+gen fn gen_foo() {}
+
+#[rustc_force_inline]
+//~^ ERROR attribute cannot be applied to a `async`, `gen` or `async gen` function
+async gen fn async_gen_foo() {}
+
+fn main() {
+    let _x = #[rustc_force_inline] || { };
+//~^ ERROR attribute should be applied to a function
+    let _y = #[rustc_force_inline] 3 + 4;
+//~^ ERROR attribute should be applied to a function
+    #[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+    let _z = 3;
+
+    match _z {
+        #[rustc_force_inline]
+//~^ ERROR attribute should be applied to a function
+        1 => (),
+        _ => (),
+    }
+}

tests/ui/force-inlining/invalid.stderr

@@ -0,0 +1,377 @@
+error: malformed `rustc_force_inline` attribute input
+  --> $DIR/invalid.rs:11:1
+   |
+LL | #[rustc_force_inline("foo")]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: the following are the possible correct uses
+   |
+LL | #[rustc_force_inline = "reason"]
+   |
+LL | #[rustc_force_inline]
+   |
+
+error: malformed `rustc_force_inline` attribute input
+  --> $DIR/invalid.rs:16:1
+   |
+LL | #[rustc_force_inline(bar, baz)]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: the following are the possible correct uses
+   |
+LL | #[rustc_force_inline = "reason"]
+   |
+LL | #[rustc_force_inline]
+   |
+
+error: malformed `rustc_force_inline` attribute input
+  --> $DIR/invalid.rs:21:1
+   |
+LL | #[rustc_force_inline(2)]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: the following are the possible correct uses
+   |
+LL | #[rustc_force_inline = "reason"]
+   |
+LL | #[rustc_force_inline]
+   |
+
+error: malformed `rustc_force_inline` attribute input
+  --> $DIR/invalid.rs:26:1
+   |
+LL | #[rustc_force_inline = 2]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+help: the following are the possible correct uses
+   |
+LL | #[rustc_force_inline = "reason"]
+   |
+LL | #[rustc_force_inline]
+   |
+
+error: allow, cfg, cfg_attr, deny, expect, forbid, and warn are the only allowed built-in attributes in function parameters
+  --> $DIR/invalid.rs:133:11
+   |
+LL | fn barqux(#[rustc_force_inline] _x: u32) {}
+   |           ^^^^^^^^^^^^^^^^^^^^^
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:31:1
+   |
+LL | #[rustc_force_inline]
+   | ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | extern crate std as other_std;
+   | ------------------------------ not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:35:1
+   |
+LL | #[rustc_force_inline]
+   | ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | use std::collections::HashMap;
+   | ------------------------------ not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:39:1
+   |
+LL | #[rustc_force_inline]
+   | ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | static _FOO: &'static str = "FOO";
+   | ---------------------------------- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:43:1
+   |
+LL | #[rustc_force_inline]
+   | ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | const _BAR: u32 = 3;
+   | -------------------- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:47:1
+   |
+LL | #[rustc_force_inline]
+   | ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | mod foo { }
+   | ----------- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:51:1
+   |
+LL |   #[rustc_force_inline]
+   |   ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | / unsafe extern "C" {
+LL | |     #[rustc_force_inline]
+LL | |
+LL | |     static X: &'static u32;
+...  |
+LL | |     fn foo();
+LL | | }
+   | |_- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:67:1
+   |
+LL | #[rustc_force_inline]
+   | ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | type Foo = u32;
+   | --------------- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:71:1
+   |
+LL |   #[rustc_force_inline]
+   |   ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | / enum Bar<#[rustc_force_inline] T> {
+LL | |
+LL | |     #[rustc_force_inline]
+...  |
+LL | | }
+   | |_- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:73:10
+   |
+LL | enum Bar<#[rustc_force_inline] T> {
+   |          ^^^^^^^^^^^^^^^^^^^^^ - not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:75:5
+   |
+LL |     #[rustc_force_inline]
+   |     ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL |     Baz(std::marker::PhantomData<T>),
+   |     -------------------------------- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:80:1
+   |
+LL |   #[rustc_force_inline]
+   |   ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | / struct Qux {
+LL | |     #[rustc_force_inline]
+LL | |
+LL | |     field: u32,
+LL | | }
+   | |_- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:83:5
+   |
+LL |     #[rustc_force_inline]
+   |     ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL |     field: u32,
+   |     ---------- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:88:1
+   |
+LL |   #[rustc_force_inline]
+   |   ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | / union FooBar {
+LL | |     x: u32,
+LL | |     y: u32,
+LL | | }
+   | |_- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:95:1
+   |
+LL |   #[rustc_force_inline]
+   |   ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | / trait FooBaz {
+LL | |     #[rustc_force_inline]
+LL | |
+LL | |     type Foo;
+...  |
+LL | |     fn foo() {}
+LL | | }
+   | |_- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:110:1
+   |
+LL | #[rustc_force_inline]
+   | ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | trait FooQux = FooBaz;
+   | ---------------------- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:114:1
+   |
+LL |   #[rustc_force_inline]
+   |   ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | / impl<T> Bar<T> {
+LL | |     #[rustc_force_inline]
+LL | |
+LL | |     fn foo() {}
+LL | | }
+   | |_- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:122:1
+   |
+LL |   #[rustc_force_inline]
+   |   ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | / impl<T> FooBaz for Bar<T> {
+LL | |     type Foo = u32;
+LL | |     const Bar: i32 = 3;
+LL | | }
+   | |_- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:129:1
+   |
+LL | #[rustc_force_inline]
+   | ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | macro_rules! barqux { ($foo:tt) => { $foo }; }
+   | ---------------------------------------------- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:133:11
+   |
+LL | fn barqux(#[rustc_force_inline] _x: u32) {}
+   |           ^^^^^^^^^^^^^^^^^^^^^--------
+   |           |
+   |           not a function definition
+
+error: attribute cannot be applied to a `async`, `gen` or `async gen` function
+  --> $DIR/invalid.rs:137:1
+   |
+LL | #[rustc_force_inline]
+   | ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | async fn async_foo() {}
+   | -------------------- `async`, `gen` or `async gen` function
+
+error: attribute cannot be applied to a `async`, `gen` or `async gen` function
+  --> $DIR/invalid.rs:141:1
+   |
+LL | #[rustc_force_inline]
+   | ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | gen fn gen_foo() {}
+   | ---------------- `async`, `gen` or `async gen` function
+
+error: attribute cannot be applied to a `async`, `gen` or `async gen` function
+  --> $DIR/invalid.rs:145:1
+   |
+LL | #[rustc_force_inline]
+   | ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL | async gen fn async_gen_foo() {}
+   | ---------------------------- `async`, `gen` or `async gen` function
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:150:14
+   |
+LL |     let _x = #[rustc_force_inline] || { };
+   |              ^^^^^^^^^^^^^^^^^^^^^ ------ not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:152:14
+   |
+LL |     let _y = #[rustc_force_inline] 3 + 4;
+   |              ^^^^^^^^^^^^^^^^^^^^^ - not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:154:5
+   |
+LL |     #[rustc_force_inline]
+   |     ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL |     let _z = 3;
+   |     ----------- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:159:9
+   |
+LL |         #[rustc_force_inline]
+   |         ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL |         1 => (),
+   |         ------- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:98:5
+   |
+LL |     #[rustc_force_inline]
+   |     ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL |     type Foo;
+   |     --------- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:101:5
+   |
+LL |     #[rustc_force_inline]
+   |     ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL |     const Bar: i32;
+   |     --------------- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:105:5
+   |
+LL |     #[rustc_force_inline]
+   |     ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL |     fn foo() {}
+   |     ----------- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:117:5
+   |
+LL |     #[rustc_force_inline]
+   |     ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL |     fn foo() {}
+   |     ----------- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:54:5
+   |
+LL |     #[rustc_force_inline]
+   |     ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL |     static X: &'static u32;
+   |     ----------------------- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:58:5
+   |
+LL |     #[rustc_force_inline]
+   |     ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL |     type Y;
+   |     ------- not a function definition
+
+error: attribute should be applied to a function
+  --> $DIR/invalid.rs:62:5
+   |
+LL |     #[rustc_force_inline]
+   |     ^^^^^^^^^^^^^^^^^^^^^
+LL |
+LL |     fn foo();
+   |     --------- not a function definition
+
+error: aborting due to 38 previous errors
+

tests/ui/force-inlining/shims.rs

@@ -0,0 +1,9 @@
+//@ build-pass
+#![allow(internal_features)]
+#![feature(rustc_attrs)]
+
+#[rustc_force_inline]
+fn f() {}
+fn g<T: FnOnce()>(t: T) { t(); }
+
+fn main() { g(f); }