Rollup of 12 pull requests

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -1146,19 +1146,51 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             (args, None)
         };
 
+        // Special logic for tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments.
+        //
+        // Normally an indirect argument with `on_stack: false` would be passed as a pointer into
+        // the caller's stack frame. For tail calls, that would be unsound, because the caller's
+        // stack frame is overwritten by the callee's stack frame.
+        //
+        // Therefore we store the argument for the callee in the corresponding caller's slot.
+        // Because guaranteed tail calls demand that the caller's signature matches the callee's,
+        // the corresponding slot has the correct type.
+        //
+        // To handle cases like the one below, the tail call arguments must first be copied to a
+        // temporary, and only then copied to the caller's argument slots.
+        //
+        // ```
+        // // A struct big enough that it is not passed via registers.
+        // pub struct Big([u64; 4]);
+        //
+        // fn swapper(a: Big, b: Big) -> (Big, Big) {
+        //     become swapper_helper(b, a);
+        // }
+        // ```
+        let mut tail_call_temporaries = vec![];
+        if kind == CallKind::Tail {
+            tail_call_temporaries = vec![None; first_args.len()];
+            // Copy the arguments that use `PassMode::Indirect { on_stack: false , ..}`
+            // to temporary stack allocations. See the comment above.
+            for (i, arg) in first_args.iter().enumerate() {
+                if !matches!(fn_abi.args[i].mode, PassMode::Indirect { on_stack: false, .. }) {
+                    continue;
+                }
+
+                let op = self.codegen_operand(bx, &arg.node);
+                let tmp = PlaceRef::alloca(bx, op.layout);
+                bx.lifetime_start(tmp.val.llval, tmp.layout.size);
+                op.store_with_annotation(bx, tmp);
+
+                tail_call_temporaries[i] = Some(tmp);
+            }
+        }
+
         // When generating arguments we sometimes introduce temporary allocations with lifetime
         // that extend for the duration of a call. Keep track of those allocations and their sizes
         // to generate `lifetime_end` when the call returns.
         let mut lifetime_ends_after_call: Vec<(Bx::Value, Size)> = Vec::new();
         'make_args: for (i, arg) in first_args.iter().enumerate() {
-            if kind == CallKind::Tail && matches!(fn_abi.args[i].mode, PassMode::Indirect { .. }) {
-                // FIXME: https://github.com/rust-lang/rust/pull/144232#discussion_r2218543841
-                span_bug!(
-                    fn_span,
-                    "arguments using PassMode::Indirect are currently not supported for tail calls"
-                );
-            }
-
             let mut op = self.codegen_operand(bx, &arg.node);
 
             if let (0, Some(ty::InstanceKind::Virtual(_, idx))) = (i, instance.map(|i| i.def)) {
@@ -1209,18 +1241,72 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
             }
 
-            // The callee needs to own the argument memory if we pass it
-            // by-ref, so make a local copy of non-immediate constants.
-            match (&arg.node, op.val) {
-                (&mir::Operand::Copy(_), Ref(PlaceValue { llextra: None, .. }))
-                | (&mir::Operand::Constant(_), Ref(PlaceValue { llextra: None, .. })) => {
-                    let tmp = PlaceRef::alloca(bx, op.layout);
-                    bx.lifetime_start(tmp.val.llval, tmp.layout.size);
-                    op.store_with_annotation(bx, tmp);
-                    op.val = Ref(tmp.val);
-                    lifetime_ends_after_call.push((tmp.val.llval, tmp.layout.size));
+            match kind {
+                CallKind::Normal => {
+                    // The callee needs to own the argument memory if we pass it
+                    // by-ref, so make a local copy of non-immediate constants.
+                    if let &mir::Operand::Copy(_) | &mir::Operand::Constant(_) = &arg.node
+                        && let Ref(PlaceValue { llextra: None, .. }) = op.val
+                    {
+                        let tmp = PlaceRef::alloca(bx, op.layout);
+                        bx.lifetime_start(tmp.val.llval, tmp.layout.size);
+                        op.store_with_annotation(bx, tmp);
+                        op.val = Ref(tmp.val);
+                        lifetime_ends_after_call.push((tmp.val.llval, tmp.layout.size));
+                    }
+                }
+                CallKind::Tail => {
+                    match fn_abi.args[i].mode {
+                        PassMode::Indirect { on_stack: false, .. } => {
+                            let Some(tmp) = tail_call_temporaries[i].take() else {
+                                span_bug!(
+                                    fn_span,
+                                    "missing temporary for indirect tail call argument #{i}"
+                                )
+                            };
+
+                            let local = self.mir.args_iter().nth(i).unwrap();
+
+                            match &self.locals[local] {
+                                LocalRef::Place(arg) => {
+                                    bx.typed_place_copy(arg.val, tmp.val, fn_abi.args[i].layout);
+                                    op.val = Ref(arg.val);
+                                }
+                                LocalRef::Operand(arg) => {
+                                    let Ref(place_value) = arg.val else {
+                                        bug!("only `Ref` should use `PassMode::Indirect`");
+                                    };
+                                    bx.typed_place_copy(
+                                        place_value,
+                                        tmp.val,
+                                        fn_abi.args[i].layout,
+                                    );
+                                    op.val = arg.val;
+                                }
+                                LocalRef::UnsizedPlace(_) => {
+                                    span_bug!(fn_span, "unsized types are not supported")
+                                }
+                                LocalRef::PendingOperand => {
+                                    span_bug!(fn_span, "argument local should not be pending")
+                                }
+                            };
+
+                            bx.lifetime_end(tmp.val.llval, tmp.layout.size);
+                        }
+                        PassMode::Indirect { on_stack: true, .. } => {
+                            // FIXME: some LLVM backends (notably x86) do not correctly pass byval
+                            // arguments to tail calls (as of LLVM 21). See also:
+                            //
+                            // - https://github.com/rust-lang/rust/pull/144232#discussion_r2218543841
+                            // - https://github.com/rust-lang/rust/issues/144855
+                            span_bug!(
+                                fn_span,
+                                "arguments using PassMode::Indirect {{ on_stack: true, .. }} are currently not supported for tail calls"
+                            )
+                        }
+                        _ => (),
+                    }
                 }
-                _ => {}
             }
 
             self.codegen_argument(

compiler/rustc_infer/src/infer/lexical_region_resolve/indexed_edges.rs

@@ -0,0 +1,68 @@
+use rustc_index::IndexVec;
+use rustc_type_ir::RegionVid;
+
+use crate::infer::SubregionOrigin;
+use crate::infer::region_constraints::{Constraint, ConstraintKind, RegionConstraintData};
+
+/// Selects either out-edges or in-edges for [`IndexedConstraintEdges::adjacent_edges`].
+#[derive(Clone, Copy, Debug)]
+pub(super) enum EdgeDirection {
+    Out,
+    In,
+}
+
+/// Type alias for the pairs stored in [`RegionConstraintData::constraints`],
+/// which we are indexing.
+type ConstraintPair<'tcx> = (Constraint<'tcx>, SubregionOrigin<'tcx>);
+
+/// An index from region variables to their corresponding constraint edges,
+/// used on some error paths.
+pub(super) struct IndexedConstraintEdges<'data, 'tcx> {
+    out_edges: IndexVec<RegionVid, Vec<&'data ConstraintPair<'tcx>>>,
+    in_edges: IndexVec<RegionVid, Vec<&'data ConstraintPair<'tcx>>>,
+}
+
+impl<'data, 'tcx> IndexedConstraintEdges<'data, 'tcx> {
+    pub(super) fn build_index(num_vars: usize, data: &'data RegionConstraintData<'tcx>) -> Self {
+        let mut out_edges = IndexVec::from_fn_n(|_| vec![], num_vars);
+        let mut in_edges = IndexVec::from_fn_n(|_| vec![], num_vars);
+
+        for pair @ (c, _) in &data.constraints {
+            // Only push a var out-edge for `VarSub...` constraints.
+            match c.kind {
+                ConstraintKind::VarSubVar | ConstraintKind::VarSubReg => {
+                    out_edges[c.sub.as_var()].push(pair)
+                }
+                ConstraintKind::RegSubVar | ConstraintKind::RegSubReg => {}
+            }
+        }
+
+        // Index in-edges in reverse order, to match what current tests expect.
+        // (It's unclear whether this is important or not.)
+        for pair @ (c, _) in data.constraints.iter().rev() {
+            // Only push a var in-edge for `...SubVar` constraints.
+            match c.kind {
+                ConstraintKind::VarSubVar | ConstraintKind::RegSubVar => {
+                    in_edges[c.sup.as_var()].push(pair)
+                }
+                ConstraintKind::VarSubReg | ConstraintKind::RegSubReg => {}
+            }
+        }
+
+        IndexedConstraintEdges { out_edges, in_edges }
+    }
+
+    /// Returns either the out-edges or in-edges of the specified region var,
+    /// as selected by `dir`.
+    pub(super) fn adjacent_edges(
+        &self,
+        region_vid: RegionVid,
+        dir: EdgeDirection,
+    ) -> &[&'data ConstraintPair<'tcx>] {
+        let edges = match dir {
+            EdgeDirection::Out => &self.out_edges,
+            EdgeDirection::In => &self.in_edges,
+        };
+        &edges[region_vid]
+    }
+}