Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 0ff7e65

Browse files
dingxiangfei2009Dirbaio
andcommittedMar 9, 2025··
relocate upvars to Unresumed state and make coroutine prefix trivial
Co-authored-by: Dario Nieuwenhuis <[email protected]>
1 parent 3ea711f commit 0ff7e65

File tree

65 files changed

+1313
-579
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

65 files changed

+1313
-579
lines changed
 

‎compiler/rustc_abi/src/layout.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -215,15 +215,15 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
215215
>(
216216
&self,
217217
local_layouts: &IndexSlice<LocalIdx, F>,
218-
prefix_layouts: IndexVec<FieldIdx, F>,
218+
relocated_upvars: &IndexSlice<LocalIdx, Option<LocalIdx>>,
219219
variant_fields: &IndexSlice<VariantIdx, IndexVec<FieldIdx, LocalIdx>>,
220220
storage_conflicts: &BitMatrix<LocalIdx, LocalIdx>,
221221
tag_to_layout: impl Fn(Scalar) -> F,
222222
) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {
223223
coroutine::layout(
224224
self,
225225
local_layouts,
226-
prefix_layouts,
226+
relocated_upvars,
227227
variant_fields,
228228
storage_conflicts,
229229
tag_to_layout,

‎compiler/rustc_abi/src/layout/coroutine.rs

Lines changed: 50 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ pub(super) fn layout<
145145
>(
146146
calc: &super::LayoutCalculator<impl HasDataLayout>,
147147
local_layouts: &IndexSlice<LocalIdx, F>,
148-
mut prefix_layouts: IndexVec<FieldIdx, F>,
148+
relocated_upvars: &IndexSlice<LocalIdx, Option<LocalIdx>>,
149149
variant_fields: &IndexSlice<VariantIdx, IndexVec<FieldIdx, LocalIdx>>,
150150
storage_conflicts: &BitMatrix<LocalIdx, LocalIdx>,
151151
tag_to_layout: impl Fn(Scalar) -> F,
@@ -155,10 +155,8 @@ pub(super) fn layout<
155155
let (ineligible_locals, assignments) =
156156
coroutine_saved_local_eligibility(local_layouts.len(), variant_fields, storage_conflicts);
157157

158-
// Build a prefix layout, including "promoting" all ineligible
159-
// locals as part of the prefix. We compute the layout of all of
160-
// these fields at once to get optimal packing.
161-
let tag_index = prefix_layouts.len();
158+
// Build a prefix layout, consisting of only the state tag.
159+
let tag_index = 0;
162160

163161
// `variant_fields` already accounts for the reserved variants, so no need to add them.
164162
let max_discr = (variant_fields.len() - 1) as u128;
@@ -169,17 +167,17 @@ pub(super) fn layout<
169167
};
170168

171169
let promoted_layouts = ineligible_locals.iter().map(|local| local_layouts[local]);
172-
prefix_layouts.push(tag_to_layout(tag));
173-
prefix_layouts.extend(promoted_layouts);
170+
let prefix_layouts: IndexVec<_, _> =
171+
[tag_to_layout(tag)].into_iter().chain(promoted_layouts).collect();
174172
let prefix =
175173
calc.univariant(&prefix_layouts, &ReprOptions::default(), StructKind::AlwaysSized)?;
176174

177175
let (prefix_size, prefix_align) = (prefix.size, prefix.align);
178176

179-
// Split the prefix layout into the "outer" fields (upvars and
180-
// discriminant) and the "promoted" fields. Promoted fields will
181-
// get included in each variant that requested them in
182-
// CoroutineLayout.
177+
// Split the prefix layout into the discriminant and
178+
// the "promoted" fields.
179+
// Promoted fields will get included in each variant
180+
// that requested them in CoroutineLayout.
183181
debug!("prefix = {:#?}", prefix);
184182
let (outer_fields, promoted_offsets, promoted_memory_index) = match prefix.fields {
185183
FieldsShape::Arbitrary { mut offsets, memory_index } => {
@@ -215,22 +213,45 @@ pub(super) fn layout<
215213

216214
let mut size = prefix.size;
217215
let mut align = prefix.align;
218-
let variants = variant_fields
216+
let variants: IndexVec<VariantIdx, _> = variant_fields
219217
.iter_enumerated()
220218
.map(|(index, variant_fields)| {
219+
let is_unresumed = index == VariantIdx::new(0);
220+
let mut is_ineligible = IndexVec::from_elem_n(None, variant_fields.len());
221+
for (field, &local) in variant_fields.iter_enumerated() {
222+
if is_unresumed
223+
&& let Some(inner_local) = relocated_upvars[local]
224+
&& let Ineligible(Some(promoted_field)) = assignments[inner_local]
225+
{
226+
is_ineligible.insert(field, promoted_field);
227+
continue;
228+
}
229+
match assignments[local] {
230+
Assigned(v) if v == index => {}
231+
Ineligible(Some(promoted_field)) => {
232+
is_ineligible.insert(field, promoted_field);
233+
}
234+
Ineligible(None) => {
235+
panic!("an ineligible local should have been promoted into the prefix")
236+
}
237+
Assigned(_) => {
238+
panic!("an eligible local should have been assigned to exactly one variant")
239+
}
240+
Unassigned => {
241+
panic!("each saved local should have been inspected at least once")
242+
}
243+
}
244+
}
221245
// Only include overlap-eligible fields when we compute our variant layout.
222-
let variant_only_tys = variant_fields
223-
.iter()
224-
.filter(|local| match assignments[**local] {
225-
Unassigned => unreachable!(),
226-
Assigned(v) if v == index => true,
227-
Assigned(_) => unreachable!("assignment does not match variant"),
228-
Ineligible(_) => false,
246+
let fields: IndexVec<_, _> = variant_fields
247+
.iter_enumerated()
248+
.filter_map(|(field, &local)| {
249+
if is_ineligible.contains(field) { None } else { Some(local_layouts[local]) }
229250
})
230-
.map(|local| local_layouts[*local]);
251+
.collect();
231252

232253
let mut variant = calc.univariant(
233-
&variant_only_tys.collect::<IndexVec<_, _>>(),
254+
&fields,
234255
&ReprOptions::default(),
235256
StructKind::Prefixed(prefix_size, prefix_align.abi),
236257
)?;
@@ -254,19 +275,14 @@ pub(super) fn layout<
254275
IndexVec::from_elem_n(FieldIdx::new(invalid_field_idx), invalid_field_idx);
255276

256277
let mut offsets_and_memory_index = iter::zip(offsets, memory_index);
257-
let combined_offsets = variant_fields
278+
let combined_offsets = is_ineligible
258279
.iter_enumerated()
259-
.map(|(i, local)| {
260-
let (offset, memory_index) = match assignments[*local] {
261-
Unassigned => unreachable!(),
262-
Assigned(_) => {
263-
let (offset, memory_index) = offsets_and_memory_index.next().unwrap();
264-
(offset, promoted_memory_index.len() as u32 + memory_index)
265-
}
266-
Ineligible(field_idx) => {
267-
let field_idx = field_idx.unwrap();
268-
(promoted_offsets[field_idx], promoted_memory_index[field_idx])
269-
}
280+
.map(|(i, &is_ineligible)| {
281+
let (offset, memory_index) = if let Some(field_idx) = is_ineligible {
282+
(promoted_offsets[field_idx], promoted_memory_index[field_idx])
283+
} else {
284+
let (offset, memory_index) = offsets_and_memory_index.next().unwrap();
285+
(offset, promoted_memory_index.len() as u32 + memory_index)
270286
};
271287
combined_inverse_memory_index[memory_index] = i;
272288
offset
@@ -287,7 +303,7 @@ pub(super) fn layout<
287303
align = align.max(variant.align);
288304
Ok(variant)
289305
})
290-
.collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
306+
.try_collect()?;
291307

292308
size = size.align_to(align.abi);
293309

0 commit comments

Comments
 (0)
Please sign in to comment.