diff --git a/compiler/rustc_borrowck/src/renumber.rs b/compiler/rustc_borrowck/src/renumber.rs index ff92b4168a86c..037ebc9c0a7a3 100644 --- a/compiler/rustc_borrowck/src/renumber.rs +++ b/compiler/rustc_borrowck/src/renumber.rs @@ -21,10 +21,10 @@ pub(crate) fn renumber_mir<'tcx>( let mut renumberer = RegionRenumberer { infcx }; for body in promoted.iter_mut() { - renumberer.visit_body(body); + renumberer.visit_body_preserves_cfg(body); } - renumberer.visit_body(body); + renumberer.visit_body_preserves_cfg(body); } // The fields are used only for debugging output in `sccs_info`. diff --git a/compiler/rustc_middle/src/mir/basic_blocks.rs b/compiler/rustc_middle/src/mir/basic_blocks.rs index d0dbf64dc5959..02984589422e2 100644 --- a/compiler/rustc_middle/src/mir/basic_blocks.rs +++ b/compiler/rustc_middle/src/mir/basic_blocks.rs @@ -1,4 +1,4 @@ -use std::sync::OnceLock; +use std::sync::{Arc, OnceLock}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::graph; @@ -15,7 +15,8 @@ use crate::mir::{BasicBlock, BasicBlockData, START_BLOCK, Terminator, Terminator #[derive(Clone, TyEncodable, TyDecodable, Debug, HashStable, TypeFoldable, TypeVisitable)] pub struct BasicBlocks<'tcx> { basic_blocks: IndexVec>, - cache: Cache, + /// Use an `Arc` so we can share the cache when we clone the MIR body, as borrowck does. + cache: Arc, } // Typically 95%+ of basic blocks have 4 or fewer predecessors. @@ -49,9 +50,10 @@ struct Cache { impl<'tcx> BasicBlocks<'tcx> { #[inline] pub fn new(basic_blocks: IndexVec>) -> Self { - BasicBlocks { basic_blocks, cache: Cache::default() } + BasicBlocks { basic_blocks, cache: Arc::new(Cache::default()) } } + #[inline] pub fn dominators(&self) -> &Dominators { self.cache.dominators.get_or_init(|| dominators(self)) } @@ -142,7 +144,14 @@ impl<'tcx> BasicBlocks<'tcx> { /// All other methods that allow you to mutate the basic blocks also call this method /// themselves, thereby avoiding any risk of accidentally cache invalidation. pub fn invalidate_cfg_cache(&mut self) { - self.cache = Cache::default(); + if let Some(cache) = Arc::get_mut(&mut self.cache) { + // If we only have a single reference to this cache, clear it. + *cache = Cache::default(); + } else { + // If we have several references to this cache, overwrite the pointer itself so other + // users can continue to use their (valid) cache. + self.cache = Arc::new(Cache::default()); + } } } diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index 8f88228d9bbd8..a54e548ad70ab 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -74,7 +74,11 @@ impl SimplifyCfg { } pub(super) fn simplify_cfg<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - CfgSimplifier::new(tcx, body).simplify(); + if CfgSimplifier::new(tcx, body).simplify() { + // `simplify` returns that it changed something. We must invalidate the CFG caches as they + // are not consistent with the modified CFG any more. + body.basic_blocks.invalidate_cfg_cache(); + } remove_dead_blocks(body); // FIXME: Should probably be moved into some kind of pass manager @@ -121,12 +125,16 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> { // Preserve `SwitchInt` reads on built and analysis MIR, or if `-Zmir-preserve-ub`. let preserve_switch_reads = matches!(body.phase, MirPhase::Built | MirPhase::Analysis(_)) || tcx.sess.opts.unstable_opts.mir_preserve_ub; - let basic_blocks = body.basic_blocks_mut(); + // Do not clear caches yet. The caller to `simplify` will do it if anything changed. + let basic_blocks = body.basic_blocks.as_mut_preserves_cfg(); CfgSimplifier { preserve_switch_reads, basic_blocks, pred_count } } - fn simplify(mut self) { + /// Returns whether we actually simplified anything. In that case, the caller *must* invalidate + /// the CFG caches of the MIR body. + #[must_use] + fn simplify(mut self) -> bool { self.strip_nops(); // Vec of the blocks that should be merged. We store the indices here, instead of the @@ -134,6 +142,7 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> { // We do not push the statements directly into the target block (`bb`) as that is slower // due to additional reallocations let mut merged_blocks = Vec::new(); + let mut outer_changed = false; loop { let mut changed = false; @@ -177,7 +186,11 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> { if !changed { break; } + + outer_changed = true; } + + outer_changed } /// This function will return `None` if