Skip to content

Make the necessary changes to support concurrency in Miri. #70598

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Apr 20, 2020
2 changes: 1 addition & 1 deletion src/librustc_mir/const_eval/error.rs
Original file line number Diff line number Diff line change
@@ -50,7 +50,7 @@ impl Error for ConstEvalErrKind {}
/// Turn an interpreter error into something to report to the user.
/// As a side-effect, if RUSTC_CTFE_BACKTRACE is set, this prints the backtrace.
/// Should be called only if the error is actually going to to be reported!
pub fn error_to_const_error<'mir, 'tcx, M: Machine<'mir, 'tcx>>(
pub fn error_to_const_error<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>>(
ecx: &InterpCx<'mir, 'tcx, M>,
mut error: InterpErrorInfo<'tcx>,
) -> ConstEvalErr<'tcx> {
30 changes: 24 additions & 6 deletions src/librustc_mir/const_eval/machine.rs
Original file line number Diff line number Diff line change
@@ -19,7 +19,7 @@ use crate::interpret::{

use super::error::*;

impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter> {
impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>> {
/// Evaluate a const function where all arguments (if any) are zero-sized types.
/// The evaluation is memoized thanks to the query system.
///
@@ -86,12 +86,15 @@ impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter> {
}

/// Extra machine state for CTFE, and the Machine instance
pub struct CompileTimeInterpreter {
pub struct CompileTimeInterpreter<'mir, 'tcx> {
/// For now, the number of terminators that can be evaluated before we throw a resource
/// exhuastion error.
///
/// Setting this to `0` disables the limit and allows the interpreter to run forever.
pub steps_remaining: usize,

/// The virtual call stack.
pub(crate) stack: Vec<Frame<'mir, 'tcx, (), ()>>,
}

#[derive(Copy, Clone, Debug)]
@@ -100,9 +103,9 @@ pub struct MemoryExtra {
pub(super) can_access_statics: bool,
}

impl CompileTimeInterpreter {
impl<'mir, 'tcx> CompileTimeInterpreter<'mir, 'tcx> {
pub(super) fn new(const_eval_limit: usize) -> Self {
CompileTimeInterpreter { steps_remaining: const_eval_limit }
CompileTimeInterpreter { steps_remaining: const_eval_limit, stack: Vec::new() }
}
}

@@ -156,7 +159,8 @@ impl<K: Hash + Eq, V> interpret::AllocMap<K, V> for FxHashMap<K, V> {
}
}

crate type CompileTimeEvalContext<'mir, 'tcx> = InterpCx<'mir, 'tcx, CompileTimeInterpreter>;
crate type CompileTimeEvalContext<'mir, 'tcx> =
InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>>;

impl interpret::MayLeak for ! {
#[inline(always)]
@@ -166,7 +170,7 @@ impl interpret::MayLeak for ! {
}
}

impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter {
impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, 'tcx> {
type MemoryKind = !;
type PointerTag = ();
type ExtraFnVal = !;
@@ -349,6 +353,20 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter {
Ok(frame)
}

#[inline(always)]
fn stack(
ecx: &'a InterpCx<'mir, 'tcx, Self>,
) -> &'a [Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>] {
&ecx.machine.stack
}

#[inline(always)]
fn stack_mut(
ecx: &'a mut InterpCx<'mir, 'tcx, Self>,
) -> &'a mut Vec<Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>> {
&mut ecx.machine.stack
}

fn before_access_global(
memory_extra: &MemoryExtra,
alloc_id: AllocId,
2 changes: 1 addition & 1 deletion src/librustc_mir/interpret/cast.rs
Original file line number Diff line number Diff line change
@@ -12,7 +12,7 @@ use rustc_middle::ty::{self, Ty, TypeAndMut, TypeFoldable};
use rustc_span::symbol::sym;
use rustc_target::abi::{LayoutOf, Size, Variants};

impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
pub fn cast(
&mut self,
src: OpTy<'tcx, M::PointerTag>,
57 changes: 32 additions & 25 deletions src/librustc_mir/interpret/eval_context.rs
Original file line number Diff line number Diff line change
@@ -28,6 +28,8 @@ use crate::util::storage::AlwaysLiveLocals;

pub struct InterpCx<'mir, 'tcx, M: Machine<'mir, 'tcx>> {
/// Stores the `Machine` instance.
///
/// Note: the stack is provided by the machine.
pub machine: M,

/// The results of the type checker, from rustc.
@@ -39,9 +41,6 @@ pub struct InterpCx<'mir, 'tcx, M: Machine<'mir, 'tcx>> {
/// The virtual memory system.
pub memory: Memory<'mir, 'tcx, M>,

/// The virtual call stack.
pub(crate) stack: Vec<Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra>>,

/// A cache for deduplicating vtables
pub(super) vtables:
FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), Pointer<M::PointerTag>>,
@@ -295,7 +294,7 @@ pub(super) fn from_known_layout<'tcx>(
}
}

impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
pub fn new(
tcx: TyCtxtAt<'tcx>,
param_env: ty::ParamEnv<'tcx>,
@@ -307,7 +306,6 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
tcx,
param_env,
memory: Memory::new(tcx, memory_extra),
stack: Vec::new(),
vtables: FxHashMap::default(),
}
}
@@ -347,24 +345,32 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}

#[inline(always)]
pub fn stack(&self) -> &[Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra>] {
&self.stack
pub(crate) fn stack(&self) -> &[Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra>] {
M::stack(self)
}

#[inline(always)]
pub(crate) fn stack_mut(
&mut self,
) -> &mut Vec<Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra>> {
M::stack_mut(self)
}

#[inline(always)]
pub fn cur_frame(&self) -> usize {
assert!(!self.stack.is_empty());
self.stack.len() - 1
pub fn frame_idx(&self) -> usize {
let stack = self.stack();
assert!(!stack.is_empty());
stack.len() - 1
}

#[inline(always)]
pub fn frame(&self) -> &Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra> {
self.stack.last().expect("no call frames exist")
self.stack().last().expect("no call frames exist")
}

#[inline(always)]
pub fn frame_mut(&mut self) -> &mut Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra> {
self.stack.last_mut().expect("no call frames exist")
self.stack_mut().last_mut().expect("no call frames exist")
}

#[inline(always)]
@@ -595,8 +601,8 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
return_place: Option<PlaceTy<'tcx, M::PointerTag>>,
return_to_block: StackPopCleanup,
) -> InterpResult<'tcx> {
if !self.stack.is_empty() {
info!("PAUSING({}) {}", self.cur_frame(), self.frame().instance);
if !self.stack().is_empty() {
info!("PAUSING({}) {}", self.frame_idx(), self.frame().instance);
}
::log_settings::settings().indentation += 1;

@@ -614,7 +620,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
extra: (),
};
let frame = M::init_frame_extra(self, pre_frame)?;
self.stack.push(frame);
self.stack_mut().push(frame);

// don't allocate at all for trivial constants
if body.local_decls.len() > 1 {
@@ -647,9 +653,9 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}

M::after_stack_push(self)?;
info!("ENTERING({}) {}", self.cur_frame(), self.frame().instance);
info!("ENTERING({}) {}", self.frame_idx(), self.frame().instance);

if self.stack.len() > *self.tcx.sess.recursion_limit.get() {
if self.stack().len() > *self.tcx.sess.recursion_limit.get() {
throw_exhaust!(StackFrameLimitReached)
} else {
Ok(())
@@ -704,7 +710,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
pub(super) fn pop_stack_frame(&mut self, unwinding: bool) -> InterpResult<'tcx> {
info!(
"LEAVING({}) {} (unwinding = {})",
self.cur_frame(),
self.frame_idx(),
self.frame().instance,
unwinding
);
@@ -719,7 +725,8 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
);

::log_settings::settings().indentation -= 1;
let frame = self.stack.pop().expect("tried to pop a stack frame, but there were none");
let frame =
self.stack_mut().pop().expect("tried to pop a stack frame, but there were none");

// Now where do we jump next?

@@ -734,7 +741,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
};

if !cleanup {
assert!(self.stack.is_empty(), "only the topmost frame should ever be leaked");
assert!(self.stack().is_empty(), "only the topmost frame should ever be leaked");
assert!(next_block.is_none(), "tried to skip cleanup when we have a next block!");
assert!(!unwinding, "tried to skip cleanup during unwinding");
// Leak the locals, skip validation, skip machine hook.
@@ -783,10 +790,10 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
}

if !self.stack.is_empty() {
if !self.stack().is_empty() {
info!(
"CONTINUING({}) {} (unwinding = {})",
self.cur_frame(),
self.frame_idx(),
self.frame().instance,
unwinding
);
@@ -894,12 +901,12 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
Place::Local { frame, local } => {
let mut allocs = Vec::new();
let mut msg = format!("{:?}", local);
if frame != self.cur_frame() {
write!(msg, " ({} frames up)", self.cur_frame() - frame).unwrap();
if frame != self.frame_idx() {
write!(msg, " ({} frames up)", self.frame_idx() - frame).unwrap();
}
write!(msg, ":").unwrap();

match self.stack[frame].locals[local].value {
match self.stack()[frame].locals[local].value {
LocalValue::Dead => write!(msg, " is dead").unwrap(),
LocalValue::Uninitialized => write!(msg, " is uninitialized").unwrap(),
LocalValue::Live(Operand::Indirect(mplace)) => match mplace.ptr {
7 changes: 5 additions & 2 deletions src/librustc_mir/interpret/intern.rs
Original file line number Diff line number Diff line change
@@ -148,7 +148,7 @@ impl<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx>> InternVisitor<'rt, 'mir
}
}

impl<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
for InternVisitor<'rt, 'mir, 'tcx, M>
{
type V = MPlaceTy<'tcx>;
@@ -284,7 +284,10 @@ pub fn intern_const_alloc_recursive<M: CompileTimeMachine<'mir, 'tcx>>(
intern_kind: InternKind,
ret: MPlaceTy<'tcx>,
ignore_interior_mut_in_const_validation: bool,
) -> InterpResult<'tcx> {
) -> InterpResult<'tcx>
where
'tcx: 'mir,
{
let tcx = ecx.tcx;
let (base_mutability, base_intern_mode) = match intern_kind {
// `static mut` doesn't care about interior mutability, it's mutable anyway
2 changes: 1 addition & 1 deletion src/librustc_mir/interpret/intrinsics.rs
Original file line number Diff line number Diff line change
@@ -73,7 +73,7 @@ crate fn eval_nullary_intrinsic<'tcx>(
})
}

impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// Returns `true` if emulation happened.
pub fn emulate_intrinsic(
&mut self,
4 changes: 2 additions & 2 deletions src/librustc_mir/interpret/intrinsics/caller_location.rs
Original file line number Diff line number Diff line change
@@ -10,11 +10,11 @@ use crate::interpret::{
MPlaceTy, MemoryKind, Scalar,
};

impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// Walks up the callstack from the intrinsic's callsite, searching for the first callsite in a
/// frame which is not `#[track_caller]`.
crate fn find_closest_untracked_caller_location(&self) -> Span {
self.stack
self.stack()
.iter()
.rev()
// Find first non-`#[track_caller]` frame.
38 changes: 38 additions & 0 deletions src/librustc_mir/interpret/machine.rs
Original file line number Diff line number Diff line change
@@ -230,6 +230,34 @@ pub trait Machine<'mir, 'tcx>: Sized {
id
}

/// Called when converting a `ty::Const` to an operand (in
/// `eval_const_to_op`).
///
/// Miri uses this callback for creating per thread allocations for thread
/// locals. In Rust, one way of creating a thread local is by marking a
/// static with `#[thread_local]`. On supported platforms this gets
/// translated to a LLVM thread local for which LLVM automatically ensures
/// that each thread gets its own copy. Since LLVM automatically handles
/// thread locals, the Rust compiler just treats thread local statics as
/// regular statics even though accessing a thread local static should be an
/// effectful computation that depends on the current thread. The long term
/// plan is to change MIR to make accesses to thread locals explicit
/// (https://github.com/rust-lang/rust/issues/70685). While the issue 70685
/// is not fixed, our current workaround in Miri is to use this function to
/// make per-thread copies of thread locals. Please note that we cannot make
/// these copies in `canonical_alloc_id` because that is too late: for
/// example, if one created a pointer in thread `t1` to a thread local and
/// sent it to another thread `t2`, resolving the access in
/// `canonical_alloc_id` would result in pointer pointing to `t2`'s thread
/// local and not `t1` as it should.
#[inline]
fn adjust_global_const(
_ecx: &InterpCx<'mir, 'tcx, Self>,
val: mir::interpret::ConstValue<'tcx>,
) -> InterpResult<'tcx, mir::interpret::ConstValue<'tcx>> {
Ok(val)
}

/// Called to initialize the "extra" state of an allocation and make the pointers
/// it contains (in relocations) tagged. The way we construct allocations is
/// to always first construct it without extra and then add the extra.
@@ -285,6 +313,16 @@ pub trait Machine<'mir, 'tcx>: Sized {
frame: Frame<'mir, 'tcx, Self::PointerTag>,
) -> InterpResult<'tcx, Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>>;

/// Borrow the current thread's stack.
fn stack(
ecx: &'a InterpCx<'mir, 'tcx, Self>,
) -> &'a [Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>];

/// Mutably borrow the current thread's stack.
fn stack_mut(
ecx: &'a mut InterpCx<'mir, 'tcx, Self>,
) -> &'a mut Vec<Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>>;

/// Called immediately after a stack frame got pushed and its locals got initialized.
fn after_stack_push(_ecx: &mut InterpCx<'mir, 'tcx, Self>) -> InterpResult<'tcx> {
Ok(())
14 changes: 12 additions & 2 deletions src/librustc_mir/interpret/operand.rs
Original file line number Diff line number Diff line change
@@ -208,7 +208,7 @@ impl<'tcx, Tag: Copy> ImmTy<'tcx, Tag> {
}
}

impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// Normalice `place.ptr` to a `Pointer` if this is a place and not a ZST.
/// Can be helpful to avoid lots of `force_ptr` calls later, if this place is used a lot.
#[inline]
@@ -439,7 +439,9 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> {
let op = match *place {
Place::Ptr(mplace) => Operand::Indirect(mplace),
Place::Local { frame, local } => *self.access_local(&self.stack[frame], local, None)?,
Place::Local { frame, local } => {
*self.access_local(&self.stack()[frame], local, None)?
}
};
Ok(OpTy { op, layout: place.layout })
}
@@ -526,6 +528,10 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// potentially requiring the current static to be evaluated again. This is not a
// problem here, because we are building an operand which means an actual read is
// happening.
//
// The machine callback `adjust_global_const` below is guaranteed to
// be called for all constants because `const_eval` calls
// `eval_const_to_op` recursively.
return Ok(self.const_eval(GlobalId { instance, promoted }, val.ty)?);
}
ty::ConstKind::Infer(..)
@@ -535,6 +541,10 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
ty::ConstKind::Value(val_val) => val_val,
};
// This call allows the machine to create fresh allocation ids for
// thread-local statics (see the `adjust_global_const` function
// documentation).
let val_val = M::adjust_global_const(self, val_val)?;
// Other cases need layout.
let layout = from_known_layout(self.tcx, layout, || self.layout_of(val.ty))?;
let op = match val_val {
4 changes: 2 additions & 2 deletions src/librustc_mir/interpret/operator.rs
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ use rustc_target::abi::LayoutOf;

use super::{ImmTy, Immediate, InterpCx, Machine, PlaceTy};

impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// Applies the binary operation `op` to the two operands and writes a tuple of the result
/// and a boolean signifying the potential overflow to the destination.
pub fn binop_with_overflow(
@@ -45,7 +45,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
}

impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
fn binary_char_op(
&self,
bin_op: mir::BinOp,
13 changes: 7 additions & 6 deletions src/librustc_mir/interpret/place.rs
Original file line number Diff line number Diff line change
@@ -283,7 +283,7 @@ impl<'tcx, Tag: ::std::fmt::Debug> PlaceTy<'tcx, Tag> {
}

// separating the pointer tag for `impl Trait`, see https://github.com/rust-lang/rust/issues/54385
impl<'mir, 'tcx, Tag, M> InterpCx<'mir, 'tcx, M>
impl<'mir, 'tcx: 'mir, Tag, M> InterpCx<'mir, 'tcx, M>
where
// FIXME: Working around https://github.com/rust-lang/rust/issues/54385
Tag: ::std::fmt::Debug + Copy + Eq + Hash + 'static,
@@ -662,7 +662,7 @@ where
}
local => PlaceTy {
// This works even for dead/uninitialized locals; we check further when writing
place: Place::Local { frame: self.cur_frame(), local },
place: Place::Local { frame: self.frame_idx(), local },
layout: self.layout_of_local(self.frame(), local, None)?,
},
};
@@ -755,7 +755,7 @@ where
// but not factored as a separate function.
let mplace = match dest.place {
Place::Local { frame, local } => {
match self.stack[frame].locals[local].access_mut()? {
match self.stack_mut()[frame].locals[local].access_mut()? {
Ok(local) => {
// Local can be updated in-place.
*local = LocalValue::Live(Operand::Immediate(src));
@@ -985,14 +985,15 @@ where
) -> InterpResult<'tcx, (MPlaceTy<'tcx, M::PointerTag>, Option<Size>)> {
let (mplace, size) = match place.place {
Place::Local { frame, local } => {
match self.stack[frame].locals[local].access_mut()? {
match self.stack_mut()[frame].locals[local].access_mut()? {
Ok(&mut local_val) => {
// We need to make an allocation.

// We need the layout of the local. We can NOT use the layout we got,
// that might e.g., be an inner field of a struct with `Scalar` layout,
// that has different alignment than the outer field.
let local_layout = self.layout_of_local(&self.stack[frame], local, None)?;
let local_layout =
self.layout_of_local(&self.stack()[frame], local, None)?;
// We also need to support unsized types, and hence cannot use `allocate`.
let (size, align) = self
.size_and_align_of(meta, local_layout)?
@@ -1008,7 +1009,7 @@ where
}
// Now we can call `access_mut` again, asserting it goes well,
// and actually overwrite things.
*self.stack[frame].locals[local].access_mut().unwrap().unwrap() =
*self.stack_mut()[frame].locals[local].access_mut().unwrap().unwrap() =
LocalValue::Live(Operand::Indirect(mplace));
(mplace, Some(size))
}
16 changes: 8 additions & 8 deletions src/librustc_mir/interpret/step.rs
Original file line number Diff line number Diff line change
@@ -29,7 +29,7 @@ fn binop_right_homogeneous(op: mir::BinOp) -> bool {
}
}

impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
pub fn run(&mut self) -> InterpResult<'tcx> {
while self.step()? {}
Ok(())
@@ -42,7 +42,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// This is marked `#inline(always)` to work around adverserial codegen when `opt-level = 3`
#[inline(always)]
pub fn step(&mut self) -> InterpResult<'tcx, bool> {
if self.stack.is_empty() {
if self.stack().is_empty() {
return Ok(false);
}

@@ -60,18 +60,18 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
let body = self.body();
let basic_block = &body.basic_blocks()[block];

let old_frames = self.cur_frame();
let old_frames = self.frame_idx();

if let Some(stmt) = basic_block.statements.get(stmt_id) {
assert_eq!(old_frames, self.cur_frame());
assert_eq!(old_frames, self.frame_idx());
self.statement(stmt)?;
return Ok(true);
}

M::before_terminator(self)?;

let terminator = basic_block.terminator();
assert_eq!(old_frames, self.cur_frame());
assert_eq!(old_frames, self.frame_idx());
self.terminator(terminator)?;
Ok(true)
}
@@ -84,7 +84,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {

// Some statements (e.g., box) push new stack frames.
// We have to record the stack frame number *before* executing the statement.
let frame_idx = self.cur_frame();
let frame_idx = self.frame_idx();

match &stmt.kind {
Assign(box (place, rvalue)) => self.eval_rvalue_into_place(rvalue, *place)?,
@@ -126,7 +126,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
LlvmInlineAsm { .. } => throw_unsup_format!("inline assembly is not supported"),
}

self.stack[frame_idx].stmt += 1;
self.stack_mut()[frame_idx].stmt += 1;
Ok(())
}

@@ -278,7 +278,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
self.set_span(terminator.source_info.span);

self.eval_terminator(terminator)?;
if !self.stack.is_empty() {
if !self.stack().is_empty() {
if let Some(block) = self.frame().block {
info!("// executing {:?}", block);
}
8 changes: 4 additions & 4 deletions src/librustc_mir/interpret/terminator.rs
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@ use super::{
FnVal, ImmTy, InterpCx, InterpResult, MPlaceTy, Machine, OpTy, PlaceTy, StackPopCleanup,
};

impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
pub(super) fn eval_terminator(
&mut self,
terminator: &mir::Terminator<'tcx>,
@@ -52,7 +52,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}

Call { ref func, ref args, destination, ref cleanup, .. } => {
let old_stack = self.cur_frame();
let old_stack = self.frame_idx();
let old_bb = self.frame().block;
let func = self.eval_operand(func, None)?;
let (fn_val, abi) = match func.layout.ty.kind {
@@ -80,7 +80,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
self.eval_fn_call(fn_val, abi, &args[..], ret, *cleanup)?;
// Sanity-check that `eval_fn_call` either pushed a new frame or
// did a jump to another block.
if self.cur_frame() == old_stack && self.frame().block == old_bb {
if self.frame_idx() == old_stack && self.frame().block == old_bb {
span_bug!(terminator.source_info.span, "evaluating this call made no progress");
}
}
@@ -372,7 +372,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
};
match res {
Err(err) => {
self.stack.pop();
self.stack_mut().pop();
Err(err)
}
Ok(()) => Ok(()),
2 changes: 1 addition & 1 deletion src/librustc_mir/interpret/traits.rs
Original file line number Diff line number Diff line change
@@ -6,7 +6,7 @@ use rustc_target::abi::{Align, HasDataLayout, LayoutOf, Size};

use super::{FnVal, InterpCx, Machine, MemoryKind};

impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// Creates a dynamic vtable for the given type and vtable origin. This is used only for
/// objects.
///
6 changes: 3 additions & 3 deletions src/librustc_mir/interpret/validity.rs
Original file line number Diff line number Diff line change
@@ -177,7 +177,7 @@ struct ValidityVisitor<'rt, 'mir, 'tcx, M: Machine<'mir, 'tcx>> {
ecx: &'rt InterpCx<'mir, 'tcx, M>,
}

impl<'rt, 'mir, 'tcx, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, 'tcx, M> {
impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, 'tcx, M> {
fn aggregate_field_path_elem(&mut self, layout: TyAndLayout<'tcx>, field: usize) -> PathElem {
// First, check if we are projecting to a variant.
match layout.variants {
@@ -604,7 +604,7 @@ impl<'rt, 'mir, 'tcx, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, 'tcx, M
}
}

impl<'rt, 'mir, 'tcx, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
for ValidityVisitor<'rt, 'mir, 'tcx, M>
{
type V = OpTy<'tcx, M::PointerTag>;
@@ -806,7 +806,7 @@ impl<'rt, 'mir, 'tcx, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
}
}

impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
fn validate_operand_internal(
&self,
op: OpTy<'tcx, M::PointerTag>,
6 changes: 4 additions & 2 deletions src/librustc_mir/interpret/visitor.rs
Original file line number Diff line number Diff line change
@@ -35,7 +35,7 @@ pub trait Value<'mir, 'tcx, M: Machine<'mir, 'tcx>>: Copy {

// Operands and memory-places are both values.
// Places in general are not due to `place_field` having to do `force_allocation`.
impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> for OpTy<'tcx, M::PointerTag> {
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> for OpTy<'tcx, M::PointerTag> {
#[inline(always)]
fn layout(&self) -> TyAndLayout<'tcx> {
self.layout
@@ -73,7 +73,9 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> for OpTy<'tcx, M::
}
}

impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> for MPlaceTy<'tcx, M::PointerTag> {
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M>
for MPlaceTy<'tcx, M::PointerTag>
{
#[inline(always)]
fn layout(&self) -> TyAndLayout<'tcx> {
self.layout
31 changes: 27 additions & 4 deletions src/librustc_mir/transform/const_prop.rs
Original file line number Diff line number Diff line change
@@ -158,9 +158,18 @@ impl<'tcx> MirPass<'tcx> for ConstProp {
}
}

struct ConstPropMachine;
struct ConstPropMachine<'mir, 'tcx> {
/// The virtual call stack.
stack: Vec<Frame<'mir, 'tcx, (), ()>>,
}

impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine {
impl<'mir, 'tcx> ConstPropMachine<'mir, 'tcx> {
fn new() -> Self {
Self { stack: Vec::new() }
}
}

impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> {
type MemoryKind = !;
type PointerTag = ();
type ExtraFnVal = !;
@@ -296,11 +305,25 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine {
) -> InterpResult<'tcx, Frame<'mir, 'tcx>> {
Ok(frame)
}

#[inline(always)]
fn stack(
ecx: &'a InterpCx<'mir, 'tcx, Self>,
) -> &'a [Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>] {
&ecx.machine.stack
}

#[inline(always)]
fn stack_mut(
ecx: &'a mut InterpCx<'mir, 'tcx, Self>,
) -> &'a mut Vec<Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>> {
&mut ecx.machine.stack
}
}

/// Finds optimization opportunities on the MIR.
struct ConstPropagator<'mir, 'tcx> {
ecx: InterpCx<'mir, 'tcx, ConstPropMachine>,
ecx: InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>,
tcx: TyCtxt<'tcx>,
can_const_prop: IndexVec<Local, ConstPropMode>,
param_env: ParamEnv<'tcx>,
@@ -349,7 +372,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
let param_env = tcx.param_env(def_id).with_reveal_all();

let span = tcx.def_span(def_id);
let mut ecx = InterpCx::new(tcx.at(span), param_env, ConstPropMachine, ());
let mut ecx = InterpCx::new(tcx.at(span), param_env, ConstPropMachine::new(), ());
let can_const_prop = CanConstProp::check(body);

let ret = ecx