diff --git a/Cargo.lock b/Cargo.lock
index 274c1eefbfb52..a684f1c8e47ac 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3631,6 +3631,7 @@ dependencies = [
  "rustc_lint",
  "rustc_metadata",
  "rustc_mir",
+ "rustc_mir_build",
  "rustc_parse",
  "rustc_passes",
  "rustc_plugin_impl",
@@ -3729,7 +3730,6 @@ dependencies = [
 name = "rustc_mir"
 version = "0.0.0"
 dependencies = [
- "arena",
  "either",
  "graphviz",
  "itertools 0.8.0",
@@ -3752,6 +3752,28 @@ dependencies = [
  "syntax",
 ]
 
+[[package]]
+name = "rustc_mir_build"
+version = "0.0.0"
+dependencies = [
+ "arena",
+ "itertools 0.8.0",
+ "log",
+ "rustc",
+ "rustc_apfloat",
+ "rustc_data_structures",
+ "rustc_error_codes",
+ "rustc_errors",
+ "rustc_hir",
+ "rustc_index",
+ "rustc_macros",
+ "rustc_span",
+ "rustc_target",
+ "serialize",
+ "smallvec 1.0.0",
+ "syntax",
+]
+
 [[package]]
 name = "rustc_msan"
 version = "0.0.0"
diff --git a/src/librustc/mir/mod.rs b/src/librustc/mir/mod.rs
index ff64302b1e506..d82681f583c1b 100644
--- a/src/librustc/mir/mod.rs
+++ b/src/librustc/mir/mod.rs
@@ -23,14 +23,12 @@ use polonius_engine::Atom;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::graph::dominators::Dominators;
 use rustc_data_structures::graph::{self, GraphSuccessors};
-use rustc_data_structures::sync::Lrc;
 use rustc_index::bit_set::BitMatrix;
 use rustc_index::vec::{Idx, IndexVec};
 use rustc_macros::HashStable;
 use rustc_serialize::{Decodable, Encodable};
 use rustc_span::symbol::Symbol;
 use rustc_span::{Span, DUMMY_SP};
-use smallvec::SmallVec;
 use std::borrow::Cow;
 use std::fmt::{self, Debug, Display, Formatter, Write};
 use std::ops::Index;
@@ -39,13 +37,15 @@ use std::{iter, mem, option, u32};
 pub use syntax::ast::Mutability;
 use syntax::ast::Name;
 
-pub use crate::mir::cache::{BodyAndCache, ReadOnlyBodyAndCache};
-pub use crate::mir::interpret::AssertMessage;
+pub use self::cache::{BodyAndCache, ReadOnlyBodyAndCache};
+pub use self::interpret::AssertMessage;
+pub use self::query::*;
 pub use crate::read_only;
 
 mod cache;
 pub mod interpret;
 pub mod mono;
+mod query;
 pub mod tcx;
 pub mod traversal;
 pub mod visit;
@@ -2648,221 +2648,6 @@ impl Location {
     }
 }
 
-#[derive(Copy, Clone, PartialEq, RustcEncodable, RustcDecodable, HashStable)]
-pub enum UnsafetyViolationKind {
-    General,
-    /// Permitted both in `const fn`s and regular `fn`s.
-    GeneralAndConstFn,
-    BorrowPacked(hir::HirId),
-}
-
-#[derive(Copy, Clone, PartialEq, RustcEncodable, RustcDecodable, HashStable)]
-pub struct UnsafetyViolation {
-    pub source_info: SourceInfo,
-    pub description: Symbol,
-    pub details: Symbol,
-    pub kind: UnsafetyViolationKind,
-}
-
-#[derive(Clone, RustcEncodable, RustcDecodable, HashStable)]
-pub struct UnsafetyCheckResult {
-    /// Violations that are propagated *upwards* from this function.
-    pub violations: Lrc<[UnsafetyViolation]>,
-    /// `unsafe` blocks in this function, along with whether they are used. This is
-    /// used for the "unused_unsafe" lint.
-    pub unsafe_blocks: Lrc<[(hir::HirId, bool)]>,
-}
-
-rustc_index::newtype_index! {
-    pub struct GeneratorSavedLocal {
-        derive [HashStable]
-        DEBUG_FORMAT = "_{}",
-    }
-}
-
-/// The layout of generator state.
-#[derive(Clone, Debug, RustcEncodable, RustcDecodable, HashStable, TypeFoldable)]
-pub struct GeneratorLayout<'tcx> {
-    /// The type of every local stored inside the generator.
-    pub field_tys: IndexVec<GeneratorSavedLocal, Ty<'tcx>>,
-
-    /// Which of the above fields are in each variant. Note that one field may
-    /// be stored in multiple variants.
-    pub variant_fields: IndexVec<VariantIdx, IndexVec<Field, GeneratorSavedLocal>>,
-
-    /// Which saved locals are storage-live at the same time. Locals that do not
-    /// have conflicts with each other are allowed to overlap in the computed
-    /// layout.
-    pub storage_conflicts: BitMatrix<GeneratorSavedLocal, GeneratorSavedLocal>,
-}
-
-#[derive(Clone, Debug, RustcEncodable, RustcDecodable, HashStable)]
-pub struct BorrowCheckResult<'tcx> {
-    pub closure_requirements: Option<ClosureRegionRequirements<'tcx>>,
-    pub used_mut_upvars: SmallVec<[Field; 8]>,
-}
-
-/// The result of the `mir_const_qualif` query.
-///
-/// Each field corresponds to an implementer of the `Qualif` trait in
-/// `librustc_mir/transform/check_consts/qualifs.rs`. See that file for more information on each
-/// `Qualif`.
-#[derive(Clone, Copy, Debug, Default, RustcEncodable, RustcDecodable, HashStable)]
-pub struct ConstQualifs {
-    pub has_mut_interior: bool,
-    pub needs_drop: bool,
-}
-
-/// After we borrow check a closure, we are left with various
-/// requirements that we have inferred between the free regions that
-/// appear in the closure's signature or on its field types. These
-/// requirements are then verified and proved by the closure's
-/// creating function. This struct encodes those requirements.
-///
-/// The requirements are listed as being between various
-/// `RegionVid`. The 0th region refers to `'static`; subsequent region
-/// vids refer to the free regions that appear in the closure (or
-/// generator's) type, in order of appearance. (This numbering is
-/// actually defined by the `UniversalRegions` struct in the NLL
-/// region checker. See for example
-/// `UniversalRegions::closure_mapping`.) Note that we treat the free
-/// regions in the closure's type "as if" they were erased, so their
-/// precise identity is not important, only their position.
-///
-/// Example: If type check produces a closure with the closure substs:
-///
-/// ```text
-/// ClosureSubsts = [
-///     i8,                                  // the "closure kind"
-///     for<'x> fn(&'a &'x u32) -> &'x u32,  // the "closure signature"
-///     &'a String,                          // some upvar
-/// ]
-/// ```
-///
-/// here, there is one unique free region (`'a`) but it appears
-/// twice. We would "renumber" each occurrence to a unique vid, as follows:
-///
-/// ```text
-/// ClosureSubsts = [
-///     i8,                                  // the "closure kind"
-///     for<'x> fn(&'1 &'x u32) -> &'x u32,  // the "closure signature"
-///     &'2 String,                          // some upvar
-/// ]
-/// ```
-///
-/// Now the code might impose a requirement like `'1: '2`. When an
-/// instance of the closure is created, the corresponding free regions
-/// can be extracted from its type and constrained to have the given
-/// outlives relationship.
-///
-/// In some cases, we have to record outlives requirements between
-/// types and regions as well. In that case, if those types include
-/// any regions, those regions are recorded as `ReClosureBound`
-/// instances assigned one of these same indices. Those regions will
-/// be substituted away by the creator. We use `ReClosureBound` in
-/// that case because the regions must be allocated in the global
-/// `TyCtxt`, and hence we cannot use `ReVar` (which is what we use
-/// internally within the rest of the NLL code).
-#[derive(Clone, Debug, RustcEncodable, RustcDecodable, HashStable)]
-pub struct ClosureRegionRequirements<'tcx> {
-    /// The number of external regions defined on the closure. In our
-    /// example above, it would be 3 -- one for `'static`, then `'1`
-    /// and `'2`. This is just used for a sanity check later on, to
-    /// make sure that the number of regions we see at the callsite
-    /// matches.
-    pub num_external_vids: usize,
-
-    /// Requirements between the various free regions defined in
-    /// indices.
-    pub outlives_requirements: Vec<ClosureOutlivesRequirement<'tcx>>,
-}
-
-/// Indicates an outlives-constraint between a type or between two
-/// free regions declared on the closure.
-#[derive(Copy, Clone, Debug, RustcEncodable, RustcDecodable, HashStable)]
-pub struct ClosureOutlivesRequirement<'tcx> {
-    // This region or type ...
-    pub subject: ClosureOutlivesSubject<'tcx>,
-
-    // ... must outlive this one.
-    pub outlived_free_region: ty::RegionVid,
-
-    // If not, report an error here ...
-    pub blame_span: Span,
-
-    // ... due to this reason.
-    pub category: ConstraintCategory,
-}
-
-/// Outlives-constraints can be categorized to determine whether and why they
-/// are interesting (for error reporting). Order of variants indicates sort
-/// order of the category, thereby influencing diagnostic output.
-///
-/// See also [rustc_mir::borrow_check::nll::constraints].
-#[derive(
-    Copy,
-    Clone,
-    Debug,
-    Eq,
-    PartialEq,
-    PartialOrd,
-    Ord,
-    Hash,
-    RustcEncodable,
-    RustcDecodable,
-    HashStable
-)]
-pub enum ConstraintCategory {
-    Return,
-    Yield,
-    UseAsConst,
-    UseAsStatic,
-    TypeAnnotation,
-    Cast,
-
-    /// A constraint that came from checking the body of a closure.
-    ///
-    /// We try to get the category that the closure used when reporting this.
-    ClosureBounds,
-    CallArgument,
-    CopyBound,
-    SizedBound,
-    Assignment,
-    OpaqueType,
-
-    /// A "boring" constraint (caused by the given location) is one that
-    /// the user probably doesn't want to see described in diagnostics,
-    /// because it is kind of an artifact of the type system setup.
-    /// Example: `x = Foo { field: y }` technically creates
-    /// intermediate regions representing the "type of `Foo { field: y
-    /// }`", and data flows from `y` into those variables, but they
-    /// are not very interesting. The assignment into `x` on the other
-    /// hand might be.
-    Boring,
-    // Boring and applicable everywhere.
-    BoringNoLocation,
-
-    /// A constraint that doesn't correspond to anything the user sees.
-    Internal,
-}
-
-/// The subject of a `ClosureOutlivesRequirement` -- that is, the thing
-/// that must outlive some region.
-#[derive(Copy, Clone, Debug, RustcEncodable, RustcDecodable, HashStable)]
-pub enum ClosureOutlivesSubject<'tcx> {
-    /// Subject is a type, typically a type parameter, but could also
-    /// be a projection. Indicates a requirement like `T: 'a` being
-    /// passed to the caller, where the type here is `T`.
-    ///
-    /// The type here is guaranteed not to contain any free regions at
-    /// present.
-    Ty(Ty<'tcx>),
-
-    /// Subject is a free region from the closure. Indicates a requirement
-    /// like `'a: 'b` being passed to the caller; the region here is `'a`.
-    Region(ty::RegionVid),
-}
-
 /*
  * `TypeFoldable` implementations for MIR types
 */
diff --git a/src/librustc/mir/query.rs b/src/librustc/mir/query.rs
new file mode 100644
index 0000000000000..34f58ab89b107
--- /dev/null
+++ b/src/librustc/mir/query.rs
@@ -0,0 +1,223 @@
+//! Values computed by queries that use MIR.
+
+use crate::ty::{self, Ty};
+use rustc_data_structures::sync::Lrc;
+use rustc_hir as hir;
+use rustc_index::bit_set::BitMatrix;
+use rustc_index::vec::IndexVec;
+use rustc_span::{Span, Symbol};
+use rustc_target::abi::VariantIdx;
+use smallvec::SmallVec;
+
+use super::{Field, SourceInfo};
+
+#[derive(Copy, Clone, PartialEq, RustcEncodable, RustcDecodable, HashStable)]
+pub enum UnsafetyViolationKind {
+    General,
+    /// Permitted both in `const fn`s and regular `fn`s.
+    GeneralAndConstFn,
+    BorrowPacked(hir::HirId),
+}
+
+#[derive(Copy, Clone, PartialEq, RustcEncodable, RustcDecodable, HashStable)]
+pub struct UnsafetyViolation {
+    pub source_info: SourceInfo,
+    pub description: Symbol,
+    pub details: Symbol,
+    pub kind: UnsafetyViolationKind,
+}
+
+#[derive(Clone, RustcEncodable, RustcDecodable, HashStable)]
+pub struct UnsafetyCheckResult {
+    /// Violations that are propagated *upwards* from this function.
+    pub violations: Lrc<[UnsafetyViolation]>,
+    /// `unsafe` blocks in this function, along with whether they are used. This is
+    /// used for the "unused_unsafe" lint.
+    pub unsafe_blocks: Lrc<[(hir::HirId, bool)]>,
+}
+
+rustc_index::newtype_index! {
+    pub struct GeneratorSavedLocal {
+        derive [HashStable]
+        DEBUG_FORMAT = "_{}",
+    }
+}
+
+/// The layout of generator state.
+#[derive(Clone, Debug, RustcEncodable, RustcDecodable, HashStable, TypeFoldable)]
+pub struct GeneratorLayout<'tcx> {
+    /// The type of every local stored inside the generator.
+    pub field_tys: IndexVec<GeneratorSavedLocal, Ty<'tcx>>,
+
+    /// Which of the above fields are in each variant. Note that one field may
+    /// be stored in multiple variants.
+    pub variant_fields: IndexVec<VariantIdx, IndexVec<Field, GeneratorSavedLocal>>,
+
+    /// Which saved locals are storage-live at the same time. Locals that do not
+    /// have conflicts with each other are allowed to overlap in the computed
+    /// layout.
+    pub storage_conflicts: BitMatrix<GeneratorSavedLocal, GeneratorSavedLocal>,
+}
+
+#[derive(Clone, Debug, RustcEncodable, RustcDecodable, HashStable)]
+pub struct BorrowCheckResult<'tcx> {
+    pub closure_requirements: Option<ClosureRegionRequirements<'tcx>>,
+    pub used_mut_upvars: SmallVec<[Field; 8]>,
+}
+
+/// The result of the `mir_const_qualif` query.
+///
+/// Each field corresponds to an implementer of the `Qualif` trait in
+/// `librustc_mir/transform/check_consts/qualifs.rs`. See that file for more information on each
+/// `Qualif`.
+#[derive(Clone, Copy, Debug, Default, RustcEncodable, RustcDecodable, HashStable)]
+pub struct ConstQualifs {
+    pub has_mut_interior: bool,
+    pub needs_drop: bool,
+}
+
+/// After we borrow check a closure, we are left with various
+/// requirements that we have inferred between the free regions that
+/// appear in the closure's signature or on its field types. These
+/// requirements are then verified and proved by the closure's
+/// creating function. This struct encodes those requirements.
+///
+/// The requirements are listed as being between various
+/// `RegionVid`. The 0th region refers to `'static`; subsequent region
+/// vids refer to the free regions that appear in the closure (or
+/// generator's) type, in order of appearance. (This numbering is
+/// actually defined by the `UniversalRegions` struct in the NLL
+/// region checker. See for example
+/// `UniversalRegions::closure_mapping`.) Note that we treat the free
+/// regions in the closure's type "as if" they were erased, so their
+/// precise identity is not important, only their position.
+///
+/// Example: If type check produces a closure with the closure substs:
+///
+/// ```text
+/// ClosureSubsts = [
+///     i8,                                  // the "closure kind"
+///     for<'x> fn(&'a &'x u32) -> &'x u32,  // the "closure signature"
+///     &'a String,                          // some upvar
+/// ]
+/// ```
+///
+/// here, there is one unique free region (`'a`) but it appears
+/// twice. We would "renumber" each occurrence to a unique vid, as follows:
+///
+/// ```text
+/// ClosureSubsts = [
+///     i8,                                  // the "closure kind"
+///     for<'x> fn(&'1 &'x u32) -> &'x u32,  // the "closure signature"
+///     &'2 String,                          // some upvar
+/// ]
+/// ```
+///
+/// Now the code might impose a requirement like `'1: '2`. When an
+/// instance of the closure is created, the corresponding free regions
+/// can be extracted from its type and constrained to have the given
+/// outlives relationship.
+///
+/// In some cases, we have to record outlives requirements between
+/// types and regions as well. In that case, if those types include
+/// any regions, those regions are recorded as `ReClosureBound`
+/// instances assigned one of these same indices. Those regions will
+/// be substituted away by the creator. We use `ReClosureBound` in
+/// that case because the regions must be allocated in the global
+/// `TyCtxt`, and hence we cannot use `ReVar` (which is what we use
+/// internally within the rest of the NLL code).
+#[derive(Clone, Debug, RustcEncodable, RustcDecodable, HashStable)]
+pub struct ClosureRegionRequirements<'tcx> {
+    /// The number of external regions defined on the closure. In our
+    /// example above, it would be 3 -- one for `'static`, then `'1`
+    /// and `'2`. This is just used for a sanity check later on, to
+    /// make sure that the number of regions we see at the callsite
+    /// matches.
+    pub num_external_vids: usize,
+
+    /// Requirements between the various free regions defined in
+    /// indices.
+    pub outlives_requirements: Vec<ClosureOutlivesRequirement<'tcx>>,
+}
+
+/// Indicates an outlives-constraint between a type or between two
+/// free regions declared on the closure.
+#[derive(Copy, Clone, Debug, RustcEncodable, RustcDecodable, HashStable)]
+pub struct ClosureOutlivesRequirement<'tcx> {
+    // This region or type ...
+    pub subject: ClosureOutlivesSubject<'tcx>,
+
+    // ... must outlive this one.
+    pub outlived_free_region: ty::RegionVid,
+
+    // If not, report an error here ...
+    pub blame_span: Span,
+
+    // ... due to this reason.
+    pub category: ConstraintCategory,
+}
+
+/// Outlives-constraints can be categorized to determine whether and why they
+/// are interesting (for error reporting). Order of variants indicates sort
+/// order of the category, thereby influencing diagnostic output.
+///
+/// See also [rustc_mir::borrow_check::nll::constraints].
+#[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, Hash)]
+#[derive(RustcEncodable, RustcDecodable, HashStable)]
+pub enum ConstraintCategory {
+    Return,
+    Yield,
+    UseAsConst,
+    UseAsStatic,
+    TypeAnnotation,
+    Cast,
+
+    /// A constraint that came from checking the body of a closure.
+    ///
+    /// We try to get the category that the closure used when reporting this.
+    ClosureBounds,
+    CallArgument,
+    CopyBound,
+    SizedBound,
+    Assignment,
+    OpaqueType,
+
+    /// A "boring" constraint (caused by the given location) is one that
+    /// the user probably doesn't want to see described in diagnostics,
+    /// because it is kind of an artifact of the type system setup.
+    /// Example: `x = Foo { field: y }` technically creates
+    /// intermediate regions representing the "type of `Foo { field: y
+    /// }`", and data flows from `y` into those variables, but they
+    /// are not very interesting. The assignment into `x` on the other
+    /// hand might be.
+    Boring,
+    // Boring and applicable everywhere.
+    BoringNoLocation,
+
+    /// A constraint that doesn't correspond to anything the user sees.
+    Internal,
+}
+
+/// The subject of a `ClosureOutlivesRequirement` -- that is, the thing
+/// that must outlive some region.
+#[derive(Copy, Clone, Debug, RustcEncodable, RustcDecodable, HashStable)]
+pub enum ClosureOutlivesSubject<'tcx> {
+    /// Subject is a type, typically a type parameter, but could also
+    /// be a projection. Indicates a requirement like `T: 'a` being
+    /// passed to the caller, where the type here is `T`.
+    ///
+    /// The type here is guaranteed not to contain any free regions at
+    /// present.
+    Ty(Ty<'tcx>),
+
+    /// Subject is a free region from the closure. Indicates a requirement
+    /// like `'a: 'b` being passed to the caller; the region here is `'a`.
+    Region(ty::RegionVid),
+}
+
+/// The constituent parts of an ADT or array.
+#[derive(Copy, Clone, Debug, HashStable)]
+pub struct DestructuredConst<'tcx> {
+    pub variant: VariantIdx,
+    pub fields: &'tcx [&'tcx ty::Const<'tcx>],
+}
diff --git a/src/librustc/query/mod.rs b/src/librustc/query/mod.rs
index 4a2ec9b9687f3..f95640c5549b4 100644
--- a/src/librustc/query/mod.rs
+++ b/src/librustc/query/mod.rs
@@ -505,6 +505,15 @@ rustc_queries! {
             desc { "extract field of const" }
         }
 
+        /// Destructure a constant ADT or array into its variant indent and its
+        /// field values.
+        query destructure_const(
+            key: ty::ParamEnvAnd<'tcx, &'tcx ty::Const<'tcx>>
+        ) -> mir::DestructuredConst<'tcx> {
+            no_force
+            desc { "destructure constant" }
+        }
+
         query const_caller_location(key: (rustc_span::Symbol, u32, u32)) -> &'tcx ty::Const<'tcx> {
             no_force
             desc { "get a &core::panic::Location referring to a span" }
diff --git a/src/librustc/ty/query/keys.rs b/src/librustc/ty/query/keys.rs
index 8a713e3b6a096..d64f27d9cc26c 100644
--- a/src/librustc/ty/query/keys.rs
+++ b/src/librustc/ty/query/keys.rs
@@ -142,7 +142,7 @@ impl<'tcx> Key for ty::PolyTraitRef<'tcx> {
     }
 }
 
-impl<'tcx> Key for ty::Const<'tcx> {
+impl<'tcx> Key for &'tcx ty::Const<'tcx> {
     fn query_crate(&self) -> CrateNum {
         LOCAL_CRATE
     }
diff --git a/src/librustc_ast_lowering/item.rs b/src/librustc_ast_lowering/item.rs
index 2515ca9d9466a..52925d2043ee6 100644
--- a/src/librustc_ast_lowering/item.rs
+++ b/src/librustc_ast_lowering/item.rs
@@ -71,6 +71,12 @@ impl<'a, 'lowering, 'hir> Visitor<'a> for ItemLowerer<'a, 'lowering, 'hir> {
             self.lctx.with_parent_item_lifetime_defs(hir_id, |this| {
                 let this = &mut ItemLowerer { lctx: this };
                 if let ItemKind::Impl(.., ref opt_trait_ref, _, _) = item.kind {
+                    if opt_trait_ref.as_ref().map(|tr| tr.constness.is_some()).unwrap_or(false) {
+                        this.lctx
+                            .diagnostic()
+                            .span_err(item.span, "const trait impls are not yet implemented");
+                    }
+
                     this.with_trait_impl_ref(opt_trait_ref, |this| visit::walk_item(this, item));
                 } else {
                     visit::walk_item(this, item);
diff --git a/src/librustc_ast_lowering/lib.rs b/src/librustc_ast_lowering/lib.rs
index bc3dfecd4a675..be7269799ebb6 100644
--- a/src/librustc_ast_lowering/lib.rs
+++ b/src/librustc_ast_lowering/lib.rs
@@ -2577,6 +2577,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         p: &PolyTraitRef,
         mut itctx: ImplTraitContext<'_, 'hir>,
     ) -> hir::PolyTraitRef<'hir> {
+        if p.trait_ref.constness.is_some() {
+            self.diagnostic().span_err(p.span, "`?const` on trait bounds is not yet implemented");
+        }
+
         let bound_generic_params = self.lower_generic_params(
             &p.bound_generic_params,
             &NodeMap::default(),
diff --git a/src/librustc_expand/build.rs b/src/librustc_expand/build.rs
index 11f94ab2e6279..bd3d6b589d00a 100644
--- a/src/librustc_expand/build.rs
+++ b/src/librustc_expand/build.rs
@@ -110,7 +110,7 @@ impl<'a> ExtCtxt<'a> {
     }
 
     pub fn trait_ref(&self, path: ast::Path) -> ast::TraitRef {
-        ast::TraitRef { path, ref_id: ast::DUMMY_NODE_ID }
+        ast::TraitRef { path, constness: None, ref_id: ast::DUMMY_NODE_ID }
     }
 
     pub fn poly_trait_ref(&self, span: Span, path: ast::Path) -> ast::PolyTraitRef {
diff --git a/src/librustc_feature/active.rs b/src/librustc_feature/active.rs
index 8cb1684491bb8..6a15cc5cb0fce 100644
--- a/src/librustc_feature/active.rs
+++ b/src/librustc_feature/active.rs
@@ -544,6 +544,12 @@ declare_features! (
     /// For example, you can write `x @ Some(y)`.
     (active, bindings_after_at, "1.41.0", Some(65490), None),
 
+    /// Allows `impl const Trait for T` syntax.
+    (active, const_trait_impl, "1.42.0", Some(67792), None),
+
+    /// Allows `T: ?const Trait` syntax in bounds.
+    (active, const_trait_bound_opt_out, "1.42.0", Some(67794), None),
+
     // -------------------------------------------------------------------------
     // feature-group-end: actual feature gates
     // -------------------------------------------------------------------------
@@ -559,4 +565,6 @@ pub const INCOMPLETE_FEATURES: &[Symbol] = &[
     sym::or_patterns,
     sym::let_chains,
     sym::raw_dylib,
+    sym::const_trait_impl,
+    sym::const_trait_bound_opt_out,
 ];
diff --git a/src/librustc_interface/Cargo.toml b/src/librustc_interface/Cargo.toml
index be60b75bc47eb..ece3189780cce 100644
--- a/src/librustc_interface/Cargo.toml
+++ b/src/librustc_interface/Cargo.toml
@@ -30,6 +30,7 @@ rustc_codegen_llvm = { path = "../librustc_codegen_llvm", optional = true }
 rustc_hir = { path = "../librustc_hir" }
 rustc_metadata = { path = "../librustc_metadata" }
 rustc_mir = { path = "../librustc_mir" }
+rustc_mir_build = { path = "../librustc_mir_build" }
 rustc_passes = { path = "../librustc_passes" }
 rustc_typeck = { path = "../librustc_typeck" }
 rustc_lint = { path = "../librustc_lint" }
diff --git a/src/librustc_interface/passes.rs b/src/librustc_interface/passes.rs
index 432f79bba030a..24943dd9e2e82 100644
--- a/src/librustc_interface/passes.rs
+++ b/src/librustc_interface/passes.rs
@@ -28,6 +28,7 @@ use rustc_expand::base::ExtCtxt;
 use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
 use rustc_incremental;
 use rustc_mir as mir;
+use rustc_mir_build as mir_build;
 use rustc_parse::{parse_crate_from_file, parse_crate_from_source_str};
 use rustc_passes::{self, ast_validation, hir_stats, layout_test};
 use rustc_plugin_impl as plugin;
@@ -678,6 +679,7 @@ pub fn default_provide(providers: &mut ty::query::Providers<'_>) {
     plugin::build::provide(providers);
     rustc::hir::provide(providers);
     mir::provide(providers);
+    mir_build::provide(providers);
     rustc_privacy::provide(providers);
     typeck::provide(providers);
     ty::provide(providers);
diff --git a/src/librustc_mir/Cargo.toml b/src/librustc_mir/Cargo.toml
index 9b6908dbbe789..f9b61b9e2c9d8 100644
--- a/src/librustc_mir/Cargo.toml
+++ b/src/librustc_mir/Cargo.toml
@@ -10,7 +10,6 @@ path = "lib.rs"
 doctest = false
 
 [dependencies]
-arena = { path = "../libarena" }
 either = "1.5.0"
 dot = { path = "../libgraphviz", package = "graphviz" }
 itertools = "0.8"
diff --git a/src/librustc_mir/const_eval.rs b/src/librustc_mir/const_eval.rs
index ac04ae285884b..4a2c1d3cfb97d 100644
--- a/src/librustc_mir/const_eval.rs
+++ b/src/librustc_mir/const_eval.rs
@@ -59,15 +59,32 @@ pub(crate) fn const_caller_location<'tcx>(
     tcx.mk_const(loc_const)
 }
 
-// this function uses `unwrap` copiously, because an already validated constant must have valid
-// fields and can thus never fail outside of compiler bugs
-pub(crate) fn const_variant_index<'tcx>(
+// this function uses `unwrap` copiously, because an already validated constant
+// must have valid fields and can thus never fail outside of compiler bugs
+pub(crate) fn destructure_const<'tcx>(
     tcx: TyCtxt<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
     val: &'tcx ty::Const<'tcx>,
-) -> VariantIdx {
-    trace!("const_variant_index: {:?}", val);
+) -> mir::DestructuredConst<'tcx> {
+    trace!("destructure_const: {:?}", val);
     let ecx = mk_eval_cx(tcx, DUMMY_SP, param_env, false);
     let op = ecx.eval_const_to_op(val, None).unwrap();
-    ecx.read_discriminant(op).unwrap().1
+
+    let variant = ecx.read_discriminant(op).unwrap().1;
+
+    let field_count = match val.ty.kind {
+        ty::Array(_, len) => len.eval_usize(tcx, param_env),
+        ty::Adt(def, _) => def.variants[variant].fields.len() as u64,
+        ty::Tuple(substs) => substs.len() as u64,
+        _ => bug!("cannot destructure constant {:?}", val),
+    };
+
+    let down = ecx.operand_downcast(op, variant).unwrap();
+    let fields_iter = (0..field_count).map(|i| {
+        let field_op = ecx.operand_field(down, i).unwrap();
+        op_to_const(&ecx, field_op)
+    });
+    let fields = tcx.arena.alloc_from_iter(fields_iter);
+
+    mir::DestructuredConst { variant, fields }
 }
diff --git a/src/librustc_mir/lib.rs b/src/librustc_mir/lib.rs
index 32b35c4139dad..14f2f0b85f211 100644
--- a/src/librustc_mir/lib.rs
+++ b/src/librustc_mir/lib.rs
@@ -39,12 +39,9 @@ extern crate rustc;
 extern crate syntax;
 
 mod borrow_check;
-mod build;
 pub mod const_eval;
 pub mod dataflow;
-mod hair;
 pub mod interpret;
-mod lints;
 pub mod monomorphize;
 mod shim;
 pub mod transform;
@@ -59,10 +56,13 @@ pub fn provide(providers: &mut Providers<'_>) {
     monomorphize::partitioning::provide(providers);
     providers.const_eval_validated = const_eval::const_eval_validated_provider;
     providers.const_eval_raw = const_eval::const_eval_raw_provider;
-    providers.check_match = hair::pattern::check_match;
     providers.const_caller_location = const_eval::const_caller_location;
     providers.const_field = |tcx, param_env_and_value| {
         let (param_env, (value, field)) = param_env_and_value.into_parts();
         const_eval::const_field(tcx, param_env, None, field, value)
     };
+    providers.destructure_const = |tcx, param_env_and_value| {
+        let (param_env, value) = param_env_and_value.into_parts();
+        const_eval::destructure_const(tcx, param_env, value)
+    }
 }
diff --git a/src/librustc_mir/transform/mod.rs b/src/librustc_mir/transform/mod.rs
index 82c31a09ce377..e72fee9d98c57 100644
--- a/src/librustc_mir/transform/mod.rs
+++ b/src/librustc_mir/transform/mod.rs
@@ -1,4 +1,4 @@
-use crate::{build, shim};
+use crate::{shim, util};
 use rustc::hir::intravisit::{self, NestedVisitorMap, Visitor};
 use rustc::mir::{BodyAndCache, ConstQualifs, MirPhase, Promoted};
 use rustc::ty::query::Providers;
@@ -40,7 +40,6 @@ pub(crate) fn provide(providers: &mut Providers<'_>) {
     self::check_unsafety::provide(providers);
     *providers = Providers {
         mir_keys,
-        mir_built,
         mir_const,
         mir_const_qualif,
         mir_validated,
@@ -96,11 +95,6 @@ fn mir_keys(tcx: TyCtxt<'_>, krate: CrateNum) -> &DefIdSet {
     tcx.arena.alloc(set)
 }
 
-fn mir_built(tcx: TyCtxt<'_>, def_id: DefId) -> &Steal<BodyAndCache<'_>> {
-    let mir = build::mir_build(tcx, def_id);
-    tcx.alloc_steal_mir(mir)
-}
-
 /// Where a specific `mir::Body` comes from.
 #[derive(Debug, Copy, Clone)]
 pub struct MirSource<'tcx> {
@@ -220,6 +214,9 @@ fn mir_const(tcx: TyCtxt<'_>, def_id: DefId) -> &Steal<BodyAndCache<'_>> {
     let _ = tcx.unsafety_check_result(def_id);
 
     let mut body = tcx.mir_built(def_id).steal();
+
+    util::dump_mir(tcx, None, "mir_map", &0, MirSource::item(def_id), &body, |_, _| Ok(()));
+
     run_passes(
         tcx,
         &mut body,
diff --git a/src/librustc_mir_build/Cargo.toml b/src/librustc_mir_build/Cargo.toml
new file mode 100644
index 0000000000000..79c7303275597
--- /dev/null
+++ b/src/librustc_mir_build/Cargo.toml
@@ -0,0 +1,28 @@
+[package]
+authors = ["The Rust Project Developers"]
+name = "rustc_mir_build"
+version = "0.0.0"
+edition = "2018"
+
+[lib]
+name = "rustc_mir_build"
+path = "lib.rs"
+doctest = false
+
+[dependencies]
+arena = { path = "../libarena" }
+itertools = "0.8"
+log = "0.4"
+rustc = { path = "../librustc" }
+rustc_apfloat = { path = "../librustc_apfloat" }
+rustc_data_structures = { path = "../librustc_data_structures" }
+rustc_index = { path = "../librustc_index" }
+rustc_errors = { path = "../librustc_errors" }
+rustc_hir = { path = "../librustc_hir" }
+rustc_macros = { path = "../librustc_macros" }
+rustc_serialize = { path = "../libserialize", package = "serialize" }
+rustc_span = { path = "../librustc_span" }
+rustc_target = { path = "../librustc_target" }
+syntax = { path = "../libsyntax" }
+smallvec = { version = "1.0", features = ["union", "may_dangle"] }
+rustc_error_codes = { path = "../librustc_error_codes" }
diff --git a/src/librustc_mir/build/block.rs b/src/librustc_mir_build/build/block.rs
similarity index 99%
rename from src/librustc_mir/build/block.rs
rename to src/librustc_mir_build/build/block.rs
index 2e133a035ee76..c517d3113c659 100644
--- a/src/librustc_mir/build/block.rs
+++ b/src/librustc_mir_build/build/block.rs
@@ -7,7 +7,7 @@ use rustc_hir as hir;
 use rustc_span::Span;
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
-    pub fn ast_block(
+    crate fn ast_block(
         &mut self,
         destination: &Place<'tcx>,
         block: BasicBlock,
diff --git a/src/librustc_mir/build/cfg.rs b/src/librustc_mir_build/build/cfg.rs
similarity index 80%
rename from src/librustc_mir/build/cfg.rs
rename to src/librustc_mir_build/build/cfg.rs
index 553701c91eead..e1971102832b5 100644
--- a/src/librustc_mir/build/cfg.rs
+++ b/src/librustc_mir_build/build/cfg.rs
@@ -4,33 +4,33 @@ use crate::build::CFG;
 use rustc::mir::*;
 
 impl<'tcx> CFG<'tcx> {
-    pub fn block_data(&self, blk: BasicBlock) -> &BasicBlockData<'tcx> {
+    crate fn block_data(&self, blk: BasicBlock) -> &BasicBlockData<'tcx> {
         &self.basic_blocks[blk]
     }
 
-    pub fn block_data_mut(&mut self, blk: BasicBlock) -> &mut BasicBlockData<'tcx> {
+    crate fn block_data_mut(&mut self, blk: BasicBlock) -> &mut BasicBlockData<'tcx> {
         &mut self.basic_blocks[blk]
     }
 
     // llvm.org/PR32488 makes this function use an excess of stack space. Mark
     // it as #[inline(never)] to keep rustc's stack use in check.
     #[inline(never)]
-    pub fn start_new_block(&mut self) -> BasicBlock {
+    crate fn start_new_block(&mut self) -> BasicBlock {
         self.basic_blocks.push(BasicBlockData::new(None))
     }
 
-    pub fn start_new_cleanup_block(&mut self) -> BasicBlock {
+    crate fn start_new_cleanup_block(&mut self) -> BasicBlock {
         let bb = self.start_new_block();
         self.block_data_mut(bb).is_cleanup = true;
         bb
     }
 
-    pub fn push(&mut self, block: BasicBlock, statement: Statement<'tcx>) {
+    crate fn push(&mut self, block: BasicBlock, statement: Statement<'tcx>) {
         debug!("push({:?}, {:?})", block, statement);
         self.block_data_mut(block).statements.push(statement);
     }
 
-    pub fn push_assign(
+    crate fn push_assign(
         &mut self,
         block: BasicBlock,
         source_info: SourceInfo,
@@ -43,7 +43,7 @@ impl<'tcx> CFG<'tcx> {
         );
     }
 
-    pub fn push_assign_constant(
+    crate fn push_assign_constant(
         &mut self,
         block: BasicBlock,
         source_info: SourceInfo,
@@ -53,7 +53,7 @@ impl<'tcx> CFG<'tcx> {
         self.push_assign(block, source_info, temp, Rvalue::Use(Operand::Constant(box constant)));
     }
 
-    pub fn push_assign_unit(
+    crate fn push_assign_unit(
         &mut self,
         block: BasicBlock,
         source_info: SourceInfo,
@@ -67,7 +67,7 @@ impl<'tcx> CFG<'tcx> {
         );
     }
 
-    pub fn push_fake_read(
+    crate fn push_fake_read(
         &mut self,
         block: BasicBlock,
         source_info: SourceInfo,
@@ -79,7 +79,7 @@ impl<'tcx> CFG<'tcx> {
         self.push(block, stmt);
     }
 
-    pub fn terminate(
+    crate fn terminate(
         &mut self,
         block: BasicBlock,
         source_info: SourceInfo,
@@ -96,7 +96,7 @@ impl<'tcx> CFG<'tcx> {
     }
 
     /// In the `origin` block, push a `goto -> target` terminator.
-    pub fn goto(&mut self, origin: BasicBlock, source_info: SourceInfo, target: BasicBlock) {
+    crate fn goto(&mut self, origin: BasicBlock, source_info: SourceInfo, target: BasicBlock) {
         self.terminate(origin, source_info, TerminatorKind::Goto { target })
     }
 }
diff --git a/src/librustc_mir/build/expr/as_constant.rs b/src/librustc_mir_build/build/expr/as_constant.rs
similarity index 95%
rename from src/librustc_mir/build/expr/as_constant.rs
rename to src/librustc_mir_build/build/expr/as_constant.rs
index ceac2a0cd030f..e4856262975e7 100644
--- a/src/librustc_mir/build/expr/as_constant.rs
+++ b/src/librustc_mir_build/build/expr/as_constant.rs
@@ -8,7 +8,7 @@ use rustc::ty::CanonicalUserTypeAnnotation;
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Compile `expr`, yielding a compile-time constant. Assumes that
     /// `expr` is a valid compile-time constant!
-    pub fn as_constant<M>(&mut self, expr: M) -> Constant<'tcx>
+    crate fn as_constant<M>(&mut self, expr: M) -> Constant<'tcx>
     where
         M: Mirror<'tcx, Output = Expr<'tcx>>,
     {
diff --git a/src/librustc_mir/build/expr/as_operand.rs b/src/librustc_mir_build/build/expr/as_operand.rs
similarity index 95%
rename from src/librustc_mir/build/expr/as_operand.rs
rename to src/librustc_mir_build/build/expr/as_operand.rs
index b969932a73646..efe328d2b3c1e 100644
--- a/src/librustc_mir/build/expr/as_operand.rs
+++ b/src/librustc_mir_build/build/expr/as_operand.rs
@@ -13,7 +13,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// The operand returned from this function will *not be valid* after
     /// an ExprKind::Scope is passed, so please do *not* return it from
     /// functions to avoid bad miscompiles.
-    pub fn as_local_operand<M>(&mut self, block: BasicBlock, expr: M) -> BlockAnd<Operand<'tcx>>
+    crate fn as_local_operand<M>(&mut self, block: BasicBlock, expr: M) -> BlockAnd<Operand<'tcx>>
     where
         M: Mirror<'tcx, Output = Expr<'tcx>>,
     {
@@ -27,7 +27,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// this time.
     ///
     /// The operand is known to be live until the end of `scope`.
-    pub fn as_operand<M>(
+    crate fn as_operand<M>(
         &mut self,
         block: BasicBlock,
         scope: Option<region::Scope>,
diff --git a/src/librustc_mir/build/expr/as_place.rs b/src/librustc_mir_build/build/expr/as_place.rs
similarity index 97%
rename from src/librustc_mir/build/expr/as_place.rs
rename to src/librustc_mir_build/build/expr/as_place.rs
index 29eac5e4b3f54..5d4699613d836 100644
--- a/src/librustc_mir/build/expr/as_place.rs
+++ b/src/librustc_mir_build/build/expr/as_place.rs
@@ -24,7 +24,7 @@ struct PlaceBuilder<'tcx> {
     projection: Vec<PlaceElem<'tcx>>,
 }
 
-impl PlaceBuilder<'tcx> {
+impl<'tcx> PlaceBuilder<'tcx> {
     fn into_place(self, tcx: TyCtxt<'tcx>) -> Place<'tcx> {
         Place { base: self.base, projection: tcx.intern_place_elems(&self.projection) }
     }
@@ -47,13 +47,13 @@ impl PlaceBuilder<'tcx> {
     }
 }
 
-impl From<Local> for PlaceBuilder<'tcx> {
+impl<'tcx> From<Local> for PlaceBuilder<'tcx> {
     fn from(local: Local) -> Self {
         Self { base: local.into(), projection: Vec::new() }
     }
 }
 
-impl From<PlaceBase<'tcx>> for PlaceBuilder<'tcx> {
+impl<'tcx> From<PlaceBase<'tcx>> for PlaceBuilder<'tcx> {
     fn from(base: PlaceBase<'tcx>) -> Self {
         Self { base, projection: Vec::new() }
     }
@@ -72,7 +72,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Extra care is needed if any user code is allowed to run between calling
     /// this method and using it, as is the case for `match` and index
     /// expressions.
-    pub fn as_place<M>(&mut self, mut block: BasicBlock, expr: M) -> BlockAnd<Place<'tcx>>
+    crate fn as_place<M>(&mut self, mut block: BasicBlock, expr: M) -> BlockAnd<Place<'tcx>>
     where
         M: Mirror<'tcx, Output = Expr<'tcx>>,
     {
@@ -95,7 +95,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// place. The place itself may or may not be mutable:
     /// * If this expr is a place expr like a.b, then we will return that place.
     /// * Otherwise, a temporary is created: in that event, it will be an immutable temporary.
-    pub fn as_read_only_place<M>(&mut self, mut block: BasicBlock, expr: M) -> BlockAnd<Place<'tcx>>
+    crate fn as_read_only_place<M>(
+        &mut self,
+        mut block: BasicBlock,
+        expr: M,
+    ) -> BlockAnd<Place<'tcx>>
     where
         M: Mirror<'tcx, Output = Expr<'tcx>>,
     {
diff --git a/src/librustc_mir/build/expr/as_rvalue.rs b/src/librustc_mir_build/build/expr/as_rvalue.rs
similarity index 99%
rename from src/librustc_mir/build/expr/as_rvalue.rs
rename to src/librustc_mir_build/build/expr/as_rvalue.rs
index 34b0cbf3b25cd..7d16e60e92dad 100644
--- a/src/librustc_mir/build/expr/as_rvalue.rs
+++ b/src/librustc_mir_build/build/expr/as_rvalue.rs
@@ -18,7 +18,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// The operand returned from this function will *not be valid* after
     /// an ExprKind::Scope is passed, so please do *not* return it from
     /// functions to avoid bad miscompiles.
-    pub fn as_local_rvalue<M>(&mut self, block: BasicBlock, expr: M) -> BlockAnd<Rvalue<'tcx>>
+    crate fn as_local_rvalue<M>(&mut self, block: BasicBlock, expr: M) -> BlockAnd<Rvalue<'tcx>>
     where
         M: Mirror<'tcx, Output = Expr<'tcx>>,
     {
@@ -276,7 +276,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         }
     }
 
-    pub fn build_binary_op(
+    crate fn build_binary_op(
         &mut self,
         mut block: BasicBlock,
         op: BinOp,
diff --git a/src/librustc_mir/build/expr/as_temp.rs b/src/librustc_mir_build/build/expr/as_temp.rs
similarity index 99%
rename from src/librustc_mir/build/expr/as_temp.rs
rename to src/librustc_mir_build/build/expr/as_temp.rs
index 3f711044b154f..f47987c56174c 100644
--- a/src/librustc_mir/build/expr/as_temp.rs
+++ b/src/librustc_mir_build/build/expr/as_temp.rs
@@ -11,7 +11,7 @@ use rustc_span::symbol::sym;
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Compile `expr` into a fresh temporary. This is used when building
     /// up rvalues so as to freeze the value that will be consumed.
-    pub fn as_temp<M>(
+    crate fn as_temp<M>(
         &mut self,
         block: BasicBlock,
         temp_lifetime: Option<region::Scope>,
diff --git a/src/librustc_mir/build/expr/category.rs b/src/librustc_mir_build/build/expr/category.rs
similarity index 96%
rename from src/librustc_mir/build/expr/category.rs
rename to src/librustc_mir_build/build/expr/category.rs
index b35616c11fbd0..c4d340953c925 100644
--- a/src/librustc_mir/build/expr/category.rs
+++ b/src/librustc_mir_build/build/expr/category.rs
@@ -1,7 +1,7 @@
 use crate::hair::*;
 
 #[derive(Debug, PartialEq)]
-pub enum Category {
+crate enum Category {
     // An assignable memory location like `x`, `x.f`, `foo()[3]`, that
     // sort of thing. Something that could appear on the LHS of an `=`
     // sign.
@@ -19,7 +19,7 @@ pub enum Category {
 // Rvalues fall into different "styles" that will determine which fn
 // is best suited to generate them.
 #[derive(Debug, PartialEq)]
-pub enum RvalueFunc {
+crate enum RvalueFunc {
     // Best generated by `into`. This is generally exprs that
     // cause branching, like `match`, but also includes calls.
     Into,
@@ -31,7 +31,7 @@ pub enum RvalueFunc {
 /// Determines the category for a given expression. Note that scope
 /// and paren expressions have no category.
 impl Category {
-    pub fn of(ek: &ExprKind<'_>) -> Option<Category> {
+    crate fn of(ek: &ExprKind<'_>) -> Option<Category> {
         match *ek {
             ExprKind::Scope { .. } => None,
 
diff --git a/src/librustc_mir/build/expr/into.rs b/src/librustc_mir_build/build/expr/into.rs
similarity index 99%
rename from src/librustc_mir/build/expr/into.rs
rename to src/librustc_mir_build/build/expr/into.rs
index 2cf2b21b65ace..503dfb6ef5b61 100644
--- a/src/librustc_mir/build/expr/into.rs
+++ b/src/librustc_mir_build/build/expr/into.rs
@@ -14,7 +14,7 @@ use rustc_target::spec::abi::Abi;
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Compile `expr`, storing the result into `destination`, which
     /// is assumed to be uninitialized.
-    pub fn into_expr(
+    crate fn into_expr(
         &mut self,
         destination: &Place<'tcx>,
         mut block: BasicBlock,
diff --git a/src/librustc_mir/build/expr/mod.rs b/src/librustc_mir_build/build/expr/mod.rs
similarity index 100%
rename from src/librustc_mir/build/expr/mod.rs
rename to src/librustc_mir_build/build/expr/mod.rs
diff --git a/src/librustc_mir/build/expr/stmt.rs b/src/librustc_mir_build/build/expr/stmt.rs
similarity index 99%
rename from src/librustc_mir/build/expr/stmt.rs
rename to src/librustc_mir_build/build/expr/stmt.rs
index ff7049278ed72..fd61cb833b1bc 100644
--- a/src/librustc_mir/build/expr/stmt.rs
+++ b/src/librustc_mir_build/build/expr/stmt.rs
@@ -10,7 +10,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// (e.g., `some().code(&here());`) then `opt_stmt_span` is the
     /// span of that statement (including its semicolon, if any).
     /// The scope is used if a statement temporary must be dropped.
-    pub fn stmt_expr(
+    crate fn stmt_expr(
         &mut self,
         mut block: BasicBlock,
         expr: Expr<'tcx>,
diff --git a/src/librustc_mir/build/into.rs b/src/librustc_mir_build/build/into.rs
similarity index 90%
rename from src/librustc_mir/build/into.rs
rename to src/librustc_mir_build/build/into.rs
index 9c3ea5f7c5619..1a2a9d2bc05fc 100644
--- a/src/librustc_mir/build/into.rs
+++ b/src/librustc_mir_build/build/into.rs
@@ -18,7 +18,12 @@ pub(in crate::build) trait EvalInto<'tcx> {
 }
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
-    pub fn into<E>(&mut self, destination: &Place<'tcx>, block: BasicBlock, expr: E) -> BlockAnd<()>
+    crate fn into<E>(
+        &mut self,
+        destination: &Place<'tcx>,
+        block: BasicBlock,
+        expr: E,
+    ) -> BlockAnd<()>
     where
         E: EvalInto<'tcx>,
     {
diff --git a/src/librustc_mir/build/matches/mod.rs b/src/librustc_mir_build/build/matches/mod.rs
similarity index 99%
rename from src/librustc_mir/build/matches/mod.rs
rename to src/librustc_mir_build/build/matches/mod.rs
index 7eea90befb008..9781b9fcbe1bc 100644
--- a/src/librustc_mir/build/matches/mod.rs
+++ b/src/librustc_mir_build/build/matches/mod.rs
@@ -81,7 +81,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     ///
     /// * From each prebinding block to the next prebinding block.
     /// * From each otherwise block to the next prebinding block.
-    pub fn match_expr(
+    crate fn match_expr(
         &mut self,
         destination: &Place<'tcx>,
         span: Span,
@@ -417,7 +417,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         }
     }
 
-    pub fn place_into_pattern(
+    crate fn place_into_pattern(
         &mut self,
         block: BasicBlock,
         irrefutable_pat: Pat<'tcx>,
@@ -488,7 +488,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// scope for the bindings in these patterns, if such a scope had to be
     /// created. NOTE: Declaring the bindings should always be done in their
     /// drop scope.
-    pub fn declare_bindings(
+    crate fn declare_bindings(
         &mut self,
         mut visibility_scope: Option<SourceScope>,
         scope_span: Span,
@@ -525,7 +525,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         visibility_scope
     }
 
-    pub fn storage_live_binding(
+    crate fn storage_live_binding(
         &mut self,
         block: BasicBlock,
         var: HirId,
@@ -540,7 +540,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         Place::from(local_id)
     }
 
-    pub fn schedule_drop_for_binding(&mut self, var: HirId, span: Span, for_guard: ForGuard) {
+    crate fn schedule_drop_for_binding(&mut self, var: HirId, span: Span, for_guard: ForGuard) {
         let local_id = self.var_local_id(var, for_guard);
         let region_scope = self.hir.region_scope_tree.var_scope(var.local_id);
         self.schedule_drop(span, region_scope, local_id, DropKind::Value);
@@ -641,7 +641,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 }
 
 #[derive(Debug)]
-pub struct Candidate<'pat, 'tcx> {
+crate struct Candidate<'pat, 'tcx> {
     // span of the original pattern that gave rise to this candidate
     span: Span,
 
@@ -685,7 +685,7 @@ struct Ascription<'tcx> {
 }
 
 #[derive(Clone, Debug)]
-pub struct MatchPair<'pat, 'tcx> {
+crate struct MatchPair<'pat, 'tcx> {
     // this place...
     place: Place<'tcx>,
 
@@ -739,7 +739,7 @@ enum TestKind<'tcx> {
 }
 
 #[derive(Debug)]
-pub struct Test<'tcx> {
+crate struct Test<'tcx> {
     span: Span,
     kind: TestKind<'tcx>,
 }
@@ -747,7 +747,7 @@ pub struct Test<'tcx> {
 /// ArmHasGuard is isomorphic to a boolean flag. It indicates whether
 /// a match arm has a guard expression attached to it.
 #[derive(Copy, Clone, Debug)]
-pub(crate) struct ArmHasGuard(pub bool);
+crate struct ArmHasGuard(crate bool);
 
 ///////////////////////////////////////////////////////////////////////////
 // Main matching algorithm
diff --git a/src/librustc_mir/build/matches/simplify.rs b/src/librustc_mir_build/build/matches/simplify.rs
similarity index 98%
rename from src/librustc_mir/build/matches/simplify.rs
rename to src/librustc_mir_build/build/matches/simplify.rs
index 9dbf8989cc5aa..a5f691add65c1 100644
--- a/src/librustc_mir/build/matches/simplify.rs
+++ b/src/librustc_mir_build/build/matches/simplify.rs
@@ -24,7 +24,7 @@ use syntax::attr::{SignedInt, UnsignedInt};
 use std::mem;
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
-    pub fn simplify_candidate<'pat>(&mut self, candidate: &mut Candidate<'pat, 'tcx>) {
+    crate fn simplify_candidate<'pat>(&mut self, candidate: &mut Candidate<'pat, 'tcx>) {
         // repeatedly simplify match pairs until fixed point is reached
         loop {
             let match_pairs = mem::take(&mut candidate.match_pairs);
diff --git a/src/librustc_mir/build/matches/test.rs b/src/librustc_mir_build/build/matches/test.rs
similarity index 99%
rename from src/librustc_mir/build/matches/test.rs
rename to src/librustc_mir_build/build/matches/test.rs
index afdb744a43a83..31fc0d121052a 100644
--- a/src/librustc_mir/build/matches/test.rs
+++ b/src/librustc_mir_build/build/matches/test.rs
@@ -24,7 +24,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Identifies what test is needed to decide if `match_pair` is applicable.
     ///
     /// It is a bug to call this with a simplifiable pattern.
-    pub fn test<'pat>(&mut self, match_pair: &MatchPair<'pat, 'tcx>) -> Test<'tcx> {
+    crate fn test<'pat>(&mut self, match_pair: &MatchPair<'pat, 'tcx>) -> Test<'tcx> {
         match *match_pair.pattern.kind {
             PatKind::Variant { ref adt_def, substs: _, variant_index: _, subpatterns: _ } => Test {
                 span: match_pair.pattern.span,
@@ -85,7 +85,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         }
     }
 
-    pub fn add_cases_to_switch<'pat>(
+    crate fn add_cases_to_switch<'pat>(
         &mut self,
         test_place: &Place<'tcx>,
         candidate: &Candidate<'pat, 'tcx>,
@@ -129,7 +129,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         }
     }
 
-    pub fn add_variants_to_switch<'pat>(
+    crate fn add_variants_to_switch<'pat>(
         &mut self,
         test_place: &Place<'tcx>,
         candidate: &Candidate<'pat, 'tcx>,
@@ -156,7 +156,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         }
     }
 
-    pub fn perform_test(
+    crate fn perform_test(
         &mut self,
         block: BasicBlock,
         place: &Place<'tcx>,
@@ -507,7 +507,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// that it *doesn't* apply. For now, we return false, indicate that the
     /// test does not apply to this candidate, but it might be we can get
     /// tighter match code if we do something a bit different.
-    pub fn sort_candidate<'pat>(
+    crate fn sort_candidate<'pat>(
         &mut self,
         test_place: &Place<'tcx>,
         test: &Test<'tcx>,
diff --git a/src/librustc_mir/build/matches/util.rs b/src/librustc_mir_build/build/matches/util.rs
similarity index 94%
rename from src/librustc_mir/build/matches/util.rs
rename to src/librustc_mir_build/build/matches/util.rs
index b6e643a65105b..def8d1b2fd8ba 100644
--- a/src/librustc_mir/build/matches/util.rs
+++ b/src/librustc_mir_build/build/matches/util.rs
@@ -8,7 +8,7 @@ use std::convert::TryInto;
 use std::u32;
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
-    pub fn field_match_pairs<'pat>(
+    crate fn field_match_pairs<'pat>(
         &mut self,
         place: Place<'tcx>,
         subpatterns: &'pat [FieldPat<'tcx>],
@@ -26,7 +26,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             .collect()
     }
 
-    pub fn prefix_slice_suffix<'pat>(
+    crate fn prefix_slice_suffix<'pat>(
         &mut self,
         match_pairs: &mut SmallVec<[MatchPair<'pat, 'tcx>; 1]>,
         place: &Place<'tcx>,
@@ -77,7 +77,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Creates a false edge to `imaginary_target` and a real edge to
     /// real_target. If `imaginary_target` is none, or is the same as the real
     /// target, a Goto is generated instead to simplify the generated MIR.
-    pub fn false_edges(
+    crate fn false_edges(
         &mut self,
         from_block: BasicBlock,
         real_target: BasicBlock,
@@ -98,7 +98,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 }
 
 impl<'pat, 'tcx> MatchPair<'pat, 'tcx> {
-    pub fn new(place: Place<'tcx>, pattern: &'pat Pat<'tcx>) -> MatchPair<'pat, 'tcx> {
+    crate fn new(place: Place<'tcx>, pattern: &'pat Pat<'tcx>) -> MatchPair<'pat, 'tcx> {
         MatchPair { place, pattern }
     }
 }
diff --git a/src/librustc_mir/build/misc.rs b/src/librustc_mir_build/build/misc.rs
similarity index 81%
rename from src/librustc_mir/build/misc.rs
rename to src/librustc_mir_build/build/misc.rs
index 7c358fef7d102..3d5145b6960f5 100644
--- a/src/librustc_mir/build/misc.rs
+++ b/src/librustc_mir_build/build/misc.rs
@@ -14,7 +14,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     ///
     /// N.B., **No cleanup is scheduled for this temporary.** You should
     /// call `schedule_drop` once the temporary is initialized.
-    pub fn temp(&mut self, ty: Ty<'tcx>, span: Span) -> Place<'tcx> {
+    crate fn temp(&mut self, ty: Ty<'tcx>, span: Span) -> Place<'tcx> {
         let temp = self.local_decls.push(LocalDecl::new_temp(ty, span));
         let place = Place::from(temp);
         debug!("temp: created temp {:?} with type {:?}", place, self.local_decls[temp].ty);
@@ -23,24 +23,28 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
     /// Convenience function for creating a literal operand, one
     /// without any user type annotation.
-    pub fn literal_operand(&mut self, span: Span, literal: &'tcx ty::Const<'tcx>) -> Operand<'tcx> {
+    crate fn literal_operand(
+        &mut self,
+        span: Span,
+        literal: &'tcx ty::Const<'tcx>,
+    ) -> Operand<'tcx> {
         let constant = box Constant { span, user_ty: None, literal };
         Operand::Constant(constant)
     }
 
-    pub fn unit_rvalue(&mut self) -> Rvalue<'tcx> {
+    crate fn unit_rvalue(&mut self) -> Rvalue<'tcx> {
         Rvalue::Aggregate(box AggregateKind::Tuple, vec![])
     }
 
     // Returns a zero literal operand for the appropriate type, works for
     // bool, char and integers.
-    pub fn zero_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> {
+    crate fn zero_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> {
         let literal = ty::Const::from_bits(self.hir.tcx(), 0, ty::ParamEnv::empty().and(ty));
 
         self.literal_operand(span, literal)
     }
 
-    pub fn push_usize(
+    crate fn push_usize(
         &mut self,
         block: BasicBlock,
         source_info: SourceInfo,
@@ -61,7 +65,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         temp
     }
 
-    pub fn consume_by_copy_or_move(&self, place: Place<'tcx>) -> Operand<'tcx> {
+    crate fn consume_by_copy_or_move(&self, place: Place<'tcx>) -> Operand<'tcx> {
         let tcx = self.hir.tcx();
         let ty = place.ty(&self.local_decls, tcx).ty;
         if !self.hir.type_is_copy_modulo_regions(ty, DUMMY_SP) {
diff --git a/src/librustc_mir/build/mod.rs b/src/librustc_mir_build/build/mod.rs
similarity index 99%
rename from src/librustc_mir/build/mod.rs
rename to src/librustc_mir_build/build/mod.rs
index d6d22db9ff24b..44b8747abb65b 100644
--- a/src/librustc_mir/build/mod.rs
+++ b/src/librustc_mir_build/build/mod.rs
@@ -2,8 +2,6 @@ use crate::build;
 use crate::build::scope::DropKind;
 use crate::hair::cx::Cx;
 use crate::hair::{BindingMode, LintLevel, PatKind};
-use crate::transform::MirSource;
-use crate::util as mir_util;
 use rustc::middle::lang_items;
 use rustc::middle::region;
 use rustc::mir::*;
@@ -22,8 +20,12 @@ use syntax::attr::{self, UnwindAttr};
 
 use super::lints;
 
+crate fn mir_built(tcx: TyCtxt<'_>, def_id: DefId) -> &ty::steal::Steal<BodyAndCache<'_>> {
+    tcx.alloc_steal_mir(mir_build(tcx, def_id))
+}
+
 /// Construct the MIR for a given `DefId`.
-pub fn mir_build(tcx: TyCtxt<'_>, def_id: DefId) -> BodyAndCache<'_> {
+fn mir_build(tcx: TyCtxt<'_>, def_id: DefId) -> BodyAndCache<'_> {
     let id = tcx.hir().as_local_hir_id(def_id).unwrap();
 
     // Figure out what primary body this item has.
@@ -172,8 +174,6 @@ pub fn mir_build(tcx: TyCtxt<'_>, def_id: DefId) -> BodyAndCache<'_> {
             build::construct_const(cx, body_id, return_ty, return_ty_span)
         };
 
-        mir_util::dump_mir(tcx, None, "mir_map", &0, MirSource::item(def_id), &body, |_, _| Ok(()));
-
         lints::check(tcx, &body, def_id);
 
         let mut body = BodyAndCache::new(body);
@@ -202,7 +202,7 @@ fn liberated_closure_env_ty(
 }
 
 #[derive(Debug, PartialEq, Eq)]
-pub enum BlockFrame {
+enum BlockFrame {
     /// Evaluation is currently within a statement.
     ///
     /// Examples include:
@@ -461,7 +461,7 @@ struct CFG<'tcx> {
 }
 
 rustc_index::newtype_index! {
-    pub struct ScopeId { .. }
+    struct ScopeId { .. }
 }
 
 ///////////////////////////////////////////////////////////////////////////
diff --git a/src/librustc_mir/build/scope.rs b/src/librustc_mir_build/build/scope.rs
similarity index 98%
rename from src/librustc_mir/build/scope.rs
rename to src/librustc_mir_build/build/scope.rs
index 0aa9773b39a3a..d994b870853c1 100644
--- a/src/librustc_mir/build/scope.rs
+++ b/src/librustc_mir_build/build/scope.rs
@@ -123,7 +123,7 @@ struct Scope {
 }
 
 #[derive(Debug, Default)]
-pub struct Scopes<'tcx> {
+crate struct Scopes<'tcx> {
     scopes: Vec<Scope>,
     /// The current set of breakable scopes. See module comment for more details.
     breakable_scopes: Vec<BreakableScope<'tcx>>,
@@ -183,7 +183,7 @@ struct BreakableScope<'tcx> {
 
 /// The target of an expression that breaks out of a scope
 #[derive(Clone, Copy, Debug)]
-pub enum BreakableTarget {
+crate enum BreakableTarget {
     Continue(region::Scope),
     Break(region::Scope),
     Return,
@@ -371,7 +371,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     // ==========================
     //  Start a breakable scope, which tracks where `continue`, `break` and
     //  `return` should branch to.
-    pub fn in_breakable_scope<F, R>(
+    crate fn in_breakable_scope<F, R>(
         &mut self,
         loop_block: Option<BasicBlock>,
         break_block: BasicBlock,
@@ -395,7 +395,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         res
     }
 
-    pub fn in_opt_scope<F, R>(
+    crate fn in_opt_scope<F, R>(
         &mut self,
         opt_scope: Option<(region::Scope, SourceInfo)>,
         f: F,
@@ -418,7 +418,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
     /// Convenience wrapper that pushes a scope and then executes `f`
     /// to build its contents, popping the scope afterwards.
-    pub fn in_scope<F, R>(
+    crate fn in_scope<F, R>(
         &mut self,
         region_scope: (region::Scope, SourceInfo),
         lint_level: LintLevel,
@@ -463,14 +463,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// scope and call `pop_scope` afterwards. Note that these two
     /// calls must be paired; using `in_scope` as a convenience
     /// wrapper maybe preferable.
-    pub fn push_scope(&mut self, region_scope: (region::Scope, SourceInfo)) {
+    crate fn push_scope(&mut self, region_scope: (region::Scope, SourceInfo)) {
         self.scopes.push_scope(region_scope, self.source_scope);
     }
 
     /// Pops a scope, which should have region scope `region_scope`,
     /// adding any drops onto the end of `block` that are needed.
     /// This must match 1-to-1 with `push_scope`.
-    pub fn pop_scope(
+    crate fn pop_scope(
         &mut self,
         region_scope: (region::Scope, SourceInfo),
         mut block: BasicBlock,
@@ -500,7 +500,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         block.unit()
     }
 
-    pub fn break_scope(
+    crate fn break_scope(
         &mut self,
         mut block: BasicBlock,
         value: Option<ExprRef<'tcx>>,
@@ -535,7 +535,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Branch out of `block` to `target`, exiting all scopes up to
     /// and including `region_scope`. This will insert whatever drops are
     /// needed. See module comment for details.
-    pub fn exit_scope(
+    crate fn exit_scope(
         &mut self,
         span: Span,
         region_scope: region::Scope,
@@ -604,7 +604,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     ///
     /// This path terminates in GeneratorDrop. Returns the start of the path.
     /// None indicates there’s no cleanup to do at this point.
-    pub fn generator_drop_cleanup(&mut self) -> Option<BasicBlock> {
+    crate fn generator_drop_cleanup(&mut self) -> Option<BasicBlock> {
         // Fill in the cache for unwinds
         self.diverge_cleanup_gen(true);
 
@@ -656,7 +656,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     }
 
     /// Creates a new source scope, nested in the current one.
-    pub fn new_source_scope(
+    crate fn new_source_scope(
         &mut self,
         span: Span,
         lint_level: LintLevel,
@@ -689,7 +689,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     }
 
     /// Given a span and the current source scope, make a SourceInfo.
-    pub fn source_info(&self, span: Span) -> SourceInfo {
+    crate fn source_info(&self, span: Span) -> SourceInfo {
         SourceInfo { span, scope: self.source_scope }
     }
 
@@ -717,7 +717,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     ///
     /// When building statics/constants, returns `None` since
     /// intermediate values do not have to be dropped in that case.
-    pub fn local_scope(&self) -> Option<region::Scope> {
+    crate fn local_scope(&self) -> Option<region::Scope> {
         match self.hir.body_owner_kind {
             hir::BodyOwnerKind::Const | hir::BodyOwnerKind::Static(_) =>
             // No need to free storage in this context.
@@ -729,7 +729,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     }
 
     // Schedule an abort block - this is used for some ABIs that cannot unwind
-    pub fn schedule_abort(&mut self) -> BasicBlock {
+    crate fn schedule_abort(&mut self) -> BasicBlock {
         let source_info = self.scopes.source_info(self.scopes.len(), self.fn_span);
         let abortblk = self.cfg.start_new_cleanup_block();
         self.cfg.terminate(abortblk, source_info, TerminatorKind::Abort);
@@ -739,7 +739,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
     // Scheduling drops
     // ================
-    pub fn schedule_drop_storage_and_value(
+    crate fn schedule_drop_storage_and_value(
         &mut self,
         span: Span,
         region_scope: region::Scope,
@@ -754,7 +754,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     ///
     /// When called with `DropKind::Storage`, `place` should be a local
     /// with an index higher than the current `self.arg_count`.
-    pub fn schedule_drop(
+    crate fn schedule_drop(
         &mut self,
         span: Span,
         region_scope: region::Scope,
@@ -884,7 +884,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// spurious borrow-check errors -- the problem, ironically, is
     /// not the `DROP(_X)` itself, but the (spurious) unwind pathways
     /// that it creates. See #64391 for an example.
-    pub fn record_operands_moved(&mut self, operands: &[Operand<'tcx>]) {
+    crate fn record_operands_moved(&mut self, operands: &[Operand<'tcx>]) {
         let scope = match self.local_scope() {
             None => {
                 // if there is no local scope, operands won't be dropped anyway
@@ -921,7 +921,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     ///
     /// This is a special case because the temporary for the condition needs to
     /// be dropped on both the true and the false arm.
-    pub fn test_bool(
+    crate fn test_bool(
         &mut self,
         mut block: BasicBlock,
         condition: Expr<'tcx>,
@@ -978,7 +978,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     ///
     /// This path terminates in Resume. Returns the start of the path.
     /// See module comment for more details.
-    pub fn diverge_cleanup(&mut self) -> BasicBlock {
+    crate fn diverge_cleanup(&mut self) -> BasicBlock {
         self.diverge_cleanup_gen(false)
     }
 
@@ -1033,7 +1033,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     }
 
     /// Utility function for *non*-scope code to build their own drops
-    pub fn build_drop_and_replace(
+    crate fn build_drop_and_replace(
         &mut self,
         block: BasicBlock,
         span: Span,
@@ -1059,7 +1059,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Creates an Assert terminator and return the success block.
     /// If the boolean condition operand is not the expected value,
     /// a runtime panic will be caused with the given message.
-    pub fn assert(
+    crate fn assert(
         &mut self,
         block: BasicBlock,
         cond: Operand<'tcx>,
@@ -1293,7 +1293,7 @@ fn build_diverge_scope<'tcx>(
     target
 }
 
-fn push_storage_deads(
+fn push_storage_deads<'tcx>(
     cfg: &mut CFG<'tcx>,
     target: BasicBlock,
     storage_deads: &mut Vec<Statement<'tcx>>,
diff --git a/src/librustc_mir/hair/constant.rs b/src/librustc_mir_build/hair/constant.rs
similarity index 100%
rename from src/librustc_mir/hair/constant.rs
rename to src/librustc_mir_build/hair/constant.rs
diff --git a/src/librustc_mir/hair/cx/block.rs b/src/librustc_mir_build/hair/cx/block.rs
similarity index 99%
rename from src/librustc_mir/hair/cx/block.rs
rename to src/librustc_mir_build/hair/cx/block.rs
index 674c1489b9b68..a883b84f8fe2f 100644
--- a/src/librustc_mir/hair/cx/block.rs
+++ b/src/librustc_mir_build/hair/cx/block.rs
@@ -101,7 +101,7 @@ fn mirror_stmts<'a, 'tcx>(
     return result;
 }
 
-pub fn to_expr_ref<'a, 'tcx>(
+crate fn to_expr_ref<'a, 'tcx>(
     cx: &mut Cx<'a, 'tcx>,
     block: &'tcx hir::Block<'tcx>,
 ) -> ExprRef<'tcx> {
diff --git a/src/librustc_mir/hair/cx/expr.rs b/src/librustc_mir_build/hair/cx/expr.rs
similarity index 99%
rename from src/librustc_mir/hair/cx/expr.rs
rename to src/librustc_mir_build/hair/cx/expr.rs
index 8fd8143ee374f..14db8125a70c9 100644
--- a/src/librustc_mir/hair/cx/expr.rs
+++ b/src/librustc_mir_build/hair/cx/expr.rs
@@ -574,8 +574,8 @@ fn make_mirror_unadjusted<'a, 'tcx>(
     Expr { temp_lifetime, ty: expr_ty, span: expr.span, kind }
 }
 
-fn user_substs_applied_to_res(
-    cx: &mut Cx<'a, 'tcx>,
+fn user_substs_applied_to_res<'tcx>(
+    cx: &mut Cx<'_, 'tcx>,
     hir_id: hir::HirId,
     res: Res,
 ) -> Option<ty::CanonicalUserType<'tcx>> {
@@ -772,7 +772,7 @@ fn convert_path_expr<'a, 'tcx>(
     }
 }
 
-fn convert_var(
+fn convert_var<'tcx>(
     cx: &mut Cx<'_, 'tcx>,
     expr: &'tcx hir::Expr<'tcx>,
     var_hir_id: hir::HirId,
diff --git a/src/librustc_mir/hair/cx/mod.rs b/src/librustc_mir_build/hair/cx/mod.rs
similarity index 77%
rename from src/librustc_mir/hair/cx/mod.rs
rename to src/librustc_mir_build/hair/cx/mod.rs
index 2e5ab3343508c..5fc9a1ecad555 100644
--- a/src/librustc_mir/hair/cx/mod.rs
+++ b/src/librustc_mir_build/hair/cx/mod.rs
@@ -21,18 +21,18 @@ use syntax::ast;
 use syntax::attr;
 
 #[derive(Clone)]
-pub struct Cx<'a, 'tcx> {
+crate struct Cx<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     infcx: &'a InferCtxt<'a, 'tcx>,
 
-    pub root_lint_level: hir::HirId,
-    pub param_env: ty::ParamEnv<'tcx>,
+    crate root_lint_level: hir::HirId,
+    crate param_env: ty::ParamEnv<'tcx>,
 
     /// Identity `InternalSubsts` for use with const-evaluation.
-    pub identity_substs: &'tcx InternalSubsts<'tcx>,
+    crate identity_substs: &'tcx InternalSubsts<'tcx>,
 
-    pub region_scope_tree: &'tcx region::ScopeTree,
-    pub tables: &'a ty::TypeckTables<'tcx>,
+    crate region_scope_tree: &'tcx region::ScopeTree,
+    crate tables: &'a ty::TypeckTables<'tcx>,
 
     /// This is `Constness::Const` if we are compiling a `static`,
     /// `const`, or the body of a `const fn`.
@@ -42,7 +42,7 @@ pub struct Cx<'a, 'tcx> {
     body_owner: DefId,
 
     /// What kind of body is being compiled.
-    pub body_owner_kind: hir::BodyOwnerKind,
+    crate body_owner_kind: hir::BodyOwnerKind,
 
     /// Whether this constant/function needs overflow checks.
     check_overflow: bool,
@@ -52,7 +52,7 @@ pub struct Cx<'a, 'tcx> {
 }
 
 impl<'a, 'tcx> Cx<'a, 'tcx> {
-    pub fn new(infcx: &'a InferCtxt<'a, 'tcx>, src_id: hir::HirId) -> Cx<'a, 'tcx> {
+    crate fn new(infcx: &'a InferCtxt<'a, 'tcx>, src_id: hir::HirId) -> Cx<'a, 'tcx> {
         let tcx = infcx.tcx;
         let src_def_id = tcx.hir().local_def_id(src_id);
         let tables = tcx.typeck_tables_of(src_def_id);
@@ -92,42 +92,42 @@ impl<'a, 'tcx> Cx<'a, 'tcx> {
         }
     }
 
-    pub fn control_flow_destroyed(self) -> Vec<(Span, String)> {
+    crate fn control_flow_destroyed(self) -> Vec<(Span, String)> {
         self.control_flow_destroyed
     }
 }
 
 impl<'a, 'tcx> Cx<'a, 'tcx> {
     /// Normalizes `ast` into the appropriate "mirror" type.
-    pub fn mirror<M: Mirror<'tcx>>(&mut self, ast: M) -> M::Output {
+    crate fn mirror<M: Mirror<'tcx>>(&mut self, ast: M) -> M::Output {
         ast.make_mirror(self)
     }
 
-    pub fn usize_ty(&mut self) -> Ty<'tcx> {
+    crate fn usize_ty(&mut self) -> Ty<'tcx> {
         self.tcx.types.usize
     }
 
-    pub fn usize_literal(&mut self, value: u64) -> &'tcx ty::Const<'tcx> {
+    crate fn usize_literal(&mut self, value: u64) -> &'tcx ty::Const<'tcx> {
         ty::Const::from_usize(self.tcx, value)
     }
 
-    pub fn bool_ty(&mut self) -> Ty<'tcx> {
+    crate fn bool_ty(&mut self) -> Ty<'tcx> {
         self.tcx.types.bool
     }
 
-    pub fn unit_ty(&mut self) -> Ty<'tcx> {
+    crate fn unit_ty(&mut self) -> Ty<'tcx> {
         self.tcx.mk_unit()
     }
 
-    pub fn true_literal(&mut self) -> &'tcx ty::Const<'tcx> {
+    crate fn true_literal(&mut self) -> &'tcx ty::Const<'tcx> {
         ty::Const::from_bool(self.tcx, true)
     }
 
-    pub fn false_literal(&mut self) -> &'tcx ty::Const<'tcx> {
+    crate fn false_literal(&mut self) -> &'tcx ty::Const<'tcx> {
         ty::Const::from_bool(self.tcx, false)
     }
 
-    pub fn const_eval_literal(
+    crate fn const_eval_literal(
         &mut self,
         lit: &'tcx ast::LitKind,
         ty: Ty<'tcx>,
@@ -151,15 +151,15 @@ impl<'a, 'tcx> Cx<'a, 'tcx> {
         }
     }
 
-    pub fn pattern_from_hir(&mut self, p: &hir::Pat<'_>) -> Pat<'tcx> {
+    crate fn pattern_from_hir(&mut self, p: &hir::Pat<'_>) -> Pat<'tcx> {
         let p = match self.tcx.hir().get(p.hir_id) {
             Node::Pat(p) | Node::Binding(p) => p,
             node => bug!("pattern became {:?}", node),
         };
-        Pat::from_hir(self.tcx, self.param_env.and(self.identity_substs), self.tables(), p)
+        Pat::from_hir(self.tcx, self.param_env, self.tables(), p)
     }
 
-    pub fn trait_method(
+    crate fn trait_method(
         &mut self,
         trait_def_id: DefId,
         method_name: Symbol,
@@ -168,6 +168,8 @@ impl<'a, 'tcx> Cx<'a, 'tcx> {
     ) -> &'tcx ty::Const<'tcx> {
         let substs = self.tcx.mk_substs_trait(self_ty, params);
         for item in self.tcx.associated_items(trait_def_id) {
+            // The unhygienic comparison here is acceptable because this is only
+            // used on known traits.
             if item.kind == ty::AssocKind::Method && item.ident.name == method_name {
                 let method_ty = self.tcx.type_of(item.def_id);
                 let method_ty = method_ty.subst(self.tcx, substs);
@@ -178,32 +180,32 @@ impl<'a, 'tcx> Cx<'a, 'tcx> {
         bug!("found no method `{}` in `{:?}`", method_name, trait_def_id);
     }
 
-    pub fn all_fields(&mut self, adt_def: &ty::AdtDef, variant_index: VariantIdx) -> Vec<Field> {
+    crate fn all_fields(&mut self, adt_def: &ty::AdtDef, variant_index: VariantIdx) -> Vec<Field> {
         (0..adt_def.variants[variant_index].fields.len()).map(Field::new).collect()
     }
 
-    pub fn needs_drop(&mut self, ty: Ty<'tcx>) -> bool {
+    crate fn needs_drop(&mut self, ty: Ty<'tcx>) -> bool {
         ty.needs_drop(self.tcx, self.param_env)
     }
 
-    pub fn tcx(&self) -> TyCtxt<'tcx> {
+    crate fn tcx(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
-    pub fn tables(&self) -> &'a ty::TypeckTables<'tcx> {
+    crate fn tables(&self) -> &'a ty::TypeckTables<'tcx> {
         self.tables
     }
 
-    pub fn check_overflow(&self) -> bool {
+    crate fn check_overflow(&self) -> bool {
         self.check_overflow
     }
 
-    pub fn type_is_copy_modulo_regions(&self, ty: Ty<'tcx>, span: Span) -> bool {
+    crate fn type_is_copy_modulo_regions(&self, ty: Ty<'tcx>, span: Span) -> bool {
         self.infcx.type_is_copy_modulo_regions(self.param_env, ty, span)
     }
 }
 
-impl UserAnnotatedTyHelpers<'tcx> for Cx<'_, 'tcx> {
+impl<'tcx> UserAnnotatedTyHelpers<'tcx> for Cx<'_, 'tcx> {
     fn tcx(&self) -> TyCtxt<'tcx> {
         self.tcx()
     }
diff --git a/src/librustc_mir/hair/cx/to_ref.rs b/src/librustc_mir_build/hair/cx/to_ref.rs
similarity index 98%
rename from src/librustc_mir/hair/cx/to_ref.rs
rename to src/librustc_mir_build/hair/cx/to_ref.rs
index d6859e356eef3..6cf8122e200db 100644
--- a/src/librustc_mir/hair/cx/to_ref.rs
+++ b/src/librustc_mir_build/hair/cx/to_ref.rs
@@ -2,7 +2,7 @@ use crate::hair::*;
 
 use rustc_hir as hir;
 
-pub trait ToRef {
+crate trait ToRef {
     type Output;
     fn to_ref(self) -> Self::Output;
 }
diff --git a/src/librustc_mir/hair/mod.rs b/src/librustc_mir_build/hair/mod.rs
similarity index 86%
rename from src/librustc_mir/hair/mod.rs
rename to src/librustc_mir_build/hair/mod.rs
index cde91cc36cab2..3257f282dc1cb 100644
--- a/src/librustc_mir/hair/mod.rs
+++ b/src/librustc_mir_build/hair/mod.rs
@@ -17,33 +17,33 @@ use rustc_hir::def_id::DefId;
 use rustc_span::Span;
 
 mod constant;
-pub mod cx;
+crate mod cx;
 
-pub mod pattern;
-pub(crate) use self::pattern::PatTyProj;
-pub use self::pattern::{BindingMode, FieldPat, Pat, PatKind, PatRange};
+crate mod pattern;
+crate use self::pattern::PatTyProj;
+crate use self::pattern::{BindingMode, FieldPat, Pat, PatKind, PatRange};
 
 mod util;
 
 #[derive(Copy, Clone, Debug)]
-pub enum LintLevel {
+crate enum LintLevel {
     Inherited,
     Explicit(hir::HirId),
 }
 
 #[derive(Clone, Debug)]
-pub struct Block<'tcx> {
-    pub targeted_by_break: bool,
-    pub region_scope: region::Scope,
-    pub opt_destruction_scope: Option<region::Scope>,
-    pub span: Span,
-    pub stmts: Vec<StmtRef<'tcx>>,
-    pub expr: Option<ExprRef<'tcx>>,
-    pub safety_mode: BlockSafety,
+crate struct Block<'tcx> {
+    crate targeted_by_break: bool,
+    crate region_scope: region::Scope,
+    crate opt_destruction_scope: Option<region::Scope>,
+    crate span: Span,
+    crate stmts: Vec<StmtRef<'tcx>>,
+    crate expr: Option<ExprRef<'tcx>>,
+    crate safety_mode: BlockSafety,
 }
 
 #[derive(Copy, Clone, Debug)]
-pub enum BlockSafety {
+crate enum BlockSafety {
     Safe,
     ExplicitUnsafe(hir::HirId),
     PushUnsafe,
@@ -51,18 +51,18 @@ pub enum BlockSafety {
 }
 
 #[derive(Clone, Debug)]
-pub enum StmtRef<'tcx> {
+crate enum StmtRef<'tcx> {
     Mirror(Box<Stmt<'tcx>>),
 }
 
 #[derive(Clone, Debug)]
-pub struct Stmt<'tcx> {
-    pub kind: StmtKind<'tcx>,
-    pub opt_destruction_scope: Option<region::Scope>,
+crate struct Stmt<'tcx> {
+    crate kind: StmtKind<'tcx>,
+    crate opt_destruction_scope: Option<region::Scope>,
 }
 
 #[derive(Clone, Debug)]
-pub enum StmtKind<'tcx> {
+crate enum StmtKind<'tcx> {
     Expr {
         /// scope for this statement; may be used as lifetime of temporaries
         scope: region::Scope,
@@ -112,23 +112,23 @@ rustc_data_structures::static_assert_size!(Expr<'_>, 168);
 /// example, method calls and overloaded operators are absent: they are
 /// expected to be converted into `Expr::Call` instances.
 #[derive(Clone, Debug)]
-pub struct Expr<'tcx> {
+crate struct Expr<'tcx> {
     /// type of this expression
-    pub ty: Ty<'tcx>,
+    crate ty: Ty<'tcx>,
 
     /// lifetime of this expression if it should be spilled into a
     /// temporary; should be None only if in a constant context
-    pub temp_lifetime: Option<region::Scope>,
+    crate temp_lifetime: Option<region::Scope>,
 
     /// span of the expression in the source
-    pub span: Span,
+    crate span: Span,
 
     /// kind of expression
-    pub kind: ExprKind<'tcx>,
+    crate kind: ExprKind<'tcx>,
 }
 
 #[derive(Clone, Debug)]
-pub enum ExprKind<'tcx> {
+crate enum ExprKind<'tcx> {
     Scope {
         region_scope: region::Scope,
         lint_level: LintLevel,
@@ -288,37 +288,37 @@ pub enum ExprKind<'tcx> {
 }
 
 #[derive(Clone, Debug)]
-pub enum ExprRef<'tcx> {
+crate enum ExprRef<'tcx> {
     Hair(&'tcx hir::Expr<'tcx>),
     Mirror(Box<Expr<'tcx>>),
 }
 
 #[derive(Clone, Debug)]
-pub struct FieldExprRef<'tcx> {
-    pub name: Field,
-    pub expr: ExprRef<'tcx>,
+crate struct FieldExprRef<'tcx> {
+    crate name: Field,
+    crate expr: ExprRef<'tcx>,
 }
 
 #[derive(Clone, Debug)]
-pub struct FruInfo<'tcx> {
-    pub base: ExprRef<'tcx>,
-    pub field_types: Vec<Ty<'tcx>>,
+crate struct FruInfo<'tcx> {
+    crate base: ExprRef<'tcx>,
+    crate field_types: Vec<Ty<'tcx>>,
 }
 
 #[derive(Clone, Debug)]
-pub struct Arm<'tcx> {
-    pub pattern: Pat<'tcx>,
-    pub guard: Option<Guard<'tcx>>,
-    pub body: ExprRef<'tcx>,
-    pub lint_level: LintLevel,
-    pub scope: region::Scope,
-    pub span: Span,
+crate struct Arm<'tcx> {
+    crate pattern: Pat<'tcx>,
+    crate guard: Option<Guard<'tcx>>,
+    crate body: ExprRef<'tcx>,
+    crate lint_level: LintLevel,
+    crate scope: region::Scope,
+    crate span: Span,
 }
 
-impl Arm<'tcx> {
+impl<'tcx> Arm<'tcx> {
     // HACK(or_patterns; Centril | dlrobertson): Remove this and
     // correctly handle each case in which this method is used.
-    pub fn top_pats_hack(&self) -> &[Pat<'tcx>] {
+    crate fn top_pats_hack(&self) -> &[Pat<'tcx>] {
         match &*self.pattern.kind {
             PatKind::Or { pats } => pats,
             _ => std::slice::from_ref(&self.pattern),
@@ -327,18 +327,18 @@ impl Arm<'tcx> {
 }
 
 #[derive(Clone, Debug)]
-pub enum Guard<'tcx> {
+crate enum Guard<'tcx> {
     If(ExprRef<'tcx>),
 }
 
 #[derive(Copy, Clone, Debug)]
-pub enum LogicalOp {
+crate enum LogicalOp {
     And,
     Or,
 }
 
 impl<'tcx> ExprRef<'tcx> {
-    pub fn span(&self) -> Span {
+    crate fn span(&self) -> Span {
         match self {
             ExprRef::Hair(expr) => expr.span,
             ExprRef::Mirror(expr) => expr.span,
@@ -361,7 +361,7 @@ impl<'tcx> ExprRef<'tcx> {
 /// mirrored. This allows a single AST node from the compiler to
 /// expand into one or more Hair nodes, which lets the Hair nodes be
 /// simpler.
-pub trait Mirror<'tcx> {
+crate trait Mirror<'tcx> {
     type Output;
 
     fn make_mirror(self, cx: &mut Cx<'_, 'tcx>) -> Self::Output;
@@ -378,7 +378,7 @@ impl<'tcx> Mirror<'tcx> for Expr<'tcx> {
 impl<'tcx> Mirror<'tcx> for ExprRef<'tcx> {
     type Output = Expr<'tcx>;
 
-    fn make_mirror(self, hir: &mut Cx<'a, 'tcx>) -> Expr<'tcx> {
+    fn make_mirror(self, hir: &mut Cx<'_, 'tcx>) -> Expr<'tcx> {
         match self {
             ExprRef::Hair(h) => h.make_mirror(hir),
             ExprRef::Mirror(m) => *m,
diff --git a/src/librustc_mir/hair/pattern/_match.rs b/src/librustc_mir_build/hair/pattern/_match.rs
similarity index 98%
rename from src/librustc_mir/hair/pattern/_match.rs
rename to src/librustc_mir_build/hair/pattern/_match.rs
index 03120e8009f0a..695e5ce97dde8 100644
--- a/src/librustc_mir/hair/pattern/_match.rs
+++ b/src/librustc_mir_build/hair/pattern/_match.rs
@@ -229,7 +229,6 @@ use self::SliceKind::*;
 use self::Usefulness::*;
 use self::WitnessPreference::*;
 
-use rustc_data_structures::fx::FxHashMap;
 use rustc_index::vec::Idx;
 
 use super::{compare_const_vals, PatternFoldable, PatternFolder};
@@ -260,7 +259,7 @@ use std::iter::{FromIterator, IntoIterator};
 use std::ops::RangeInclusive;
 use std::u128;
 
-pub fn expand_pattern<'a, 'tcx>(cx: &MatchCheckCtxt<'a, 'tcx>, pat: Pat<'tcx>) -> Pat<'tcx> {
+crate fn expand_pattern<'a, 'tcx>(cx: &MatchCheckCtxt<'a, 'tcx>, pat: Pat<'tcx>) -> Pat<'tcx> {
     LiteralExpander { tcx: cx.tcx, param_env: cx.param_env }.fold_pattern(&pat)
 }
 
@@ -269,7 +268,7 @@ struct LiteralExpander<'tcx> {
     param_env: ty::ParamEnv<'tcx>,
 }
 
-impl LiteralExpander<'tcx> {
+impl<'tcx> LiteralExpander<'tcx> {
     /// Derefs `val` and potentially unsizes the value if `crty` is an array and `rty` a slice.
     ///
     /// `crty` and `rty` can differ because you can use array constants in the presence of slice
@@ -323,7 +322,7 @@ impl LiteralExpander<'tcx> {
     }
 }
 
-impl PatternFolder<'tcx> for LiteralExpander<'tcx> {
+impl<'tcx> PatternFolder<'tcx> for LiteralExpander<'tcx> {
     fn fold_pattern(&mut self, pat: &Pat<'tcx>) -> Pat<'tcx> {
         debug!("fold_pattern {:?} {:?} {:?}", pat, pat.ty.kind, pat.kind);
         match (&pat.ty.kind, &*pat.kind) {
@@ -381,10 +380,10 @@ impl<'tcx> Pat<'tcx> {
 /// A row of a matrix. Rows of len 1 are very common, which is why `SmallVec[_; 2]`
 /// works well.
 #[derive(Debug, Clone)]
-pub struct PatStack<'p, 'tcx>(SmallVec<[&'p Pat<'tcx>; 2]>);
+crate struct PatStack<'p, 'tcx>(SmallVec<[&'p Pat<'tcx>; 2]>);
 
 impl<'p, 'tcx> PatStack<'p, 'tcx> {
-    pub fn from_pattern(pat: &'p Pat<'tcx>) -> Self {
+    crate fn from_pattern(pat: &'p Pat<'tcx>) -> Self {
         PatStack(smallvec![pat])
     }
 
@@ -472,15 +471,15 @@ impl<'p, 'tcx> FromIterator<&'p Pat<'tcx>> for PatStack<'p, 'tcx> {
 
 /// A 2D matrix.
 #[derive(Clone)]
-pub struct Matrix<'p, 'tcx>(Vec<PatStack<'p, 'tcx>>);
+crate struct Matrix<'p, 'tcx>(Vec<PatStack<'p, 'tcx>>);
 
 impl<'p, 'tcx> Matrix<'p, 'tcx> {
-    pub fn empty() -> Self {
+    crate fn empty() -> Self {
         Matrix(vec![])
     }
 
     /// Pushes a new row to the matrix. If the row starts with an or-pattern, this expands it.
-    pub fn push(&mut self, row: PatStack<'p, 'tcx>) {
+    crate fn push(&mut self, row: PatStack<'p, 'tcx>) {
         if let Some(rows) = row.expand_or_pat() {
             self.0.extend(rows);
         } else {
@@ -569,22 +568,21 @@ impl<'p, 'tcx> FromIterator<PatStack<'p, 'tcx>> for Matrix<'p, 'tcx> {
     }
 }
 
-pub struct MatchCheckCtxt<'a, 'tcx> {
-    pub tcx: TyCtxt<'tcx>,
+crate struct MatchCheckCtxt<'a, 'tcx> {
+    crate tcx: TyCtxt<'tcx>,
     /// The module in which the match occurs. This is necessary for
     /// checking inhabited-ness of types because whether a type is (visibly)
     /// inhabited can depend on whether it was defined in the current module or
     /// not. E.g., `struct Foo { _private: ! }` cannot be seen to be empty
     /// outside it's module and should not be matchable with an empty match
     /// statement.
-    pub module: DefId,
+    crate module: DefId,
     param_env: ty::ParamEnv<'tcx>,
-    pub pattern_arena: &'a TypedArena<Pat<'tcx>>,
-    pub byte_array_map: FxHashMap<*const Pat<'tcx>, Vec<&'a Pat<'tcx>>>,
+    crate pattern_arena: &'a TypedArena<Pat<'tcx>>,
 }
 
 impl<'a, 'tcx> MatchCheckCtxt<'a, 'tcx> {
-    pub fn create_and_enter<F, R>(
+    crate fn create_and_enter<F, R>(
         tcx: TyCtxt<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
         module: DefId,
@@ -595,13 +593,7 @@ impl<'a, 'tcx> MatchCheckCtxt<'a, 'tcx> {
     {
         let pattern_arena = TypedArena::default();
 
-        f(MatchCheckCtxt {
-            tcx,
-            param_env,
-            module,
-            pattern_arena: &pattern_arena,
-            byte_array_map: FxHashMap::default(),
-        })
+        f(MatchCheckCtxt { tcx, param_env, module, pattern_arena: &pattern_arena })
     }
 
     fn is_uninhabited(&self, ty: Ty<'tcx>) -> bool {
@@ -613,7 +605,7 @@ impl<'a, 'tcx> MatchCheckCtxt<'a, 'tcx> {
     }
 
     // Returns whether the given type is an enum from another crate declared `#[non_exhaustive]`.
-    pub fn is_foreign_non_exhaustive_enum(&self, ty: Ty<'tcx>) -> bool {
+    crate fn is_foreign_non_exhaustive_enum(&self, ty: Ty<'tcx>) -> bool {
         match ty.kind {
             ty::Adt(def, ..) => {
                 def.is_enum() && def.is_variant_list_non_exhaustive() && !def.did.is_local()
@@ -773,13 +765,13 @@ impl<'tcx> Constructor<'tcx> {
         cx: &MatchCheckCtxt<'a, 'tcx>,
         adt: &'tcx ty::AdtDef,
     ) -> VariantIdx {
-        match self {
-            Variant(id) => adt.variant_index_with_id(*id),
+        match *self {
+            Variant(id) => adt.variant_index_with_id(id),
             Single => {
                 assert!(!adt.is_enum());
                 VariantIdx::new(0)
             }
-            ConstantValue(c) => crate::const_eval::const_variant_index(cx.tcx, cx.param_env, c),
+            ConstantValue(c) => cx.tcx.destructure_const(cx.param_env.and(c)).variant,
             _ => bug!("bad constructor {:?} for adt {:?}", self, adt),
         }
     }
@@ -1058,7 +1050,7 @@ impl<'tcx> Constructor<'tcx> {
 }
 
 #[derive(Clone, Debug)]
-pub enum Usefulness<'tcx, 'p> {
+crate enum Usefulness<'tcx, 'p> {
     /// Carries a list of unreachable subpatterns. Used only in the presence of or-patterns.
     Useful(Vec<&'p Pat<'tcx>>),
     /// Carries a list of witnesses of non-exhaustiveness.
@@ -1146,7 +1138,7 @@ impl<'tcx, 'p> Usefulness<'tcx, 'p> {
 }
 
 #[derive(Copy, Clone, Debug)]
-pub enum WitnessPreference {
+crate enum WitnessPreference {
     ConstructWitness,
     LeaveOutWitness,
 }
@@ -1190,10 +1182,10 @@ struct PatCtxt<'tcx> {
 ///
 /// The final `Pair(Some(_), true)` is then the resulting witness.
 #[derive(Clone, Debug)]
-pub struct Witness<'tcx>(Vec<Pat<'tcx>>);
+crate struct Witness<'tcx>(Vec<Pat<'tcx>>);
 
 impl<'tcx> Witness<'tcx> {
-    pub fn single_pattern(self) -> Pat<'tcx> {
+    crate fn single_pattern(self) -> Pat<'tcx> {
         assert_eq!(self.0.len(), 1);
         self.0.into_iter().next().unwrap()
     }
@@ -1358,9 +1350,9 @@ fn all_constructors<'a, 'tcx>(
 /// around the (offset) space: i.e., `range.lo <= range.hi`.
 #[derive(Clone, Debug)]
 struct IntRange<'tcx> {
-    pub range: RangeInclusive<u128>,
-    pub ty: Ty<'tcx>,
-    pub span: Span,
+    range: RangeInclusive<u128>,
+    ty: Ty<'tcx>,
+    span: Span,
 }
 
 impl<'tcx> IntRange<'tcx> {
@@ -1631,7 +1623,7 @@ impl<'tcx> fmt::Debug for MissingConstructors<'tcx> {
 /// relation to preceding patterns, it is not reachable) and exhaustiveness
 /// checking (if a wildcard pattern is useful in relation to a matrix, the
 /// matrix isn't exhaustive).
-pub fn is_useful<'p, 'tcx>(
+crate fn is_useful<'p, 'tcx>(
     cx: &mut MatchCheckCtxt<'p, 'tcx>,
     matrix: &Matrix<'p, 'tcx>,
     v: &PatStack<'p, 'tcx>,
@@ -2238,7 +2230,7 @@ fn split_grouped_constructors<'p, 'tcx>(
     split_ctors
 }
 
-fn lint_overlapping_patterns(
+fn lint_overlapping_patterns<'tcx>(
     tcx: TyCtxt<'tcx>,
     hir_id: Option<HirId>,
     ctor_range: IntRange<'tcx>,
diff --git a/src/librustc_mir/hair/pattern/check_match.rs b/src/librustc_mir_build/hair/pattern/check_match.rs
similarity index 97%
rename from src/librustc_mir/hair/pattern/check_match.rs
rename to src/librustc_mir_build/hair/pattern/check_match.rs
index ca7912b447e28..ef5ba76a8c8ef 100644
--- a/src/librustc_mir/hair/pattern/check_match.rs
+++ b/src/librustc_mir_build/hair/pattern/check_match.rs
@@ -7,7 +7,6 @@ use super::{PatCtxt, PatKind, PatternError};
 use rustc::hir::intravisit::{self, NestedVisitorMap, Visitor};
 use rustc::lint;
 use rustc::session::Session;
-use rustc::ty::subst::{InternalSubsts, SubstsRef};
 use rustc::ty::{self, Ty, TyCtxt};
 use rustc_error_codes::*;
 use rustc_errors::{Applicability, DiagnosticBuilder};
@@ -28,12 +27,8 @@ crate fn check_match(tcx: TyCtxt<'_>, def_id: DefId) {
         Some(id) => tcx.hir().body_owned_by(id),
     };
 
-    let mut visitor = MatchVisitor {
-        tcx,
-        tables: tcx.body_tables(body_id),
-        param_env: tcx.param_env(def_id),
-        identity_substs: InternalSubsts::identity_for_item(tcx, def_id),
-    };
+    let mut visitor =
+        MatchVisitor { tcx, tables: tcx.body_tables(body_id), param_env: tcx.param_env(def_id) };
     visitor.visit_body(tcx.hir().body(body_id));
 }
 
@@ -45,7 +40,6 @@ struct MatchVisitor<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     tables: &'a ty::TypeckTables<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
-    identity_substs: SubstsRef<'tcx>,
 }
 
 impl<'tcx> Visitor<'tcx> for MatchVisitor<'_, 'tcx> {
@@ -150,11 +144,7 @@ impl<'tcx> MatchVisitor<'_, 'tcx> {
             let inlined_arms: Vec<_> = arms
                 .iter()
                 .map(|arm| {
-                    let mut patcx = PatCtxt::new(
-                        self.tcx,
-                        self.param_env.and(self.identity_substs),
-                        self.tables,
-                    );
+                    let mut patcx = PatCtxt::new(self.tcx, self.param_env, self.tables);
                     patcx.include_lint_checks();
                     let pattern = patcx.lower_pattern(&arm.pat);
                     let pattern: &_ = cx.pattern_arena.alloc(expand_pattern(cx, pattern));
@@ -186,8 +176,7 @@ impl<'tcx> MatchVisitor<'_, 'tcx> {
     fn check_irrefutable(&self, pat: &'tcx Pat<'tcx>, origin: &str, sp: Option<Span>) {
         let module = self.tcx.hir().get_module_parent(pat.hir_id);
         MatchCheckCtxt::create_and_enter(self.tcx, self.param_env, module, |ref mut cx| {
-            let mut patcx =
-                PatCtxt::new(self.tcx, self.param_env.and(self.identity_substs), self.tables);
+            let mut patcx = PatCtxt::new(self.tcx, self.param_env, self.tables);
             patcx.include_lint_checks();
             let pattern = patcx.lower_pattern(pat);
             let pattern_ty = pattern.ty;
diff --git a/src/librustc_mir/hair/pattern/const_to_pat.rs b/src/librustc_mir_build/hair/pattern/const_to_pat.rs
similarity index 87%
rename from src/librustc_mir/hair/pattern/const_to_pat.rs
rename to src/librustc_mir_build/hair/pattern/const_to_pat.rs
index d4975df2e68cb..b2750940b83ea 100644
--- a/src/librustc_mir/hair/pattern/const_to_pat.rs
+++ b/src/librustc_mir_build/hair/pattern/const_to_pat.rs
@@ -1,5 +1,3 @@
-use crate::const_eval::const_variant_index;
-
 use rustc::infer::InferCtxt;
 use rustc::lint;
 use rustc::mir::Field;
@@ -165,18 +163,14 @@ impl<'a, 'tcx> ConstToPat<'a, 'tcx> {
         let tcx = self.tcx();
         let param_env = self.param_env;
 
-        let adt_subpattern = |i, variant_opt| {
-            let field = Field::new(i);
-            let val = crate::const_eval::const_field(tcx, param_env, variant_opt, field, cv);
-            self.recur(val)
-        };
-        let adt_subpatterns = |n, variant_opt| {
-            (0..n)
-                .map(|i| {
-                    let field = Field::new(i);
-                    FieldPat { field, pattern: adt_subpattern(i, variant_opt) }
+        let field_pats = |vals: &[&'tcx ty::Const<'tcx>]| {
+            vals.iter()
+                .enumerate()
+                .map(|(idx, val)| {
+                    let field = Field::new(idx);
+                    FieldPat { field, pattern: self.recur(val) }
                 })
-                .collect::<Vec<_>>()
+                .collect()
         };
 
         let kind = match cv.ty.kind {
@@ -233,21 +227,28 @@ impl<'a, 'tcx> ConstToPat<'a, 'tcx> {
                 PatKind::Wild
             }
             ty::Adt(adt_def, substs) if adt_def.is_enum() => {
-                let variant_index = const_variant_index(tcx, self.param_env, cv);
-                let subpatterns = adt_subpatterns(
-                    adt_def.variants[variant_index].fields.len(),
-                    Some(variant_index),
-                );
-                PatKind::Variant { adt_def, substs, variant_index, subpatterns }
+                let destructured = tcx.destructure_const(param_env.and(cv));
+                PatKind::Variant {
+                    adt_def,
+                    substs,
+                    variant_index: destructured.variant,
+                    subpatterns: field_pats(destructured.fields),
+                }
+            }
+            ty::Adt(_, _) => {
+                let destructured = tcx.destructure_const(param_env.and(cv));
+                PatKind::Leaf { subpatterns: field_pats(destructured.fields) }
             }
-            ty::Adt(adt_def, _) => {
-                let struct_var = adt_def.non_enum_variant();
-                PatKind::Leaf { subpatterns: adt_subpatterns(struct_var.fields.len(), None) }
+            ty::Tuple(_) => {
+                let destructured = tcx.destructure_const(param_env.and(cv));
+                PatKind::Leaf { subpatterns: field_pats(destructured.fields) }
             }
-            ty::Tuple(fields) => PatKind::Leaf { subpatterns: adt_subpatterns(fields.len(), None) },
-            ty::Array(_, n) => PatKind::Array {
-                prefix: (0..n.eval_usize(tcx, self.param_env))
-                    .map(|i| adt_subpattern(i as usize, None))
+            ty::Array(..) => PatKind::Array {
+                prefix: tcx
+                    .destructure_const(param_env.and(cv))
+                    .fields
+                    .iter()
+                    .map(|val| self.recur(val))
                     .collect(),
                 slice: None,
                 suffix: Vec::new(),
diff --git a/src/librustc_mir/hair/pattern/mod.rs b/src/librustc_mir_build/hair/pattern/mod.rs
similarity index 96%
rename from src/librustc_mir/hair/pattern/mod.rs
rename to src/librustc_mir_build/hair/pattern/mod.rs
index bac40a06c78d5..835a18603684d 100644
--- a/src/librustc_mir/hair/pattern/mod.rs
+++ b/src/librustc_mir_build/hair/pattern/mod.rs
@@ -30,7 +30,7 @@ use std::fmt;
 use rustc_error_codes::*;
 
 #[derive(Clone, Debug)]
-pub enum PatternError {
+crate enum PatternError {
     AssocConstInPattern(Span),
     StaticInPattern(Span),
     FloatBug,
@@ -38,22 +38,22 @@ pub enum PatternError {
 }
 
 #[derive(Copy, Clone, Debug)]
-pub enum BindingMode {
+crate enum BindingMode {
     ByValue,
     ByRef(BorrowKind),
 }
 
 #[derive(Clone, Debug)]
-pub struct FieldPat<'tcx> {
-    pub field: Field,
-    pub pattern: Pat<'tcx>,
+crate struct FieldPat<'tcx> {
+    crate field: Field,
+    crate pattern: Pat<'tcx>,
 }
 
 #[derive(Clone, Debug)]
-pub struct Pat<'tcx> {
-    pub ty: Ty<'tcx>,
-    pub span: Span,
-    pub kind: Box<PatKind<'tcx>>,
+crate struct Pat<'tcx> {
+    crate ty: Ty<'tcx>,
+    crate span: Span,
+    crate kind: Box<PatKind<'tcx>>,
 }
 
 impl<'tcx> Pat<'tcx> {
@@ -63,8 +63,8 @@ impl<'tcx> Pat<'tcx> {
 }
 
 #[derive(Copy, Clone, Debug, PartialEq)]
-pub struct PatTyProj<'tcx> {
-    pub user_ty: CanonicalUserType<'tcx>,
+crate struct PatTyProj<'tcx> {
+    crate user_ty: CanonicalUserType<'tcx>,
 }
 
 impl<'tcx> PatTyProj<'tcx> {
@@ -90,8 +90,8 @@ impl<'tcx> PatTyProj<'tcx> {
 }
 
 #[derive(Copy, Clone, Debug, PartialEq)]
-pub struct Ascription<'tcx> {
-    pub user_ty: PatTyProj<'tcx>,
+crate struct Ascription<'tcx> {
+    crate user_ty: PatTyProj<'tcx>,
     /// Variance to use when relating the type `user_ty` to the **type of the value being
     /// matched**. Typically, this is `Variance::Covariant`, since the value being matched must
     /// have a type that is some subtype of the ascribed type.
@@ -110,12 +110,12 @@ pub struct Ascription<'tcx> {
     /// requires that `&'static str <: T_x`, where `T_x` is the type of `x`. Really, we should
     /// probably be checking for a `PartialEq` impl instead, but this preserves the behavior
     /// of the old type-check for now. See #57280 for details.
-    pub variance: ty::Variance,
-    pub user_ty_span: Span,
+    crate variance: ty::Variance,
+    crate user_ty_span: Span,
 }
 
 #[derive(Clone, Debug)]
-pub enum PatKind<'tcx> {
+crate enum PatKind<'tcx> {
     Wild,
 
     AscribeUserType {
@@ -183,10 +183,10 @@ pub enum PatKind<'tcx> {
 }
 
 #[derive(Copy, Clone, Debug, PartialEq)]
-pub struct PatRange<'tcx> {
-    pub lo: &'tcx ty::Const<'tcx>,
-    pub hi: &'tcx ty::Const<'tcx>,
-    pub end: RangeEnd,
+crate struct PatRange<'tcx> {
+    crate lo: &'tcx ty::Const<'tcx>,
+    crate hi: &'tcx ty::Const<'tcx>,
+    crate end: RangeEnd,
 }
 
 impl<'tcx> fmt::Display for Pat<'tcx> {
@@ -341,23 +341,22 @@ impl<'tcx> fmt::Display for Pat<'tcx> {
     }
 }
 
-pub struct PatCtxt<'a, 'tcx> {
-    pub tcx: TyCtxt<'tcx>,
-    pub param_env: ty::ParamEnv<'tcx>,
-    pub tables: &'a ty::TypeckTables<'tcx>,
-    pub substs: SubstsRef<'tcx>,
-    pub errors: Vec<PatternError>,
+crate struct PatCtxt<'a, 'tcx> {
+    crate tcx: TyCtxt<'tcx>,
+    crate param_env: ty::ParamEnv<'tcx>,
+    crate tables: &'a ty::TypeckTables<'tcx>,
+    crate errors: Vec<PatternError>,
     include_lint_checks: bool,
 }
 
 impl<'a, 'tcx> Pat<'tcx> {
-    pub fn from_hir(
+    crate fn from_hir(
         tcx: TyCtxt<'tcx>,
-        param_env_and_substs: ty::ParamEnvAnd<'tcx, SubstsRef<'tcx>>,
+        param_env: ty::ParamEnv<'tcx>,
         tables: &'a ty::TypeckTables<'tcx>,
         pat: &'tcx hir::Pat<'tcx>,
     ) -> Self {
-        let mut pcx = PatCtxt::new(tcx, param_env_and_substs, tables);
+        let mut pcx = PatCtxt::new(tcx, param_env, tables);
         let result = pcx.lower_pattern(pat);
         if !pcx.errors.is_empty() {
             let msg = format!("encountered errors lowering pattern: {:?}", pcx.errors);
@@ -369,27 +368,20 @@ impl<'a, 'tcx> Pat<'tcx> {
 }
 
 impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
-    pub fn new(
+    crate fn new(
         tcx: TyCtxt<'tcx>,
-        param_env_and_substs: ty::ParamEnvAnd<'tcx, SubstsRef<'tcx>>,
+        param_env: ty::ParamEnv<'tcx>,
         tables: &'a ty::TypeckTables<'tcx>,
     ) -> Self {
-        PatCtxt {
-            tcx,
-            param_env: param_env_and_substs.param_env,
-            tables,
-            substs: param_env_and_substs.value,
-            errors: vec![],
-            include_lint_checks: false,
-        }
+        PatCtxt { tcx, param_env, tables, errors: vec![], include_lint_checks: false }
     }
 
-    pub fn include_lint_checks(&mut self) -> &mut Self {
+    crate fn include_lint_checks(&mut self) -> &mut Self {
         self.include_lint_checks = true;
         self
     }
 
-    pub fn lower_pattern(&mut self, pat: &'tcx hir::Pat<'tcx>) -> Pat<'tcx> {
+    crate fn lower_pattern(&mut self, pat: &'tcx hir::Pat<'tcx>) -> Pat<'tcx> {
         // When implicit dereferences have been inserted in this pattern, the unadjusted lowered
         // pattern has the type that results *after* dereferencing. For example, in this code:
         //
@@ -834,7 +826,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
     }
 }
 
-impl UserAnnotatedTyHelpers<'tcx> for PatCtxt<'_, 'tcx> {
+impl<'tcx> UserAnnotatedTyHelpers<'tcx> for PatCtxt<'_, 'tcx> {
     fn tcx(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
@@ -844,7 +836,7 @@ impl UserAnnotatedTyHelpers<'tcx> for PatCtxt<'_, 'tcx> {
     }
 }
 
-pub trait PatternFoldable<'tcx>: Sized {
+crate trait PatternFoldable<'tcx>: Sized {
     fn fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
         self.super_fold_with(folder)
     }
@@ -852,7 +844,7 @@ pub trait PatternFoldable<'tcx>: Sized {
     fn super_fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self;
 }
 
-pub trait PatternFolder<'tcx>: Sized {
+crate trait PatternFolder<'tcx>: Sized {
     fn fold_pattern(&mut self, pattern: &Pat<'tcx>) -> Pat<'tcx> {
         pattern.super_fold_with(self)
     }
@@ -980,7 +972,7 @@ impl<'tcx> PatternFoldable<'tcx> for PatKind<'tcx> {
     }
 }
 
-pub fn compare_const_vals<'tcx>(
+crate fn compare_const_vals<'tcx>(
     tcx: TyCtxt<'tcx>,
     a: &'tcx ty::Const<'tcx>,
     b: &'tcx ty::Const<'tcx>,
diff --git a/src/librustc_mir/hair/util.rs b/src/librustc_mir_build/hair/util.rs
similarity index 100%
rename from src/librustc_mir/hair/util.rs
rename to src/librustc_mir_build/hair/util.rs
diff --git a/src/librustc_mir_build/lib.rs b/src/librustc_mir_build/lib.rs
new file mode 100644
index 0000000000000..8f02801522239
--- /dev/null
+++ b/src/librustc_mir_build/lib.rs
@@ -0,0 +1,27 @@
+//! Construction of MIR from HIR.
+//!
+//! This crate also contains the match exhaustiveness and usefulness checking.
+
+#![feature(box_patterns)]
+#![feature(box_syntax)]
+#![feature(crate_visibility_modifier)]
+#![feature(slice_patterns)]
+#![feature(bool_to_option)]
+
+#[macro_use]
+extern crate log;
+#[macro_use]
+extern crate rustc;
+#[macro_use]
+extern crate syntax;
+
+mod build;
+mod hair;
+mod lints;
+
+use rustc::ty::query::Providers;
+
+pub fn provide(providers: &mut Providers<'_>) {
+    providers.check_match = hair::pattern::check_match;
+    providers.mir_built = build::mir_built;
+}
diff --git a/src/librustc_mir/lints.rs b/src/librustc_mir_build/lints.rs
similarity index 97%
rename from src/librustc_mir/lints.rs
rename to src/librustc_mir_build/lints.rs
index fc7f2eb18b23a..b89378e7cad2c 100644
--- a/src/librustc_mir/lints.rs
+++ b/src/librustc_mir_build/lints.rs
@@ -7,7 +7,7 @@ use rustc::ty::{self, AssocItem, AssocItemContainer, Instance, TyCtxt};
 use rustc_hir::def_id::DefId;
 use rustc_index::bit_set::BitSet;
 
-pub fn check(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, def_id: DefId) {
+crate fn check<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, def_id: DefId) {
     let hir_id = tcx.hir().as_local_hir_id(def_id).unwrap();
 
     if let Some(fn_like_node) = FnLikeNode::from_node(tcx.hir().get(hir_id)) {
@@ -15,7 +15,7 @@ pub fn check(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, def_id: DefId) {
     }
 }
 
-fn check_fn_for_unconditional_recursion(
+fn check_fn_for_unconditional_recursion<'tcx>(
     tcx: TyCtxt<'tcx>,
     fn_kind: FnKind<'_>,
     body: &Body<'tcx>,
diff --git a/src/librustc_parse/parser/item.rs b/src/librustc_parse/parser/item.rs
index a05bc48981efe..8a725df6c95bb 100644
--- a/src/librustc_parse/parser/item.rs
+++ b/src/librustc_parse/parser/item.rs
@@ -5,7 +5,7 @@ use crate::maybe_whole;
 
 use rustc_error_codes::*;
 use rustc_errors::{Applicability, DiagnosticBuilder, PResult, StashKey};
-use rustc_span::source_map::{self, respan, Span};
+use rustc_span::source_map::{self, respan, Span, Spanned};
 use rustc_span::symbol::{kw, sym, Symbol};
 use rustc_span::BytePos;
 use syntax::ast::{self, AttrKind, AttrStyle, AttrVec, Attribute, Ident, DUMMY_NODE_ID};
@@ -543,10 +543,11 @@ impl<'a> Parser<'a> {
     ///    impl<'a, T> TYPE { /* impl items */ }
     ///    impl<'a, T> TRAIT for TYPE { /* impl items */ }
     ///    impl<'a, T> !TRAIT for TYPE { /* impl items */ }
+    ///    impl<'a, T> const TRAIT for TYPE { /* impl items */ }
     ///
     /// We actually parse slightly more relaxed grammar for better error reporting and recovery.
-    ///     `impl` GENERICS `!`? TYPE `for`? (TYPE | `..`) (`where` PREDICATES)? `{` BODY `}`
-    ///     `impl` GENERICS `!`? TYPE (`where` PREDICATES)? `{` BODY `}`
+    ///   `impl` GENERICS `const`? `!`? TYPE `for`? (TYPE | `..`) (`where` PREDICATES)? `{` BODY `}`
+    ///   `impl` GENERICS `const`? `!`? TYPE (`where` PREDICATES)? `{` BODY `}`
     fn parse_item_impl(
         &mut self,
         unsafety: Unsafety,
@@ -559,6 +560,14 @@ impl<'a> Parser<'a> {
             Generics::default()
         };
 
+        let constness = if self.eat_keyword(kw::Const) {
+            let span = self.prev_span;
+            self.sess.gated_spans.gate(sym::const_trait_impl, span);
+            Some(respan(span, Constness::Const))
+        } else {
+            None
+        };
+
         // Disambiguate `impl !Trait for Type { ... }` and `impl ! { ... }` for the never type.
         let polarity = if self.check(&token::Not) && self.look_ahead(1, |t| t.can_begin_type()) {
             self.bump(); // `!`
@@ -619,7 +628,8 @@ impl<'a> Parser<'a> {
                         err_path(ty_first.span)
                     }
                 };
-                let trait_ref = TraitRef { path, ref_id: ty_first.id };
+                let constness = constness.map(|c| c.node);
+                let trait_ref = TraitRef { path, constness, ref_id: ty_first.id };
 
                 ItemKind::Impl(
                     unsafety,
@@ -632,6 +642,13 @@ impl<'a> Parser<'a> {
                 )
             }
             None => {
+                // Reject `impl const Type {}` here
+                if let Some(Spanned { node: Constness::Const, span }) = constness {
+                    self.struct_span_err(span, "`const` cannot modify an inherent impl")
+                        .help("only a trait impl can be `const`")
+                        .emit();
+                }
+
                 // impl Type
                 ItemKind::Impl(
                     unsafety,
diff --git a/src/librustc_parse/parser/ty.rs b/src/librustc_parse/parser/ty.rs
index 4122aa17f83d3..2a220fb8d2467 100644
--- a/src/librustc_parse/parser/ty.rs
+++ b/src/librustc_parse/parser/ty.rs
@@ -6,7 +6,7 @@ use crate::{maybe_recover_from_interpolated_ty_qpath, maybe_whole};
 use rustc_error_codes::*;
 use rustc_errors::{pluralize, Applicability, PResult};
 use rustc_span::source_map::Span;
-use rustc_span::symbol::kw;
+use rustc_span::symbol::{kw, sym};
 use syntax::ast::{
     self, BareFnTy, FunctionRetTy, GenericParam, Ident, Lifetime, MutTy, Ty, TyKind,
 };
@@ -18,6 +18,24 @@ use syntax::ptr::P;
 use syntax::struct_span_err;
 use syntax::token::{self, Token};
 
+/// Any `?` or `?const` modifiers that appear at the start of a bound.
+struct BoundModifiers {
+    /// `?Trait`.
+    maybe: Option<Span>,
+
+    /// `?const Trait`.
+    maybe_const: Option<Span>,
+}
+
+impl BoundModifiers {
+    fn trait_bound_modifier(&self) -> TraitBoundModifier {
+        match self.maybe {
+            Some(_) => TraitBoundModifier::Maybe,
+            None => TraitBoundModifier::None,
+        }
+    }
+}
+
 /// Returns `true` if `IDENT t` can start a type -- `IDENT::a::b`, `IDENT<u8, u8>`,
 /// `IDENT<<u8 as Trait>::AssocTy>`.
 ///
@@ -196,7 +214,9 @@ impl<'a> Parser<'a> {
         lo: Span,
         parse_plus: bool,
     ) -> PResult<'a, TyKind> {
-        let poly_trait_ref = PolyTraitRef::new(generic_params, path, lo.to(self.prev_span));
+        assert_ne!(self.token, token::Question);
+
+        let poly_trait_ref = PolyTraitRef::new(generic_params, path, None, lo.to(self.prev_span));
         let mut bounds = vec![GenericBound::Trait(poly_trait_ref, TraitBoundModifier::None)];
         if parse_plus {
             self.eat_plus(); // `+`, or `+=` gets split and `+` is discarded
@@ -422,12 +442,15 @@ impl<'a> Parser<'a> {
         let has_parens = self.eat(&token::OpenDelim(token::Paren));
         let inner_lo = self.token.span;
         let is_negative = self.eat(&token::Not);
-        let question = self.eat(&token::Question).then_some(self.prev_span);
+
+        let modifiers = self.parse_ty_bound_modifiers();
         let bound = if self.token.is_lifetime() {
-            self.parse_generic_lt_bound(lo, inner_lo, has_parens, question)?
+            self.error_lt_bound_with_modifiers(modifiers);
+            self.parse_generic_lt_bound(lo, inner_lo, has_parens)?
         } else {
-            self.parse_generic_ty_bound(lo, has_parens, question)?
+            self.parse_generic_ty_bound(lo, has_parens, modifiers)?
         };
+
         Ok(if is_negative { Err(anchor_lo.to(self.prev_span)) } else { Ok(bound) })
     }
 
@@ -440,9 +463,7 @@ impl<'a> Parser<'a> {
         lo: Span,
         inner_lo: Span,
         has_parens: bool,
-        question: Option<Span>,
     ) -> PResult<'a, GenericBound> {
-        self.error_opt_out_lifetime(question);
         let bound = GenericBound::Outlives(self.expect_lifetime());
         if has_parens {
             // FIXME(Centril): Consider not erroring here and accepting `('lt)` instead,
@@ -452,8 +473,17 @@ impl<'a> Parser<'a> {
         Ok(bound)
     }
 
-    fn error_opt_out_lifetime(&self, question: Option<Span>) {
-        if let Some(span) = question {
+    /// Emits an error if any trait bound modifiers were present.
+    fn error_lt_bound_with_modifiers(&self, modifiers: BoundModifiers) {
+        if let Some(span) = modifiers.maybe_const {
+            self.struct_span_err(
+                span,
+                "`?const` may only modify trait bounds, not lifetime bounds",
+            )
+            .emit();
+        }
+
+        if let Some(span) = modifiers.maybe {
             self.struct_span_err(span, "`?` may only modify trait bounds, not lifetime bounds")
                 .emit();
         }
@@ -479,25 +509,58 @@ impl<'a> Parser<'a> {
         Ok(())
     }
 
+    /// Parses the modifiers that may precede a trait in a bound, e.g. `?Trait` or `?const Trait`.
+    ///
+    /// If no modifiers are present, this does not consume any tokens.
+    ///
+    /// ```
+    /// TY_BOUND_MODIFIERS = "?" ["const" ["?"]]
+    /// ```
+    fn parse_ty_bound_modifiers(&mut self) -> BoundModifiers {
+        if !self.eat(&token::Question) {
+            return BoundModifiers { maybe: None, maybe_const: None };
+        }
+
+        // `? ...`
+        let first_question = self.prev_span;
+        if !self.eat_keyword(kw::Const) {
+            return BoundModifiers { maybe: Some(first_question), maybe_const: None };
+        }
+
+        // `?const ...`
+        let maybe_const = first_question.to(self.prev_span);
+        self.sess.gated_spans.gate(sym::const_trait_bound_opt_out, maybe_const);
+        if !self.eat(&token::Question) {
+            return BoundModifiers { maybe: None, maybe_const: Some(maybe_const) };
+        }
+
+        // `?const ? ...`
+        let second_question = self.prev_span;
+        BoundModifiers { maybe: Some(second_question), maybe_const: Some(maybe_const) }
+    }
+
     /// Parses a type bound according to:
     /// ```
     /// TY_BOUND = TY_BOUND_NOPAREN | (TY_BOUND_NOPAREN)
-    /// TY_BOUND_NOPAREN = [?] [for<LT_PARAM_DEFS>] SIMPLE_PATH (e.g., `?for<'a: 'b> m::Trait<'a>`)
+    /// TY_BOUND_NOPAREN = [TY_BOUND_MODIFIERS] [for<LT_PARAM_DEFS>] SIMPLE_PATH
     /// ```
+    ///
+    /// For example, this grammar accepts `?const ?for<'a: 'b> m::Trait<'a>`.
     fn parse_generic_ty_bound(
         &mut self,
         lo: Span,
         has_parens: bool,
-        question: Option<Span>,
+        modifiers: BoundModifiers,
     ) -> PResult<'a, GenericBound> {
         let lifetime_defs = self.parse_late_bound_lifetime_defs()?;
         let path = self.parse_path(PathStyle::Type)?;
         if has_parens {
             self.expect(&token::CloseDelim(token::Paren))?;
         }
-        let poly_trait = PolyTraitRef::new(lifetime_defs, path, lo.to(self.prev_span));
-        let modifier = question.map_or(TraitBoundModifier::None, |_| TraitBoundModifier::Maybe);
-        Ok(GenericBound::Trait(poly_trait, modifier))
+
+        let constness = modifiers.maybe_const.map(|_| ast::Constness::NotConst);
+        let poly_trait = PolyTraitRef::new(lifetime_defs, path, constness, lo.to(self.prev_span));
+        Ok(GenericBound::Trait(poly_trait, modifiers.trait_bound_modifier()))
     }
 
     /// Optionally parses `for<$generic_params>`.
diff --git a/src/librustc_passes/ast_validation.rs b/src/librustc_passes/ast_validation.rs
index 038c4284f252b..e353e2ab777ee 100644
--- a/src/librustc_passes/ast_validation.rs
+++ b/src/librustc_passes/ast_validation.rs
@@ -24,6 +24,24 @@ use syntax::{span_err, struct_span_err, walk_list};
 
 use rustc_error_codes::*;
 
+/// A syntactic context that disallows certain kinds of bounds (e.g., `?Trait` or `?const Trait`).
+#[derive(Clone, Copy)]
+enum BoundContext {
+    ImplTrait,
+    TraitBounds,
+    TraitObject,
+}
+
+impl BoundContext {
+    fn description(&self) -> &'static str {
+        match self {
+            Self::ImplTrait => "`impl Trait`",
+            Self::TraitBounds => "supertraits",
+            Self::TraitObject => "trait objects",
+        }
+    }
+}
+
 struct AstValidator<'a> {
     session: &'a Session,
     has_proc_macro_decls: bool,
@@ -33,6 +51,12 @@ struct AstValidator<'a> {
     /// e.g., `impl Iterator<Item = impl Debug>`.
     outer_impl_trait: Option<Span>,
 
+    /// Keeps track of the `BoundContext` as we recurse.
+    ///
+    /// This is used to forbid `?const Trait` bounds in, e.g.,
+    /// `impl Iterator<Item = Box<dyn ?const Trait>`.
+    bound_context: Option<BoundContext>,
+
     /// Used to ban `impl Trait` in path projections like `<impl Iterator>::Item`
     /// or `Foo::Bar<impl Trait>`
     is_impl_trait_banned: bool,
@@ -59,10 +83,20 @@ impl<'a> AstValidator<'a> {
 
     fn with_impl_trait(&mut self, outer: Option<Span>, f: impl FnOnce(&mut Self)) {
         let old = mem::replace(&mut self.outer_impl_trait, outer);
-        f(self);
+        if outer.is_some() {
+            self.with_bound_context(BoundContext::ImplTrait, |this| f(this));
+        } else {
+            f(self)
+        }
         self.outer_impl_trait = old;
     }
 
+    fn with_bound_context(&mut self, ctx: BoundContext, f: impl FnOnce(&mut Self)) {
+        let old = self.bound_context.replace(ctx);
+        f(self);
+        self.bound_context = old;
+    }
+
     fn visit_assoc_ty_constraint_from_generic_args(&mut self, constraint: &'a AssocTyConstraint) {
         match constraint.kind {
             AssocTyConstraintKind::Equality { .. } => {}
@@ -84,6 +118,9 @@ impl<'a> AstValidator<'a> {
             TyKind::ImplTrait(..) => {
                 self.with_impl_trait(Some(t.span), |this| visit::walk_ty(this, t))
             }
+            TyKind::TraitObject(..) => {
+                self.with_bound_context(BoundContext::TraitObject, |this| visit::walk_ty(this, t));
+            }
             TyKind::Path(ref qself, ref path) => {
                 // We allow these:
                 //  - `Option<impl Trait>`
@@ -192,6 +229,7 @@ impl<'a> AstValidator<'a> {
         }
     }
 
+    // FIXME(ecstaticmorse): Instead, use `bound_context` to check this in `visit_param_bound`.
     fn no_questions_in_bounds(&self, bounds: &GenericBounds, where_: &str, is_trait: bool) {
         for bound in bounds {
             if let GenericBound::Trait(ref poly, TraitBoundModifier::Maybe) = *bound {
@@ -678,6 +716,18 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                     }
                 }
                 self.no_questions_in_bounds(bounds, "supertraits", true);
+
+                // Equivalent of `visit::walk_item` for `ItemKind::Trait` that inserts a bound
+                // context for the supertraits.
+                self.visit_vis(&item.vis);
+                self.visit_ident(item.ident);
+                self.visit_generics(generics);
+                self.with_bound_context(BoundContext::TraitBounds, |this| {
+                    walk_list!(this, visit_param_bound, bounds);
+                });
+                walk_list!(self, visit_trait_item, trait_items);
+                walk_list!(self, visit_attribute, &item.attrs);
+                return;
             }
             ItemKind::Mod(_) => {
                 // Ensure that `path` attributes on modules are recorded as used (cf. issue #35584).
@@ -822,6 +872,29 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         visit::walk_generic_param(self, param);
     }
 
+    fn visit_param_bound(&mut self, bound: &'a GenericBound) {
+        if let GenericBound::Trait(poly, maybe_bound) = bound {
+            match poly.trait_ref.constness {
+                Some(Constness::NotConst) => {
+                    if *maybe_bound == TraitBoundModifier::Maybe {
+                        self.err_handler()
+                            .span_err(bound.span(), "`?const` and `?` are mutually exclusive");
+                    }
+
+                    if let Some(ctx) = self.bound_context {
+                        let msg = format!("`?const` is not permitted in {}", ctx.description());
+                        self.err_handler().span_err(bound.span(), &msg);
+                    }
+                }
+
+                Some(Constness::Const) => bug!("Parser should reject bare `const` on bounds"),
+                None => {}
+            }
+        }
+
+        visit::walk_param_bound(self, bound)
+    }
+
     fn visit_pat(&mut self, pat: &'a Pat) {
         match pat.kind {
             PatKind::Lit(ref expr) => {
@@ -930,6 +1003,7 @@ pub fn check_crate(session: &Session, krate: &Crate, lints: &mut lint::LintBuffe
         session,
         has_proc_macro_decls: false,
         outer_impl_trait: None,
+        bound_context: None,
         is_impl_trait_banned: false,
         is_assoc_ty_bound_banned: false,
         lint_buffer: lints,
diff --git a/src/librustc_span/symbol.rs b/src/librustc_span/symbol.rs
index 40abc8b2179b8..d9f4b72560ceb 100644
--- a/src/librustc_span/symbol.rs
+++ b/src/librustc_span/symbol.rs
@@ -219,6 +219,8 @@ symbols! {
         const_raw_ptr_deref,
         const_raw_ptr_to_usize_cast,
         const_transmute,
+        const_trait_bound_opt_out,
+        const_trait_impl,
         contents,
         context,
         convert,
diff --git a/src/librustdoc/config.rs b/src/librustdoc/config.rs
index 0af25efc04234..26b49d2f9624e 100644
--- a/src/librustdoc/config.rs
+++ b/src/librustdoc/config.rs
@@ -24,7 +24,7 @@ use crate::html;
 use crate::html::markdown::IdMap;
 use crate::html::static_files;
 use crate::opts;
-use crate::passes::{self, DefaultPassOption};
+use crate::passes::{self, Condition, DefaultPassOption};
 use crate::theme;
 
 /// Configuration options for rustdoc.
@@ -98,6 +98,10 @@ pub struct Options {
     ///
     /// Be aware: This option can come both from the CLI and from crate attributes!
     pub default_passes: DefaultPassOption,
+    /// Document items that have lower than `pub` visibility.
+    pub document_private: bool,
+    /// Document items that have `doc(hidden)`.
+    pub document_hidden: bool,
     /// Any passes manually selected by the user.
     ///
     /// Be aware: This option can come both from the CLI and from crate attributes!
@@ -146,6 +150,8 @@ impl fmt::Debug for Options {
             .field("test_args", &self.test_args)
             .field("persist_doctests", &self.persist_doctests)
             .field("default_passes", &self.default_passes)
+            .field("document_private", &self.document_private)
+            .field("document_hidden", &self.document_hidden)
             .field("manual_passes", &self.manual_passes)
             .field("display_warnings", &self.display_warnings)
             .field("show_coverage", &self.show_coverage)
@@ -240,22 +246,26 @@ impl Options {
                 println!("{:>20} - {}", pass.name, pass.description);
             }
             println!("\nDefault passes for rustdoc:");
-            for pass in passes::DEFAULT_PASSES {
-                println!("{:>20}", pass.name);
-            }
-            println!("\nPasses run with `--document-private-items`:");
-            for pass in passes::DEFAULT_PRIVATE_PASSES {
-                println!("{:>20}", pass.name);
+            for p in passes::DEFAULT_PASSES {
+                print!("{:>20}", p.pass.name);
+                println_condition(p.condition);
             }
 
             if nightly_options::is_nightly_build() {
                 println!("\nPasses run with `--show-coverage`:");
-                for pass in passes::DEFAULT_COVERAGE_PASSES {
-                    println!("{:>20}", pass.name);
+                for p in passes::COVERAGE_PASSES {
+                    print!("{:>20}", p.pass.name);
+                    println_condition(p.condition);
                 }
-                println!("\nPasses run with `--show-coverage --document-private-items`:");
-                for pass in passes::PRIVATE_COVERAGE_PASSES {
-                    println!("{:>20}", pass.name);
+            }
+
+            fn println_condition(condition: Condition) {
+                use Condition::*;
+                match condition {
+                    Always => println!(),
+                    WhenDocumentPrivate => println!("  (when --document-private-items)"),
+                    WhenNotDocumentPrivate => println!("  (when not --document-private-items)"),
+                    WhenNotDocumentHidden => println!("  (when not --document-hidden-items)"),
                 }
             }
 
@@ -444,16 +454,11 @@ impl Options {
             });
 
         let show_coverage = matches.opt_present("show-coverage");
-        let document_private = matches.opt_present("document-private-items");
 
         let default_passes = if matches.opt_present("no-defaults") {
             passes::DefaultPassOption::None
-        } else if show_coverage && document_private {
-            passes::DefaultPassOption::PrivateCoverage
         } else if show_coverage {
             passes::DefaultPassOption::Coverage
-        } else if document_private {
-            passes::DefaultPassOption::Private
         } else {
             passes::DefaultPassOption::Default
         };
@@ -492,6 +497,8 @@ impl Options {
         let runtool = matches.opt_str("runtool");
         let runtool_args = matches.opt_strs("runtool-arg");
         let enable_per_target_ignores = matches.opt_present("enable-per-target-ignores");
+        let document_private = matches.opt_present("document-private-items");
+        let document_hidden = matches.opt_present("document-hidden-items");
 
         let (lint_opts, describe_lints, lint_cap) = get_cmd_lint_options(matches, error_format);
 
@@ -518,6 +525,8 @@ impl Options {
             should_test,
             test_args,
             default_passes,
+            document_private,
+            document_hidden,
             manual_passes,
             display_warnings,
             show_coverage,
diff --git a/src/librustdoc/core.rs b/src/librustdoc/core.rs
index 50d62027c8c6d..be7f7ea364f47 100644
--- a/src/librustdoc/core.rs
+++ b/src/librustdoc/core.rs
@@ -33,7 +33,7 @@ use crate::clean::{AttributesExt, MAX_DEF_ID};
 use crate::config::{Options as RustdocOptions, RenderOptions};
 use crate::html::render::RenderInfo;
 
-use crate::passes;
+use crate::passes::{self, Condition::*, ConditionalPass};
 
 pub use rustc::session::config::{CodegenOptions, DebuggingOptions, Input, Options};
 pub use rustc::session::search_paths::SearchPath;
@@ -221,6 +221,8 @@ pub fn run_core(options: RustdocOptions) -> (clean::Crate, RenderInfo, RenderOpt
         describe_lints,
         lint_cap,
         mut default_passes,
+        mut document_private,
+        document_hidden,
         mut manual_passes,
         display_warnings,
         render_options,
@@ -457,16 +459,14 @@ pub fn run_core(options: RustdocOptions) -> (clean::Crate, RenderInfo, RenderOpt
                     }
 
                     if attr.is_word() && name == sym::document_private_items {
-                        if default_passes == passes::DefaultPassOption::Default {
-                            default_passes = passes::DefaultPassOption::Private;
-                        }
+                        document_private = true;
                     }
                 }
 
-                let passes = passes::defaults(default_passes).iter().chain(
+                let passes = passes::defaults(default_passes).iter().copied().chain(
                     manual_passes.into_iter().flat_map(|name| {
                         if let Some(pass) = passes::find_pass(&name) {
-                            Some(pass)
+                            Some(ConditionalPass::always(pass))
                         } else {
                             error!("unknown pass {}, skipping", name);
                             None
@@ -476,9 +476,17 @@ pub fn run_core(options: RustdocOptions) -> (clean::Crate, RenderInfo, RenderOpt
 
                 info!("Executing passes");
 
-                for pass in passes {
-                    debug!("running pass {}", pass.name);
-                    krate = (pass.pass)(krate, &ctxt);
+                for p in passes {
+                    let run = match p.condition {
+                        Always => true,
+                        WhenDocumentPrivate => document_private,
+                        WhenNotDocumentPrivate => !document_private,
+                        WhenNotDocumentHidden => !document_hidden,
+                    };
+                    if run {
+                        debug!("running pass {}", p.pass.name);
+                        krate = (p.pass.run)(krate, &ctxt);
+                    }
                 }
 
                 ctxt.sess().abort_if_errors();
diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs
index eeaac1d8c7431..c9a1601ab87fb 100644
--- a/src/librustdoc/lib.rs
+++ b/src/librustdoc/lib.rs
@@ -173,6 +173,9 @@ fn opts() -> Vec<RustcOptGroup> {
         stable("document-private-items", |o| {
             o.optflag("", "document-private-items", "document private items")
         }),
+        unstable("document-hidden-items", |o| {
+            o.optflag("", "document-hidden-items", "document items that have doc(hidden)")
+        }),
         stable("test", |o| o.optflag("", "test", "run code examples as tests")),
         stable("test-args", |o| {
             o.optmulti("", "test-args", "arguments to pass to the test runner", "ARGS")
diff --git a/src/librustdoc/passes/calculate_doc_coverage.rs b/src/librustdoc/passes/calculate_doc_coverage.rs
index 803bcc2cfdf86..7ed531c9206af 100644
--- a/src/librustdoc/passes/calculate_doc_coverage.rs
+++ b/src/librustdoc/passes/calculate_doc_coverage.rs
@@ -12,7 +12,7 @@ use std::ops;
 
 pub const CALCULATE_DOC_COVERAGE: Pass = Pass {
     name: "calculate-doc-coverage",
-    pass: calculate_doc_coverage,
+    run: calculate_doc_coverage,
     description: "counts the number of items with and without documentation",
 };
 
diff --git a/src/librustdoc/passes/check_code_block_syntax.rs b/src/librustdoc/passes/check_code_block_syntax.rs
index a4ca9010f2e9f..79548eb6d647a 100644
--- a/src/librustdoc/passes/check_code_block_syntax.rs
+++ b/src/librustdoc/passes/check_code_block_syntax.rs
@@ -14,7 +14,7 @@ use crate::passes::Pass;
 
 pub const CHECK_CODE_BLOCK_SYNTAX: Pass = Pass {
     name: "check-code-block-syntax",
-    pass: check_code_block_syntax,
+    run: check_code_block_syntax,
     description: "validates syntax inside Rust code blocks",
 };
 
diff --git a/src/librustdoc/passes/collapse_docs.rs b/src/librustdoc/passes/collapse_docs.rs
index c6b22883e9723..c2185592d1483 100644
--- a/src/librustdoc/passes/collapse_docs.rs
+++ b/src/librustdoc/passes/collapse_docs.rs
@@ -8,7 +8,7 @@ use std::mem::take;
 
 pub const COLLAPSE_DOCS: Pass = Pass {
     name: "collapse-docs",
-    pass: collapse_docs,
+    run: collapse_docs,
     description: "concatenates all document attributes into one document attribute",
 };
 
diff --git a/src/librustdoc/passes/collect_intra_doc_links.rs b/src/librustdoc/passes/collect_intra_doc_links.rs
index 8a3966c320b8e..b1cd3deecb479 100644
--- a/src/librustdoc/passes/collect_intra_doc_links.rs
+++ b/src/librustdoc/passes/collect_intra_doc_links.rs
@@ -28,7 +28,7 @@ use super::span_of_attrs;
 
 pub const COLLECT_INTRA_DOC_LINKS: Pass = Pass {
     name: "collect-intra-doc-links",
-    pass: collect_intra_doc_links,
+    run: collect_intra_doc_links,
     description: "reads a crate's documentation to resolve intra-doc-links",
 };
 
diff --git a/src/librustdoc/passes/collect_trait_impls.rs b/src/librustdoc/passes/collect_trait_impls.rs
index c00e231ef70a2..da0e97f1075b0 100644
--- a/src/librustdoc/passes/collect_trait_impls.rs
+++ b/src/librustdoc/passes/collect_trait_impls.rs
@@ -9,7 +9,7 @@ use rustc_span::symbol::sym;
 
 pub const COLLECT_TRAIT_IMPLS: Pass = Pass {
     name: "collect-trait-impls",
-    pass: collect_trait_impls,
+    run: collect_trait_impls,
     description: "retrieves trait impls for items in the crate",
 };
 
diff --git a/src/librustdoc/passes/mod.rs b/src/librustdoc/passes/mod.rs
index aebed9171d0e4..355ea15223b0c 100644
--- a/src/librustdoc/passes/mod.rs
+++ b/src/librustdoc/passes/mod.rs
@@ -8,6 +8,7 @@ use rustc_span::{InnerSpan, Span, DUMMY_SP};
 use std::mem;
 use std::ops::Range;
 
+use self::Condition::*;
 use crate::clean::{self, GetDefId, Item};
 use crate::core::DocContext;
 use crate::fold::{DocFolder, StripItem};
@@ -52,10 +53,29 @@ pub use self::calculate_doc_coverage::CALCULATE_DOC_COVERAGE;
 #[derive(Copy, Clone)]
 pub struct Pass {
     pub name: &'static str,
-    pub pass: fn(clean::Crate, &DocContext<'_>) -> clean::Crate,
+    pub run: fn(clean::Crate, &DocContext<'_>) -> clean::Crate,
     pub description: &'static str,
 }
 
+/// In a list of passes, a pass that may or may not need to be run depending on options.
+#[derive(Copy, Clone)]
+pub struct ConditionalPass {
+    pub pass: Pass,
+    pub condition: Condition,
+}
+
+/// How to decide whether to run a conditional pass.
+#[derive(Copy, Clone)]
+pub enum Condition {
+    Always,
+    /// When `--document-private-items` is passed.
+    WhenDocumentPrivate,
+    /// When `--document-private-items` is not passed.
+    WhenNotDocumentPrivate,
+    /// When `--document-hidden-items` is not passed.
+    WhenNotDocumentHidden,
+}
+
 /// The full list of passes.
 pub const PASSES: &[Pass] = &[
     CHECK_PRIVATE_ITEMS_DOC_TESTS,
@@ -72,63 +92,58 @@ pub const PASSES: &[Pass] = &[
 ];
 
 /// The list of passes run by default.
-pub const DEFAULT_PASSES: &[Pass] = &[
-    COLLECT_TRAIT_IMPLS,
-    COLLAPSE_DOCS,
-    UNINDENT_COMMENTS,
-    CHECK_PRIVATE_ITEMS_DOC_TESTS,
-    STRIP_HIDDEN,
-    STRIP_PRIVATE,
-    COLLECT_INTRA_DOC_LINKS,
-    CHECK_CODE_BLOCK_SYNTAX,
-    PROPAGATE_DOC_CFG,
+pub const DEFAULT_PASSES: &[ConditionalPass] = &[
+    ConditionalPass::always(COLLECT_TRAIT_IMPLS),
+    ConditionalPass::always(COLLAPSE_DOCS),
+    ConditionalPass::always(UNINDENT_COMMENTS),
+    ConditionalPass::always(CHECK_PRIVATE_ITEMS_DOC_TESTS),
+    ConditionalPass::new(STRIP_HIDDEN, WhenNotDocumentHidden),
+    ConditionalPass::new(STRIP_PRIVATE, WhenNotDocumentPrivate),
+    ConditionalPass::new(STRIP_PRIV_IMPORTS, WhenDocumentPrivate),
+    ConditionalPass::always(COLLECT_INTRA_DOC_LINKS),
+    ConditionalPass::always(CHECK_CODE_BLOCK_SYNTAX),
+    ConditionalPass::always(PROPAGATE_DOC_CFG),
 ];
 
-/// The list of default passes run with `--document-private-items` is passed to rustdoc.
-pub const DEFAULT_PRIVATE_PASSES: &[Pass] = &[
-    COLLECT_TRAIT_IMPLS,
-    COLLAPSE_DOCS,
-    UNINDENT_COMMENTS,
-    CHECK_PRIVATE_ITEMS_DOC_TESTS,
-    STRIP_PRIV_IMPORTS,
-    COLLECT_INTRA_DOC_LINKS,
-    CHECK_CODE_BLOCK_SYNTAX,
-    PROPAGATE_DOC_CFG,
+/// The list of default passes run when `--doc-coverage` is passed to rustdoc.
+pub const COVERAGE_PASSES: &[ConditionalPass] = &[
+    ConditionalPass::always(COLLECT_TRAIT_IMPLS),
+    ConditionalPass::new(STRIP_HIDDEN, WhenNotDocumentHidden),
+    ConditionalPass::new(STRIP_PRIVATE, WhenNotDocumentPrivate),
+    ConditionalPass::always(CALCULATE_DOC_COVERAGE),
 ];
 
-/// The list of default passes run when `--doc-coverage` is passed to rustdoc.
-pub const DEFAULT_COVERAGE_PASSES: &[Pass] =
-    &[COLLECT_TRAIT_IMPLS, STRIP_HIDDEN, STRIP_PRIVATE, CALCULATE_DOC_COVERAGE];
+impl ConditionalPass {
+    pub const fn always(pass: Pass) -> Self {
+        Self::new(pass, Always)
+    }
 
-/// The list of default passes run when `--doc-coverage --document-private-items` is passed to
-/// rustdoc.
-pub const PRIVATE_COVERAGE_PASSES: &[Pass] = &[COLLECT_TRAIT_IMPLS, CALCULATE_DOC_COVERAGE];
+    pub const fn new(pass: Pass, condition: Condition) -> Self {
+        ConditionalPass { pass, condition }
+    }
+}
 
 /// A shorthand way to refer to which set of passes to use, based on the presence of
-/// `--no-defaults` or `--document-private-items`.
+/// `--no-defaults` and `--show-coverage`.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum DefaultPassOption {
     Default,
-    Private,
     Coverage,
-    PrivateCoverage,
     None,
 }
 
 /// Returns the given default set of passes.
-pub fn defaults(default_set: DefaultPassOption) -> &'static [Pass] {
+pub fn defaults(default_set: DefaultPassOption) -> &'static [ConditionalPass] {
     match default_set {
         DefaultPassOption::Default => DEFAULT_PASSES,
-        DefaultPassOption::Private => DEFAULT_PRIVATE_PASSES,
-        DefaultPassOption::Coverage => DEFAULT_COVERAGE_PASSES,
-        DefaultPassOption::PrivateCoverage => PRIVATE_COVERAGE_PASSES,
+        DefaultPassOption::Coverage => COVERAGE_PASSES,
         DefaultPassOption::None => &[],
     }
 }
 
 /// If the given name matches a known pass, returns its information.
-pub fn find_pass(pass_name: &str) -> Option<&'static Pass> {
-    PASSES.iter().find(|p| p.name == pass_name)
+pub fn find_pass(pass_name: &str) -> Option<Pass> {
+    PASSES.iter().find(|p| p.name == pass_name).copied()
 }
 
 struct Stripper<'a> {
diff --git a/src/librustdoc/passes/private_items_doc_tests.rs b/src/librustdoc/passes/private_items_doc_tests.rs
index 23e272709705d..aec5a6bd4e221 100644
--- a/src/librustdoc/passes/private_items_doc_tests.rs
+++ b/src/librustdoc/passes/private_items_doc_tests.rs
@@ -5,7 +5,7 @@ use crate::passes::{look_for_tests, Pass};
 
 pub const CHECK_PRIVATE_ITEMS_DOC_TESTS: Pass = Pass {
     name: "check-private-items-doc-tests",
-    pass: check_private_items_doc_tests,
+    run: check_private_items_doc_tests,
     description: "check private items doc tests",
 };
 
diff --git a/src/librustdoc/passes/propagate_doc_cfg.rs b/src/librustdoc/passes/propagate_doc_cfg.rs
index a296e73e3b5fd..64b0c45ba65d3 100644
--- a/src/librustdoc/passes/propagate_doc_cfg.rs
+++ b/src/librustdoc/passes/propagate_doc_cfg.rs
@@ -8,7 +8,7 @@ use crate::passes::Pass;
 
 pub const PROPAGATE_DOC_CFG: Pass = Pass {
     name: "propagate-doc-cfg",
-    pass: propagate_doc_cfg,
+    run: propagate_doc_cfg,
     description: "propagates `#[doc(cfg(...))]` to child items",
 };
 
diff --git a/src/librustdoc/passes/strip_hidden.rs b/src/librustdoc/passes/strip_hidden.rs
index 08a88c00cb0dd..f82e72b488bb7 100644
--- a/src/librustdoc/passes/strip_hidden.rs
+++ b/src/librustdoc/passes/strip_hidden.rs
@@ -10,7 +10,7 @@ use crate::passes::{ImplStripper, Pass};
 
 pub const STRIP_HIDDEN: Pass = Pass {
     name: "strip-hidden",
-    pass: strip_hidden,
+    run: strip_hidden,
     description: "strips all doc(hidden) items from the output",
 };
 
diff --git a/src/librustdoc/passes/strip_priv_imports.rs b/src/librustdoc/passes/strip_priv_imports.rs
index af34842ad0f89..35b26fb8ab0be 100644
--- a/src/librustdoc/passes/strip_priv_imports.rs
+++ b/src/librustdoc/passes/strip_priv_imports.rs
@@ -5,7 +5,7 @@ use crate::passes::{ImportStripper, Pass};
 
 pub const STRIP_PRIV_IMPORTS: Pass = Pass {
     name: "strip-priv-imports",
-    pass: strip_priv_imports,
+    run: strip_priv_imports,
     description: "strips all private import statements (`use`, `extern crate`) from a crate",
 };
 
diff --git a/src/librustdoc/passes/strip_private.rs b/src/librustdoc/passes/strip_private.rs
index f4ec9cc364904..f244956e50336 100644
--- a/src/librustdoc/passes/strip_private.rs
+++ b/src/librustdoc/passes/strip_private.rs
@@ -7,7 +7,7 @@ use crate::passes::{ImplStripper, ImportStripper, Pass, Stripper};
 
 pub const STRIP_PRIVATE: Pass = Pass {
     name: "strip-private",
-    pass: strip_private,
+    run: strip_private,
     description: "strips all private items from a crate which cannot be seen externally, \
         implies strip-priv-imports",
 };
diff --git a/src/librustdoc/passes/unindent_comments.rs b/src/librustdoc/passes/unindent_comments.rs
index 3212af055efc5..d4e09ce47a3c1 100644
--- a/src/librustdoc/passes/unindent_comments.rs
+++ b/src/librustdoc/passes/unindent_comments.rs
@@ -12,7 +12,7 @@ mod tests;
 
 pub const UNINDENT_COMMENTS: Pass = Pass {
     name: "unindent-comments",
-    pass: unindent_comments,
+    run: unindent_comments,
     description: "removes excess indentation on comments in order for markdown to like it",
 };
 
diff --git a/src/libstd/sys/vxworks/mod.rs b/src/libstd/sys/vxworks/mod.rs
index f102e4d6adf59..12bbfa1d4e1a6 100644
--- a/src/libstd/sys/vxworks/mod.rs
+++ b/src/libstd/sys/vxworks/mod.rs
@@ -36,18 +36,10 @@ pub use crate::sys_common::os_str_bytes as os_str;
 
 #[cfg(not(test))]
 pub fn init() {
-    // By default, some platforms will send a *signal* when an EPIPE error
-    // would otherwise be delivered. This runtime doesn't install a SIGPIPE
-    // handler, causing it to kill the program, which isn't exactly what we
-    // want!
-    //
-    // Hence, we set SIGPIPE to ignore when the program starts up in order
-    // to prevent this problem.
+    // ignore SIGPIPE
     unsafe {
-        reset_sigpipe();
+        assert!(signal(libc::SIGPIPE, libc::SIG_IGN) != libc::SIG_ERR);
     }
-
-    unsafe fn reset_sigpipe() {}
 }
 
 pub use libc::signal;
diff --git a/src/libstd/sys/vxworks/weak.rs b/src/libstd/sys/vxworks/weak.rs
deleted file mode 100644
index 4c6fddefd3f84..0000000000000
--- a/src/libstd/sys/vxworks/weak.rs
+++ /dev/null
@@ -1,56 +0,0 @@
-//! Support for "weak linkage" to symbols on Unix
-//!
-//! Some I/O operations we do in libstd require newer versions of OSes but we
-//! need to maintain binary compatibility with older releases for now. In order
-//! to use the new functionality when available we use this module for
-//! detection.
-//!
-//! One option to use here is weak linkage, but that is unfortunately only
-//! really workable on Linux. Hence, use dlsym to get the symbol value at
-//! runtime. This is also done for compatibility with older versions of glibc,
-//! and to avoid creating dependencies on GLIBC_PRIVATE symbols. It assumes that
-//! we've been dynamically linked to the library the symbol comes from, but that
-//! is currently always the case for things like libpthread/libc.
-//!
-//! A long time ago this used weak linkage for the __pthread_get_minstack
-//! symbol, but that caused Debian to detect an unnecessarily strict versioned
-//! dependency on libc6 (#23628).
-
-use crate::ffi::CStr;
-use crate::marker;
-use crate::mem;
-use crate::sync::atomic::{AtomicUsize, Ordering};
-
-pub struct Weak<F> {
-    name: &'static str,
-    addr: AtomicUsize,
-    _marker: marker::PhantomData<F>,
-}
-
-impl<F> Weak<F> {
-    pub const fn new(name: &'static str) -> Weak<F> {
-        Weak { name, addr: AtomicUsize::new(1), _marker: marker::PhantomData }
-    }
-
-    pub fn get(&self) -> Option<F> {
-        assert_eq!(mem::size_of::<F>(), mem::size_of::<usize>());
-        unsafe {
-            if self.addr.load(Ordering::SeqCst) == 1 {
-                self.addr.store(fetch(self.name), Ordering::SeqCst);
-            }
-            match self.addr.load(Ordering::SeqCst) {
-                0 => None,
-                addr => Some(mem::transmute_copy::<usize, F>(&addr)),
-            }
-        }
-    }
-}
-
-unsafe fn fetch(name: &str) -> usize {
-    let name = match CStr::from_bytes_with_nul(name.as_bytes()) {
-        Ok(cstr) => cstr,
-        Err(..) => return 0,
-    };
-    assert!(false, "FIXME: fetch");
-    libc::dlsym(libc::RTLD_DEFAULT, name.as_ptr()) as usize
-}
diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index 47070261385a2..1d3bb7d87686c 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -1033,7 +1033,7 @@ impl Expr {
     pub fn to_bound(&self) -> Option<GenericBound> {
         match &self.kind {
             ExprKind::Path(None, path) => Some(GenericBound::Trait(
-                PolyTraitRef::new(Vec::new(), path.clone(), self.span),
+                PolyTraitRef::new(Vec::new(), path.clone(), None, self.span),
                 TraitBoundModifier::None,
             )),
             _ => None,
@@ -2376,6 +2376,15 @@ pub enum AttrKind {
 pub struct TraitRef {
     pub path: Path,
     pub ref_id: NodeId,
+
+    /// The `const` modifier, if any, that appears before this trait.
+    ///
+    /// |                | `constness`                 |
+    /// |----------------|-----------------------------|
+    /// | `Trait`        | `None`                      |
+    /// | `const Trait`  | `Some(Constness::Const)`    |
+    /// | `?const Trait` | `Some(Constness::NotConst)` |
+    pub constness: Option<Constness>,
 }
 
 #[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
@@ -2390,10 +2399,15 @@ pub struct PolyTraitRef {
 }
 
 impl PolyTraitRef {
-    pub fn new(generic_params: Vec<GenericParam>, path: Path, span: Span) -> Self {
+    pub fn new(
+        generic_params: Vec<GenericParam>,
+        path: Path,
+        constness: Option<Constness>,
+        span: Span,
+    ) -> Self {
         PolyTraitRef {
             bound_generic_params: generic_params,
-            trait_ref: TraitRef { path, ref_id: DUMMY_NODE_ID },
+            trait_ref: TraitRef { path, constness, ref_id: DUMMY_NODE_ID },
             span,
         }
     }
diff --git a/src/libsyntax/feature_gate/check.rs b/src/libsyntax/feature_gate/check.rs
index 0e94a72277751..f8dccce6d1a07 100644
--- a/src/libsyntax/feature_gate/check.rs
+++ b/src/libsyntax/feature_gate/check.rs
@@ -916,6 +916,8 @@ pub fn check_crate(
     gate_all!(or_patterns, "or-patterns syntax is experimental");
     gate_all!(const_extern_fn, "`const extern fn` definitions are unstable");
     gate_all!(raw_ref_op, "raw address of syntax is experimental");
+    gate_all!(const_trait_bound_opt_out, "`?const` on trait bounds is experimental");
+    gate_all!(const_trait_impl, "const trait impls are experimental");
 
     // All uses of `gate_all!` below this point were added in #65742,
     // and subsequently disabled (with the non-early gating readded).
diff --git a/src/libsyntax/mut_visit.rs b/src/libsyntax/mut_visit.rs
index 1413f1566d043..264ba25cedecc 100644
--- a/src/libsyntax/mut_visit.rs
+++ b/src/libsyntax/mut_visit.rs
@@ -838,7 +838,8 @@ pub fn noop_visit_variant_data<T: MutVisitor>(vdata: &mut VariantData, vis: &mut
     }
 }
 
-pub fn noop_visit_trait_ref<T: MutVisitor>(TraitRef { path, ref_id }: &mut TraitRef, vis: &mut T) {
+pub fn noop_visit_trait_ref<T: MutVisitor>(tr: &mut TraitRef, vis: &mut T) {
+    let TraitRef { path, ref_id, constness: _ } = tr;
     vis.visit_path(path);
     vis.visit_id(ref_id);
 }
diff --git a/src/test/rustdoc/issue-46380.rs b/src/test/rustdoc/issue-46380.rs
deleted file mode 100644
index 8837a6b463e88..0000000000000
--- a/src/test/rustdoc/issue-46380.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-// compile-flags: --document-private-items
-
-// @has issue_46380/struct.Hidden.html
-#[doc(hidden)]
-pub struct Hidden;
diff --git a/src/test/rustdoc/issue-67851-both.rs b/src/test/rustdoc/issue-67851-both.rs
new file mode 100644
index 0000000000000..d69b943173412
--- /dev/null
+++ b/src/test/rustdoc/issue-67851-both.rs
@@ -0,0 +1,8 @@
+// compile-flags: -Zunstable-options --document-private-items --document-hidden-items
+
+// @has issue_67851_both/struct.Hidden.html
+#[doc(hidden)]
+pub struct Hidden;
+
+// @has issue_67851_both/struct.Private.html
+struct Private;
diff --git a/src/test/rustdoc/issue-67851-hidden.rs b/src/test/rustdoc/issue-67851-hidden.rs
new file mode 100644
index 0000000000000..8a3cafe4c3dc7
--- /dev/null
+++ b/src/test/rustdoc/issue-67851-hidden.rs
@@ -0,0 +1,8 @@
+// compile-flags: -Zunstable-options --document-hidden-items
+
+// @has issue_67851_hidden/struct.Hidden.html
+#[doc(hidden)]
+pub struct Hidden;
+
+// @!has issue_67851_hidden/struct.Private.html
+struct Private;
diff --git a/src/test/rustdoc/issue-67851-neither.rs b/src/test/rustdoc/issue-67851-neither.rs
new file mode 100644
index 0000000000000..4e3cd83285388
--- /dev/null
+++ b/src/test/rustdoc/issue-67851-neither.rs
@@ -0,0 +1,6 @@
+// @!has issue_67851_neither/struct.Hidden.html
+#[doc(hidden)]
+pub struct Hidden;
+
+// @!has issue_67851_neither/struct.Private.html
+struct Private;
diff --git a/src/test/rustdoc/issue-67851-private.rs b/src/test/rustdoc/issue-67851-private.rs
new file mode 100644
index 0000000000000..8addc7f3e4b53
--- /dev/null
+++ b/src/test/rustdoc/issue-67851-private.rs
@@ -0,0 +1,8 @@
+// compile-flags: --document-private-items
+
+// @!has issue_67851_private/struct.Hidden.html
+#[doc(hidden)]
+pub struct Hidden;
+
+// @has issue_67851_private/struct.Private.html
+struct Private;
diff --git a/src/test/ui/parser/bounds-type.rs b/src/test/ui/parser/bounds-type.rs
index 9122cb49ebc1a..7a187a0518af9 100644
--- a/src/test/ui/parser/bounds-type.rs
+++ b/src/test/ui/parser/bounds-type.rs
@@ -8,6 +8,11 @@ struct S<
     T: ?for<'a> Trait, // OK
     T: Tr +, // OK
     T: ?'a, //~ ERROR `?` may only modify trait bounds, not lifetime bounds
+
+    T: ?const Tr, // OK
+    T: ?const ?Tr, // OK
+    T: ?const Tr + 'a, // OK
+    T: ?const 'a, //~ ERROR `?const` may only modify trait bounds, not lifetime bounds
 >;
 
 fn main() {}
diff --git a/src/test/ui/parser/bounds-type.stderr b/src/test/ui/parser/bounds-type.stderr
index 0b714e40a1012..9a1f2ed398240 100644
--- a/src/test/ui/parser/bounds-type.stderr
+++ b/src/test/ui/parser/bounds-type.stderr
@@ -4,5 +4,11 @@ error: `?` may only modify trait bounds, not lifetime bounds
 LL |     T: ?'a,
    |        ^
 
-error: aborting due to previous error
+error: `?const` may only modify trait bounds, not lifetime bounds
+  --> $DIR/bounds-type.rs:15:8
+   |
+LL |     T: ?const 'a,
+   |        ^^^^^^
+
+error: aborting due to 2 previous errors
 
diff --git a/src/test/ui/pattern/const-pat-ice.stderr b/src/test/ui/pattern/const-pat-ice.stderr
index 7a0f14425b746..7217fe1b02f4e 100644
--- a/src/test/ui/pattern/const-pat-ice.stderr
+++ b/src/test/ui/pattern/const-pat-ice.stderr
@@ -1,4 +1,4 @@
-thread 'rustc' panicked at 'assertion failed: rows.iter().all(|r| r.len() == v.len())', src/librustc_mir/hair/pattern/_match.rs:LL:CC
+thread 'rustc' panicked at 'assertion failed: rows.iter().all(|r| r.len() == v.len())', src/librustc_mir_build/hair/pattern/_match.rs:LL:CC
 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace.
 
 error: internal compiler error: unexpected panic
diff --git a/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/feature-gate.gated.stderr b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/feature-gate.gated.stderr
new file mode 100644
index 0000000000000..0bf337ad08dbf
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/feature-gate.gated.stderr
@@ -0,0 +1,8 @@
+error: `?const` on trait bounds is not yet implemented
+  --> $DIR/feature-gate.rs:11:29
+   |
+LL | const fn get_assoc_const<S: ?const T>() -> i32 { <S as T>::CONST }
+   |                             ^^^^^^^^
+
+error: aborting due to previous error
+
diff --git a/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/feature-gate.rs b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/feature-gate.rs
new file mode 100644
index 0000000000000..cf1ed30da0fcc
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/feature-gate.rs
@@ -0,0 +1,15 @@
+// revisions: stock gated
+// gate-test-const_trait_bound_opt_out
+
+#![cfg_attr(gated, feature(const_trait_bound_opt_out))]
+#![allow(incomplete_features)]
+
+trait T {
+    const CONST: i32;
+}
+
+const fn get_assoc_const<S: ?const T>() -> i32 { <S as T>::CONST }
+//[stock]~^ ERROR `?const` on trait bounds is experimental
+//[stock,gated]~^^ ERROR `?const` on trait bounds is not yet implemented
+
+fn main() {}
diff --git a/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/feature-gate.stock.stderr b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/feature-gate.stock.stderr
new file mode 100644
index 0000000000000..64388004b5b72
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/feature-gate.stock.stderr
@@ -0,0 +1,18 @@
+error[E0658]: `?const` on trait bounds is experimental
+  --> $DIR/feature-gate.rs:11:29
+   |
+LL | const fn get_assoc_const<S: ?const T>() -> i32 { <S as T>::CONST }
+   |                             ^^^^^^
+   |
+   = note: for more information, see https://github.com/rust-lang/rust/issues/67794
+   = help: add `#![feature(const_trait_bound_opt_out)]` to the crate attributes to enable
+
+error: `?const` on trait bounds is not yet implemented
+  --> $DIR/feature-gate.rs:11:29
+   |
+LL | const fn get_assoc_const<S: ?const T>() -> i32 { <S as T>::CONST }
+   |                             ^^^^^^^^
+
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0658`.
diff --git a/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/in-impl-trait.rs b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/in-impl-trait.rs
new file mode 100644
index 0000000000000..e4e6bedd93746
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/in-impl-trait.rs
@@ -0,0 +1,25 @@
+#![feature(const_trait_bound_opt_out)]
+#![feature(associated_type_bounds)]
+#![allow(incomplete_features)]
+
+trait T {}
+struct S;
+impl T for S {}
+
+fn rpit() -> impl ?const T { S }
+//~^ ERROR `?const` is not permitted in `impl Trait`
+//~| ERROR `?const` on trait bounds is not yet implemented
+
+fn apit(_: impl ?const T) {}
+//~^ ERROR `?const` is not permitted in `impl Trait`
+//~| ERROR `?const` on trait bounds is not yet implemented
+
+fn rpit_assoc_bound() -> impl IntoIterator<Item: ?const T> { Some(S) }
+//~^ ERROR `?const` is not permitted in `impl Trait`
+//~| ERROR `?const` on trait bounds is not yet implemented
+
+fn apit_assoc_bound(_: impl IntoIterator<Item: ?const T>) {}
+//~^ ERROR `?const` is not permitted in `impl Trait`
+//~| ERROR `?const` on trait bounds is not yet implemented
+
+fn main() {}
diff --git a/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/in-impl-trait.stderr b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/in-impl-trait.stderr
new file mode 100644
index 0000000000000..f4abd4b714e8a
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/in-impl-trait.stderr
@@ -0,0 +1,50 @@
+error: `?const` is not permitted in `impl Trait`
+  --> $DIR/in-impl-trait.rs:9:19
+   |
+LL | fn rpit() -> impl ?const T { S }
+   |                   ^^^^^^^^
+
+error: `?const` is not permitted in `impl Trait`
+  --> $DIR/in-impl-trait.rs:13:17
+   |
+LL | fn apit(_: impl ?const T) {}
+   |                 ^^^^^^^^
+
+error: `?const` is not permitted in `impl Trait`
+  --> $DIR/in-impl-trait.rs:17:50
+   |
+LL | fn rpit_assoc_bound() -> impl IntoIterator<Item: ?const T> { Some(S) }
+   |                                                  ^^^^^^^^
+
+error: `?const` is not permitted in `impl Trait`
+  --> $DIR/in-impl-trait.rs:21:48
+   |
+LL | fn apit_assoc_bound(_: impl IntoIterator<Item: ?const T>) {}
+   |                                                ^^^^^^^^
+
+error: `?const` on trait bounds is not yet implemented
+  --> $DIR/in-impl-trait.rs:9:19
+   |
+LL | fn rpit() -> impl ?const T { S }
+   |                   ^^^^^^^^
+
+error: `?const` on trait bounds is not yet implemented
+  --> $DIR/in-impl-trait.rs:13:17
+   |
+LL | fn apit(_: impl ?const T) {}
+   |                 ^^^^^^^^
+
+error: `?const` on trait bounds is not yet implemented
+  --> $DIR/in-impl-trait.rs:17:50
+   |
+LL | fn rpit_assoc_bound() -> impl IntoIterator<Item: ?const T> { Some(S) }
+   |                                                  ^^^^^^^^
+
+error: `?const` on trait bounds is not yet implemented
+  --> $DIR/in-impl-trait.rs:21:48
+   |
+LL | fn apit_assoc_bound(_: impl IntoIterator<Item: ?const T>) {}
+   |                                                ^^^^^^^^
+
+error: aborting due to 8 previous errors
+
diff --git a/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/in-trait-bounds.rs b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/in-trait-bounds.rs
new file mode 100644
index 0000000000000..4523b46bc51f6
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/in-trait-bounds.rs
@@ -0,0 +1,9 @@
+#![feature(const_trait_bound_opt_out)]
+#![allow(incomplete_features)]
+
+trait Super {}
+trait T: ?const Super {}
+//~^ ERROR `?const` is not permitted in supertraits
+//~| ERROR `?const` on trait bounds is not yet implemented
+
+fn main() {}
diff --git a/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/in-trait-bounds.stderr b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/in-trait-bounds.stderr
new file mode 100644
index 0000000000000..8003361be7d2e
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/in-trait-bounds.stderr
@@ -0,0 +1,14 @@
+error: `?const` is not permitted in supertraits
+  --> $DIR/in-trait-bounds.rs:5:10
+   |
+LL | trait T: ?const Super {}
+   |          ^^^^^^^^^^^^
+
+error: `?const` on trait bounds is not yet implemented
+  --> $DIR/in-trait-bounds.rs:5:10
+   |
+LL | trait T: ?const Super {}
+   |          ^^^^^^^^^^^^
+
+error: aborting due to 2 previous errors
+
diff --git a/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/in-trait-object.rs b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/in-trait-object.rs
new file mode 100644
index 0000000000000..6cfca71548674
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/in-trait-object.rs
@@ -0,0 +1,22 @@
+#![feature(const_trait_bound_opt_out)]
+#![allow(bare_trait_objects)]
+#![allow(incomplete_features)]
+
+struct S;
+trait T {}
+impl T for S {}
+
+// An inherent impl for the trait object `?const T`.
+impl ?const T {}
+//~^ ERROR `?const` is not permitted in trait objects
+//~| ERROR `?const` on trait bounds is not yet implemented
+
+fn trait_object() -> &'static dyn ?const T { &S }
+//~^ ERROR `?const` is not permitted in trait objects
+//~| ERROR `?const` on trait bounds is not yet implemented
+
+fn trait_object_in_apit(_: impl IntoIterator<Item = Box<dyn ?const T>>) {}
+//~^ ERROR `?const` is not permitted in trait objects
+//~| ERROR `?const` on trait bounds is not yet implemented
+
+fn main() {}
diff --git a/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/in-trait-object.stderr b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/in-trait-object.stderr
new file mode 100644
index 0000000000000..c059f16902250
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/in-trait-object.stderr
@@ -0,0 +1,38 @@
+error: `?const` is not permitted in trait objects
+  --> $DIR/in-trait-object.rs:10:6
+   |
+LL | impl ?const T {}
+   |      ^^^^^^^^
+
+error: `?const` is not permitted in trait objects
+  --> $DIR/in-trait-object.rs:14:35
+   |
+LL | fn trait_object() -> &'static dyn ?const T { &S }
+   |                                   ^^^^^^^^
+
+error: `?const` is not permitted in trait objects
+  --> $DIR/in-trait-object.rs:18:61
+   |
+LL | fn trait_object_in_apit(_: impl IntoIterator<Item = Box<dyn ?const T>>) {}
+   |                                                             ^^^^^^^^
+
+error: `?const` on trait bounds is not yet implemented
+  --> $DIR/in-trait-object.rs:10:6
+   |
+LL | impl ?const T {}
+   |      ^^^^^^^^
+
+error: `?const` on trait bounds is not yet implemented
+  --> $DIR/in-trait-object.rs:14:35
+   |
+LL | fn trait_object() -> &'static dyn ?const T { &S }
+   |                                   ^^^^^^^^
+
+error: `?const` on trait bounds is not yet implemented
+  --> $DIR/in-trait-object.rs:18:61
+   |
+LL | fn trait_object_in_apit(_: impl IntoIterator<Item = Box<dyn ?const T>>) {}
+   |                                                             ^^^^^^^^
+
+error: aborting due to 6 previous errors
+
diff --git a/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/opt-out-twice.rs b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/opt-out-twice.rs
new file mode 100644
index 0000000000000..01e941a8fba45
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/opt-out-twice.rs
@@ -0,0 +1,8 @@
+// compile-flags: -Z parse-only
+
+#![feature(const_trait_bound_opt_out)]
+#![allow(incomplete_features)]
+
+struct S<T: ?const ?const Tr>;
+//~^ ERROR expected identifier, found keyword `const`
+//~| ERROR expected one of `(`, `+`, `,`, `::`, `<`, `=`, or `>`
diff --git a/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/opt-out-twice.stderr b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/opt-out-twice.stderr
new file mode 100644
index 0000000000000..f7924b3f24db3
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/opt-out-twice.stderr
@@ -0,0 +1,14 @@
+error: expected identifier, found keyword `const`
+  --> $DIR/opt-out-twice.rs:6:21
+   |
+LL | struct S<T: ?const ?const Tr>;
+   |                     ^^^^^ expected identifier, found keyword
+
+error: expected one of `(`, `+`, `,`, `::`, `<`, `=`, or `>`, found `Tr`
+  --> $DIR/opt-out-twice.rs:6:27
+   |
+LL | struct S<T: ?const ?const Tr>;
+   |                           ^^ expected one of 7 possible tokens
+
+error: aborting due to 2 previous errors
+
diff --git a/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/syntax.rs b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/syntax.rs
new file mode 100644
index 0000000000000..a0d9610bbb5e2
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/syntax.rs
@@ -0,0 +1,10 @@
+// compile-flags: -Z parse-only
+// check-pass
+
+#![feature(const_trait_bound_opt_out)]
+#![allow(incomplete_features)]
+
+struct S<
+    T: ?const ?for<'a> Tr<'a> + 'static + ?const std::ops::Add,
+    T: ?const ?for<'a: 'b> m::Trait<'a>,
+>;
diff --git a/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/with-maybe-sized.rs b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/with-maybe-sized.rs
new file mode 100644
index 0000000000000..425784f4e4326
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/with-maybe-sized.rs
@@ -0,0 +1,8 @@
+#![feature(const_trait_bound_opt_out)]
+#![allow(incomplete_features)]
+
+struct S<T: ?const ?Sized>(std::marker::PhantomData<T>);
+//~^ ERROR `?const` and `?` are mutually exclusive
+//~| ERROR `?const` on trait bounds is not yet implemented
+
+fn main() {}
diff --git a/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/with-maybe-sized.stderr b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/with-maybe-sized.stderr
new file mode 100644
index 0000000000000..44f6d464ae6a8
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/with-maybe-sized.stderr
@@ -0,0 +1,14 @@
+error: `?const` and `?` are mutually exclusive
+  --> $DIR/with-maybe-sized.rs:4:13
+   |
+LL | struct S<T: ?const ?Sized>(std::marker::PhantomData<T>);
+   |             ^^^^^^^^^^^^^
+
+error: `?const` on trait bounds is not yet implemented
+  --> $DIR/with-maybe-sized.rs:4:13
+   |
+LL | struct S<T: ?const ?Sized>(std::marker::PhantomData<T>);
+   |             ^^^^^^^^^^^^^
+
+error: aborting due to 2 previous errors
+
diff --git a/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/without-question-mark.rs b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/without-question-mark.rs
new file mode 100644
index 0000000000000..b904a2eec0dd0
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/without-question-mark.rs
@@ -0,0 +1,7 @@
+// compile-flags: -Z parse-only
+
+#![feature(const_trait_bound_opt_out)]
+#![allow(incomplete_features)]
+
+struct S<T: const Tr>;
+//~^ ERROR expected one of `!`, `(`, `,`, `=`, `>`, `?`, `for`, lifetime, or path
diff --git a/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/without-question-mark.stderr b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/without-question-mark.stderr
new file mode 100644
index 0000000000000..0dbca952c037e
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/const-trait-bound-opt-out/without-question-mark.stderr
@@ -0,0 +1,8 @@
+error: expected one of `!`, `(`, `,`, `=`, `>`, `?`, `for`, lifetime, or path, found keyword `const`
+  --> $DIR/without-question-mark.rs:6:13
+   |
+LL | struct S<T: const Tr>;
+   |             ^^^^^ expected one of 9 possible tokens
+
+error: aborting due to previous error
+
diff --git a/src/test/ui/rfc-2632-const-trait-impl/feature-gate.gated.stderr b/src/test/ui/rfc-2632-const-trait-impl/feature-gate.gated.stderr
new file mode 100644
index 0000000000000..b196f9ef57380
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/feature-gate.gated.stderr
@@ -0,0 +1,8 @@
+error: const trait impls are not yet implemented
+  --> $DIR/feature-gate.rs:9:1
+   |
+LL | impl const T for S {}
+   | ^^^^^^^^^^^^^^^^^^^^^
+
+error: aborting due to previous error
+
diff --git a/src/test/ui/rfc-2632-const-trait-impl/feature-gate.rs b/src/test/ui/rfc-2632-const-trait-impl/feature-gate.rs
new file mode 100644
index 0000000000000..49b6c0926c50c
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/feature-gate.rs
@@ -0,0 +1,13 @@
+// revisions: stock gated
+// gate-test-const_trait_impl
+
+#![cfg_attr(gated, feature(const_trait_impl))]
+#![allow(incomplete_features)]
+
+struct S;
+trait T {}
+impl const T for S {}
+//[stock]~^ ERROR const trait impls are experimental
+//[stock,gated]~^^ ERROR const trait impls are not yet implemented
+
+fn main() {}
diff --git a/src/test/ui/rfc-2632-const-trait-impl/feature-gate.stock.stderr b/src/test/ui/rfc-2632-const-trait-impl/feature-gate.stock.stderr
new file mode 100644
index 0000000000000..093946f859ac3
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/feature-gate.stock.stderr
@@ -0,0 +1,18 @@
+error[E0658]: const trait impls are experimental
+  --> $DIR/feature-gate.rs:9:6
+   |
+LL | impl const T for S {}
+   |      ^^^^^
+   |
+   = note: for more information, see https://github.com/rust-lang/rust/issues/67792
+   = help: add `#![feature(const_trait_impl)]` to the crate attributes to enable
+
+error: const trait impls are not yet implemented
+  --> $DIR/feature-gate.rs:9:1
+   |
+LL | impl const T for S {}
+   | ^^^^^^^^^^^^^^^^^^^^^
+
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0658`.
diff --git a/src/test/ui/rfc-2632-const-trait-impl/impl-opt-out-trait.rs b/src/test/ui/rfc-2632-const-trait-impl/impl-opt-out-trait.rs
new file mode 100644
index 0000000000000..98d3a220d8674
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/impl-opt-out-trait.rs
@@ -0,0 +1,11 @@
+#![feature(const_trait_bound_opt_out)]
+#![feature(const_trait_impl)]
+#![allow(incomplete_features)]
+
+struct S;
+trait T {}
+
+impl ?const T for S {}
+//~^ ERROR expected a trait, found type
+
+fn main() {}
diff --git a/src/test/ui/rfc-2632-const-trait-impl/impl-opt-out-trait.stderr b/src/test/ui/rfc-2632-const-trait-impl/impl-opt-out-trait.stderr
new file mode 100644
index 0000000000000..8f923efb093f3
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/impl-opt-out-trait.stderr
@@ -0,0 +1,8 @@
+error: expected a trait, found type
+  --> $DIR/impl-opt-out-trait.rs:8:6
+   |
+LL | impl ?const T for S {}
+   |      ^^^^^^^^
+
+error: aborting due to previous error
+
diff --git a/src/test/ui/rfc-2632-const-trait-impl/inherent-impl.rs b/src/test/ui/rfc-2632-const-trait-impl/inherent-impl.rs
new file mode 100644
index 0000000000000..9cffe75addd63
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/inherent-impl.rs
@@ -0,0 +1,14 @@
+// compile-flags: -Z parse-only
+
+#![feature(const_trait_impl)]
+#![feature(const_trait_bound_opt_out)]
+#![allow(incomplete_features)]
+#![allow(bare_trait_objects)]
+
+struct S;
+trait T {}
+
+impl const T {}
+//~^ ERROR `const` cannot modify an inherent impl
+
+fn main() {}
diff --git a/src/test/ui/rfc-2632-const-trait-impl/inherent-impl.stderr b/src/test/ui/rfc-2632-const-trait-impl/inherent-impl.stderr
new file mode 100644
index 0000000000000..1d24557655951
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/inherent-impl.stderr
@@ -0,0 +1,10 @@
+error: `const` cannot modify an inherent impl
+  --> $DIR/inherent-impl.rs:11:6
+   |
+LL | impl const T {}
+   |      ^^^^^
+   |
+   = help: only a trait impl can be `const`
+
+error: aborting due to previous error
+
diff --git a/src/test/ui/rfc-2632-const-trait-impl/syntax.rs b/src/test/ui/rfc-2632-const-trait-impl/syntax.rs
new file mode 100644
index 0000000000000..354d48d630f7b
--- /dev/null
+++ b/src/test/ui/rfc-2632-const-trait-impl/syntax.rs
@@ -0,0 +1,9 @@
+// compile-flags: -Z parse-only
+// check-pass
+
+#![feature(const_trait_bound_opt_out)]
+#![feature(const_trait_impl)]
+#![allow(incomplete_features)]
+
+// For now, this parses since an error does not occur until AST lowering.
+impl ?const T {}