diff --git a/Cargo.lock b/Cargo.lock
index 3237b90a7c06f..30ff11915258d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3802,6 +3802,7 @@ dependencies = [
"rustc_data_structures",
"rustc_fs_util",
"rustc_hir",
+ "rustc_query_system",
"rustc_session",
"rustc_span",
"serialize",
diff --git a/src/librustc/dep_graph/mod.rs b/src/librustc/dep_graph/mod.rs
index f56df19bfb061..5653af6b2bfb3 100644
--- a/src/librustc/dep_graph/mod.rs
+++ b/src/librustc/dep_graph/mod.rs
@@ -11,8 +11,8 @@ mod dep_node;
pub(crate) use rustc_query_system::dep_graph::DepNodeParams;
pub use rustc_query_system::dep_graph::{
- debug, hash_result, DepContext, DepNodeColor, DepNodeIndex, SerializedDepNodeIndex,
- WorkProduct, WorkProductFileKind, WorkProductId,
+ debug, hash_result, DepContext, DepNodeColor, DepNodeIndex, WorkProduct, WorkProductFileKind,
+ WorkProductId,
};
pub use dep_node::{label_strs, DepConstructor, DepKind, DepNode, DepNodeExt};
@@ -159,8 +159,8 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
try_load_from_on_disk_cache(*self, dep_node)
}
- fn load_diagnostics(&self, prev_dep_node_index: SerializedDepNodeIndex) -> Vec<Diagnostic> {
- self.queries.on_disk_cache.load_diagnostics(*self, prev_dep_node_index)
+ fn load_diagnostics(&self, dep_node_index: DepNodeIndex) -> Vec<Diagnostic> {
+ self.queries.on_disk_cache.load_diagnostics(*self, dep_node_index)
}
fn store_diagnostics(&self, dep_node_index: DepNodeIndex, diagnostics: ThinVec<Diagnostic>) {
diff --git a/src/librustc/query/mod.rs b/src/librustc/query/mod.rs
index c1ece6275092d..0ab133f513a53 100644
--- a/src/librustc/query/mod.rs
+++ b/src/librustc/query/mod.rs
@@ -1,4 +1,4 @@
-use crate::dep_graph::SerializedDepNodeIndex;
+use crate::dep_graph::DepNodeIndex;
use crate::mir;
use crate::mir::interpret::{GlobalId, LitToConstInput};
use crate::traits;
diff --git a/src/librustc/ty/query/on_disk_cache.rs b/src/librustc/ty/query/on_disk_cache.rs
index 8aecc0e698a8e..967e3a130f3b6 100644
--- a/src/librustc/ty/query/on_disk_cache.rs
+++ b/src/librustc/ty/query/on_disk_cache.rs
@@ -1,4 +1,4 @@
-use crate::dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
+use crate::dep_graph::DepNodeIndex;
use crate::mir::interpret::{AllocDecodingSession, AllocDecodingState};
use crate::mir::{self, interpret};
use crate::ty::codec::{self as ty_codec, TyDecoder, TyEncoder};
@@ -12,7 +12,7 @@ use rustc_data_structures::thin_vec::ThinVec;
use rustc_errors::Diagnostic;
use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, LOCAL_CRATE};
use rustc_hir::definitions::DefPathHash;
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_index::vec::IndexVec;
use rustc_serialize::{
opaque, Decodable, Decoder, Encodable, Encoder, SpecializedDecoder, SpecializedEncoder,
UseSpecializedDecodable, UseSpecializedEncodable,
@@ -60,11 +60,11 @@ pub struct OnDiskCache<'sess> {
// A map from dep-node to the position of the cached query result in
// `serialized_data`.
- query_result_index: FxHashMap<SerializedDepNodeIndex, AbsoluteBytePos>,
+ query_result_index: FxHashMap<DepNodeIndex, AbsoluteBytePos>,
// A map from dep-node to the position of any associated diagnostics in
// `serialized_data`.
- prev_diagnostics_index: FxHashMap<SerializedDepNodeIndex, AbsoluteBytePos>,
+ prev_diagnostics_index: FxHashMap<DepNodeIndex, AbsoluteBytePos>,
alloc_decoding_state: AllocDecodingState,
}
@@ -80,8 +80,8 @@ struct Footer {
interpret_alloc_index: Vec<u32>,
}
-type EncodedQueryResultIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>;
-type EncodedDiagnosticsIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>;
+type EncodedQueryResultIndex = Vec<(DepNodeIndex, AbsoluteBytePos)>;
+type EncodedDiagnosticsIndex = Vec<(DepNodeIndex, AbsoluteBytePos)>;
type EncodedDiagnostics = Vec<Diagnostic>;
#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, RustcEncodable, RustcDecodable)]
@@ -224,11 +224,10 @@ impl<'sess> OnDiskCache<'sess> {
.current_diagnostics
.borrow()
.iter()
- .map(|(dep_node_index, diagnostics)| {
+ .map(|(&dep_node_index, diagnostics)| {
let pos = AbsoluteBytePos::new(encoder.position());
// Let's make sure we get the expected type here.
let diagnostics: &EncodedDiagnostics = diagnostics;
- let dep_node_index = SerializedDepNodeIndex::new(dep_node_index.index());
encoder.encode_tagged(dep_node_index, diagnostics)?;
Ok((dep_node_index, pos))
@@ -304,7 +303,7 @@ impl<'sess> OnDiskCache<'sess> {
pub fn load_diagnostics(
&self,
tcx: TyCtxt<'_>,
- dep_node_index: SerializedDepNodeIndex,
+ dep_node_index: DepNodeIndex,
) -> Vec<Diagnostic> {
let diagnostics: Option<EncodedDiagnostics> =
self.load_indexed(tcx, dep_node_index, &self.prev_diagnostics_index, "diagnostics");
@@ -328,11 +327,11 @@ impl<'sess> OnDiskCache<'sess> {
}
/// Returns the cached query result if there is something in the cache for
- /// the given `SerializedDepNodeIndex`; otherwise returns `None`.
+ /// the given `DepNodeIndex`; otherwise returns `None`.
pub fn try_load_query_result<T>(
&self,
tcx: TyCtxt<'_>,
- dep_node_index: SerializedDepNodeIndex,
+ dep_node_index: DepNodeIndex,
) -> Option<T>
where
T: Decodable,
@@ -361,8 +360,8 @@ impl<'sess> OnDiskCache<'sess> {
fn load_indexed<'tcx, T>(
&self,
tcx: TyCtxt<'tcx>,
- dep_node_index: SerializedDepNodeIndex,
- index: &FxHashMap<SerializedDepNodeIndex, AbsoluteBytePos>,
+ dep_node_index: DepNodeIndex,
+ index: &FxHashMap<DepNodeIndex, AbsoluteBytePos>,
debug_tag: &'static str,
) -> Option<T>
where
@@ -1009,12 +1008,10 @@ where
state.iter_results(|results| {
for (key, value, dep_node) in results {
if Q::cache_on_disk(tcx, key.clone(), Some(&value)) {
- let dep_node = SerializedDepNodeIndex::new(dep_node.index());
-
// Record position of the cache entry.
query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position())));
- // Encode the type check tables with the `SerializedDepNodeIndex`
+ // Encode the type check tables with the `DepNodeIndex`
// as tag.
encoder.encode_tagged(dep_node, &value)?;
}
diff --git a/src/librustc_data_structures/sharded.rs b/src/librustc_data_structures/sharded.rs
index 485719c517564..0cc0b5eb8399b 100644
--- a/src/librustc_data_structures/sharded.rs
+++ b/src/librustc_data_structures/sharded.rs
@@ -30,16 +30,16 @@ pub struct Sharded<T> {
impl<T: Default> Default for Sharded<T> {
#[inline]
fn default() -> Self {
- Self::new(T::default)
+ Self::new(|_| T::default())
}
}
impl<T> Sharded<T> {
#[inline]
- pub fn new(mut value: impl FnMut() -> T) -> Self {
+ pub fn new(mut value: impl FnMut(usize) -> T) -> Self {
// Create a vector of the values we want
let mut values: SmallVec<[_; SHARDS]> =
- (0..SHARDS).map(|_| CacheAligned(Lock::new(value()))).collect();
+ (0..SHARDS).map(|i| CacheAligned(Lock::new(value(i)))).collect();
// Create an uninitialized array
let mut shards: mem::MaybeUninit<[CacheAligned<Lock<T>>; SHARDS]> =
@@ -63,6 +63,15 @@ impl<T> Sharded<T> {
if SHARDS == 1 { &self.shards[0].0 } else { self.get_shard_by_hash(make_hash(val)) }
}
+ /// The shard is selected by hashing `val` with `FxHasher`.
+ pub fn get_shard_by_value_mut<K: Hash + ?Sized>(&mut self, val: &K) -> &mut T {
+ if SHARDS == 1 {
+ self.shards[0].0.get_mut()
+ } else {
+ self.shards[self.get_shard_index_by_hash(make_hash(val))].0.get_mut()
+ }
+ }
+
/// Get a shard with a pre-computed hash value. If `get_shard_by_value` is
/// ever used in combination with `get_shard_by_hash` on a single `Sharded`
/// instance, then `hash` must be computed with `FxHasher`. Otherwise,
diff --git a/src/librustc_incremental/Cargo.toml b/src/librustc_incremental/Cargo.toml
index eddfb81ea8b0a..c930b2b7e2592 100644
--- a/src/librustc_incremental/Cargo.toml
+++ b/src/librustc_incremental/Cargo.toml
@@ -21,3 +21,4 @@ rustc_ast = { path = "../librustc_ast" }
rustc_span = { path = "../librustc_span" }
rustc_fs_util = { path = "../librustc_fs_util" }
rustc_session = { path = "../librustc_session" }
+rustc_query_system = { path = "../librustc_query_system" }
diff --git a/src/librustc_incremental/persist/load.rs b/src/librustc_incremental/persist/load.rs
index 537906eb87189..7e962d72c267a 100644
--- a/src/librustc_incremental/persist/load.rs
+++ b/src/librustc_incremental/persist/load.rs
@@ -1,9 +1,10 @@
//! Code to save/load the dep-graph from files.
-use rustc::dep_graph::{PreviousDepGraph, SerializedDepGraph, WorkProduct, WorkProductId};
+use rustc::dep_graph::{DepKind, PreviousDepGraph, SerializedDepGraph};
use rustc::ty::query::OnDiskCache;
use rustc::ty::TyCtxt;
use rustc_data_structures::fx::FxHashMap;
+use rustc_query_system::dep_graph::{CurrentDepGraph, DepGraphArgs};
use rustc_serialize::opaque::Decoder;
use rustc_serialize::Decodable as RustcDecodable;
use rustc_session::Session;
@@ -22,16 +23,14 @@ pub fn dep_graph_tcx_init(tcx: TyCtxt<'_>) {
tcx.allocate_metadata_dep_nodes();
}
-type WorkProductMap = FxHashMap<WorkProductId, WorkProduct>;
-
pub enum LoadResult<T> {
Ok { data: T },
DataOutOfDate,
Error { message: String },
}
-impl LoadResult<(PreviousDepGraph, WorkProductMap)> {
- pub fn open(self, sess: &Session) -> (PreviousDepGraph, WorkProductMap) {
+impl LoadResult<DepGraphArgs<DepKind>> {
+ pub fn open(self, sess: &Session) -> DepGraphArgs<DepKind> {
match self {
LoadResult::Error { message } => {
sess.warn(&message);
@@ -88,7 +87,7 @@ impl<T> MaybeAsync<T> {
}
}
-pub type DepGraphFuture = MaybeAsync<LoadResult<(PreviousDepGraph, WorkProductMap)>>;
+pub type DepGraphFuture = MaybeAsync<LoadResult<DepGraphArgs<DepKind>>>;
/// Launch a thread and load the dependency graph in the background.
pub fn load_dep_graph(sess: &Session) -> DepGraphFuture {
@@ -188,7 +187,13 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture {
let dep_graph = SerializedDepGraph::decode(&mut decoder)
.expect("Error reading cached dep-graph");
- LoadResult::Ok { data: (PreviousDepGraph::new(dep_graph), prev_work_products) }
+ let (prev_graph, state) = PreviousDepGraph::new_and_state(dep_graph);
+ let current = prof
+ .generic_activity("incr_comp_load_setup_dep_graph")
+ .run(|| CurrentDepGraph::new(&prev_graph));
+ LoadResult::Ok {
+ data: DepGraphArgs { state, prev_graph, prev_work_products, current },
+ }
}
}
}))
diff --git a/src/librustc_interface/queries.rs b/src/librustc_interface/queries.rs
index b0eeb57173fa3..6b13704e83373 100644
--- a/src/librustc_interface/queries.rs
+++ b/src/librustc_interface/queries.rs
@@ -191,16 +191,15 @@ impl<'tcx> Queries<'tcx> {
Ok(match self.dep_graph_future()?.take() {
None => DepGraph::new_disabled(),
Some(future) => {
- let (prev_graph, prev_work_products) =
- self.session().time("blocked_on_dep_graph_loading", || {
- future
- .open()
- .unwrap_or_else(|e| rustc_incremental::LoadResult::Error {
- message: format!("could not decode incremental cache: {:?}", e),
- })
- .open(self.session())
- });
- DepGraph::new(prev_graph, prev_work_products)
+ let args = self.session().time("blocked_on_dep_graph_loading", || {
+ future
+ .open()
+ .unwrap_or_else(|e| rustc_incremental::LoadResult::Error {
+ message: format!("could not decode incremental cache: {:?}", e),
+ })
+ .open(self.session())
+ });
+ DepGraph::new(args)
}
})
})
diff --git a/src/librustc_macros/src/query.rs b/src/librustc_macros/src/query.rs
index 26c3bce4a9a02..ba33accbd9104 100644
--- a/src/librustc_macros/src/query.rs
+++ b/src/librustc_macros/src/query.rs
@@ -324,7 +324,7 @@ fn add_query_description_impl(
#[inline]
fn try_load_from_disk(
#tcx: TyCtxt<'tcx>,
- #id: SerializedDepNodeIndex
+ #id: DepNodeIndex
) -> Option<Self::Value> {
#block
}
@@ -335,7 +335,7 @@ fn add_query_description_impl(
#[inline]
fn try_load_from_disk(
tcx: TyCtxt<'tcx>,
- id: SerializedDepNodeIndex
+ id: DepNodeIndex
) -> Option<Self::Value> {
tcx.queries.on_disk_cache.try_load_query_result(tcx, id)
}
diff --git a/src/librustc_query_system/dep_graph/graph.rs b/src/librustc_query_system/dep_graph/graph.rs
index 73983e1644cb0..8871c37b73aba 100644
--- a/src/librustc_query_system/dep_graph/graph.rs
+++ b/src/librustc_query_system/dep_graph/graph.rs
@@ -3,13 +3,13 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_data_structures::profiling::QueryInvocationId;
use rustc_data_structures::sharded::{self, Sharded};
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_data_structures::sync::{AtomicU32, AtomicU64, Lock, Lrc, Ordering};
+use rustc_data_structures::sync::{AtomicCell, AtomicU32, AtomicU64, Lock, Lrc};
use rustc_data_structures::unlikely;
use rustc_errors::Diagnostic;
use rustc_index::vec::{Idx, IndexVec};
use parking_lot::{Condvar, Mutex};
-use smallvec::{smallvec, SmallVec};
+use smallvec::SmallVec;
use std::collections::hash_map::Entry;
use std::env;
use std::hash::Hash;
@@ -20,7 +20,7 @@ use std::sync::atomic::Ordering::Relaxed;
use super::debug::EdgeFilter;
use super::prev::PreviousDepGraph;
use super::query::DepGraphQuery;
-use super::serialized::{SerializedDepGraph, SerializedDepNodeIndex};
+use super::serialized::SerializedDepGraph;
use super::{DepContext, DepKind, DepNode, WorkProductId};
#[derive(Clone)]
@@ -49,17 +49,41 @@ impl std::convert::From<DepNodeIndex> for QueryInvocationId {
}
}
-#[derive(PartialEq)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub enum DepNodeColor {
Red,
- Green(DepNodeIndex),
+ Green,
}
impl DepNodeColor {
pub fn is_green(self) -> bool {
match self {
DepNodeColor::Red => false,
- DepNodeColor::Green(_) => true,
+ DepNodeColor::Green => true,
+ }
+ }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+pub enum DepNodeState {
+ /// The dep node index is invalid and does not refer to any dep node.
+ Invalid,
+
+ /// The node is from the previous session, but its state is unknown
+ Unknown,
+
+ Red,
+
+ Green,
+}
+
+impl DepNodeState {
+ pub fn color(self) -> Option<DepNodeColor> {
+ match self {
+ DepNodeState::Invalid => panic!(),
+ DepNodeState::Unknown => None,
+ DepNodeState::Red => Some(DepNodeColor::Red),
+ DepNodeState::Green => Some(DepNodeColor::Green),
}
}
}
@@ -102,22 +126,37 @@ where
Some(stable_hasher.finish())
}
-impl<K: DepKind> DepGraph<K> {
- pub fn new(
- prev_graph: PreviousDepGraph<K>,
- prev_work_products: FxHashMap<WorkProductId, WorkProduct>,
- ) -> DepGraph<K> {
- let prev_graph_node_count = prev_graph.node_count();
+pub struct DepGraphArgs<K: DepKind> {
+ pub prev_graph: PreviousDepGraph<K>,
+ pub prev_work_products: FxHashMap<WorkProductId, WorkProduct>,
+ pub state: IndexVec<DepNodeIndex, AtomicCell<DepNodeState>>,
+ pub current: CurrentDepGraph<K>,
+}
+
+impl<K: DepKind> Default for DepGraphArgs<K> {
+ fn default() -> Self {
+ let prev_graph = Default::default();
+ Self {
+ current: CurrentDepGraph::new(&prev_graph),
+ prev_work_products: Default::default(),
+ state: Default::default(),
+ prev_graph,
+ }
+ }
+}
+impl<K: DepKind> DepGraph<K> {
+ pub fn new(args: DepGraphArgs<K>) -> Self {
+ let colors = DepNodeColorMap { values: args.state };
DepGraph {
data: Some(Lrc::new(DepGraphData {
- previous_work_products: prev_work_products,
+ previous_work_products: args.prev_work_products,
dep_node_debug: Default::default(),
- current: CurrentDepGraph::new(prev_graph_node_count),
+ current: args.current,
emitting_diagnostics: Default::default(),
emitting_diagnostics_cond_var: Condvar::new(),
- previous: prev_graph,
- colors: DepNodeColorMap::new(prev_graph_node_count),
+ colors,
+ previous: args.prev_graph,
})),
virtual_dep_node_index: Lrc::new(AtomicU32::new(0)),
}
@@ -134,12 +173,18 @@ impl<K: DepKind> DepGraph<K> {
}
pub fn query(&self) -> DepGraphQuery<K> {
- let data = self.data.as_ref().unwrap().current.data.lock();
- let nodes: Vec<_> = data.iter().map(|n| n.node).collect();
+ let data = self.data.as_ref().unwrap();
+ let node_data = data.current.data.lock();
+
+ // Recreate the dep graph, but ignore nodes from the previous dep graph
+ let current_nodes =
+ || node_data.iter_enumerated().filter(|&(i, _)| data.current(i)).map(|(_, n)| n);
+ let nodes: Vec<_> = current_nodes().map(|n| n.node).collect();
let mut edges = Vec::new();
- for (from, edge_targets) in data.iter().map(|d| (d.node, &d.edges)) {
+ for (from, edge_targets) in current_nodes().map(|d| (d.node, &d.edges)) {
for &edge_target in edge_targets.iter() {
- let to = data[edge_target].node;
+ assert!(data.current(edge_target));
+ let to = node_data[edge_target].node;
edges.push((from, to));
}
}
@@ -212,7 +257,6 @@ impl<K: DepKind> DepGraph<K> {
phantom_data: PhantomData,
})
},
- |data, key, fingerprint, task| data.complete_task(key, task.unwrap(), fingerprint),
hash_result,
)
}
@@ -225,12 +269,6 @@ impl<K: DepKind> DepGraph<K> {
no_tcx: bool,
task: fn(Ctxt, A) -> R,
create_task: fn(DepNode<K>) -> Option<TaskDeps<K>>,
- finish_task_and_alloc_depnode: fn(
- &CurrentDepGraph<K>,
- DepNode<K>,
- Fingerprint,
- Option<TaskDeps<K>>,
- ) -> DepNodeIndex,
hash_result: impl FnOnce(&mut Ctxt::StableHashingContext, &R) -> Option<Fingerprint>,
) -> (R, DepNodeIndex) {
if let Some(ref data) = self.data {
@@ -251,52 +289,54 @@ impl<K: DepKind> DepGraph<K> {
let current_fingerprint = hash_result(&mut hcx, &result);
- let dep_node_index = finish_task_and_alloc_depnode(
- &data.current,
- key,
- current_fingerprint.unwrap_or(Fingerprint::ZERO),
- task_deps.map(|lock| lock.into_inner()),
- );
+ let deps = task_deps.map(|lock| lock.into_inner().reads).unwrap_or(SmallVec::new());
+ let fingerprint = current_fingerprint.unwrap_or(Fingerprint::ZERO);
let print_status = cfg!(debug_assertions) && cx.debug_dep_tasks();
- // Determine the color of the new DepNode.
- if let Some(prev_index) = data.previous.node_to_index_opt(&key) {
+ let dep_node_index = if let Some(prev_index) = data.previous.node_to_index_opt(&key) {
let prev_fingerprint = data.previous.fingerprint_by_index(prev_index);
+ // Determine the color of the DepNode.
let color = if let Some(current_fingerprint) = current_fingerprint {
if current_fingerprint == prev_fingerprint {
if print_status {
eprintln!("[task::green] {:?}", key);
}
- DepNodeColor::Green(dep_node_index)
+ DepNodeState::Green
} else {
if print_status {
eprintln!("[task::red] {:?}", key);
}
- DepNodeColor::Red
+ DepNodeState::Red
}
} else {
if print_status {
eprintln!("[task::unknown] {:?}", key);
}
// Mark the node as Red if we can't hash the result
- DepNodeColor::Red
+ DepNodeState::Red
};
- debug_assert!(
- data.colors.get(prev_index).is_none(),
- "DepGraph::with_task() - Duplicate DepNodeColor \
- insertion for {:?}",
+ debug_assert_eq!(
+ data.colors.get(prev_index).color(),
+ None,
+ "DepGraph::with_task() - Duplicate DepNodeState insertion for {:?}",
key
);
data.colors.insert(prev_index, color);
+
+ data.current.update_node(prev_index, key, deps, fingerprint);
+
+ prev_index
} else {
if print_status {
eprintln!("[task::new] {:?}", key);
}
- }
+
+ data.current.new_node(key, deps, fingerprint, &data.previous)
+ };
(result, dep_node_index)
} else {
@@ -333,16 +373,7 @@ impl<K: DepKind> DepGraph<K> {
task: fn(Ctxt, A) -> R,
hash_result: impl FnOnce(&mut Ctxt::StableHashingContext, &R) -> Option<Fingerprint>,
) -> (R, DepNodeIndex) {
- self.with_task_impl(
- key,
- cx,
- arg,
- false,
- task,
- |_| None,
- |data, key, fingerprint, _| data.alloc_node(key, smallvec![], fingerprint),
- hash_result,
- )
+ self.with_task_impl(key, cx, arg, false, task, |_| None, hash_result)
}
#[inline]
@@ -386,7 +417,8 @@ impl<K: DepKind> DepGraph<K> {
.node_to_node_index
.get_shard_by_value(&dep_node)
.lock()
- .contains_key(dep_node)
+ .get(dep_node)
+ .map_or(false, |index| data.current(*index))
} else {
false
}
@@ -394,19 +426,15 @@ impl<K: DepKind> DepGraph<K> {
#[inline]
pub fn fingerprint_of(&self, dep_node_index: DepNodeIndex) -> Fingerprint {
- let data = self.data.as_ref().expect("dep graph enabled").current.data.lock();
- data[dep_node_index].fingerprint
+ let data = self.data.as_ref().expect("dep graph enabled");
+ assert!(data.current(dep_node_index));
+ data.current.data.lock()[dep_node_index].fingerprint
}
pub fn prev_fingerprint_of(&self, dep_node: &DepNode<K>) -> Option<Fingerprint> {
self.data.as_ref().unwrap().previous.fingerprint_of(dep_node)
}
- #[inline]
- pub fn prev_dep_node_index_of(&self, dep_node: &DepNode<K>) -> SerializedDepNodeIndex {
- self.data.as_ref().unwrap().previous.node_to_index(dep_node)
- }
-
/// Checks whether a previous work product exists for `v` and, if
/// so, return the path that leads to it. Used to skip doing work.
pub fn previous_work_product(&self, v: &WorkProductId) -> Option<WorkProduct> {
@@ -451,11 +479,39 @@ impl<K: DepKind> DepGraph<K> {
}
pub fn serialize(&self) -> SerializedDepGraph<K> {
- let data = self.data.as_ref().unwrap().current.data.lock();
+ let data = self.data.as_ref().unwrap();
+ let colors = &data.colors;
+ let data = data.current.data.lock();
+
+ let fingerprints: IndexVec<DepNodeIndex, _> = data.iter().map(|d| d.fingerprint).collect();
+ let mut nodes: IndexVec<DepNodeIndex, _> = data.iter().map(|d| d.node).collect();
+
+ // Invalidate dep nodes with unknown state as these cannot safely
+ // be marked green in the next session. One of the dependencies of the
+ // unknown node may have changed in this session (and is currently marked red),
+ // but might be green again in the next session, which may cause the unknown node
+ // to incorrectly be marked green in the next session, even though one of its dependencies
+ // did actually change.
+ for index in colors.values.indices() {
+ match colors.get(index) {
+ // In order to this invalidation to be safe, none of the valid nodes can
+ // point to unknown nodes.
+ DepNodeState::Unknown => {
+ // Change the node kind to Null so we know this node is invalidated when
+ // we load the dep graph
+ nodes[index] = DepNode::new_no_params(DepKind::NULL);
+ // FIXME: Also clear edges
+ }
- let fingerprints: IndexVec<SerializedDepNodeIndex, _> =
- data.iter().map(|d| d.fingerprint).collect();
- let nodes: IndexVec<SerializedDepNodeIndex, _> = data.iter().map(|d| d.node).collect();
+ // For green nodes, we either executed the query (which always uses valid nodes)
+ // or we marked it as green because all its dependencies are green and valid.
+ DepNodeState::Green |
+ // Red nodes were always exexuted.
+ DepNodeState::Red |
+ // We don't need to invalidate already invalid nodes
+ DepNodeState::Invalid => {},
+ }
+ }
let total_edge_count: usize = data.iter().map(|d| d.edges.len()).sum();
@@ -465,7 +521,7 @@ impl<K: DepKind> DepGraph<K> {
for (current_dep_node_index, edges) in data.iter_enumerated().map(|(i, d)| (i, &d.edges)) {
let start = edge_list_data.len() as u32;
// This should really just be a memcpy :/
- edge_list_data.extend(edges.iter().map(|i| SerializedDepNodeIndex::new(i.index())));
+ edge_list_data.extend(edges.iter().map(|i| DepNodeIndex::new(i.index())));
let end = edge_list_data.len() as u32;
debug_assert_eq!(current_dep_node_index.index(), edge_list_indices.len());
@@ -481,7 +537,7 @@ impl<K: DepKind> DepGraph<K> {
pub fn node_color(&self, dep_node: &DepNode<K>) -> Option<DepNodeColor> {
if let Some(ref data) = self.data {
if let Some(prev_index) = data.previous.node_to_index_opt(dep_node) {
- return data.colors.get(prev_index);
+ return data.colors.get(prev_index).color();
} else {
// This is a node that did not exist in the previous compilation
// session, so we consider it to be red.
@@ -500,11 +556,11 @@ impl<K: DepKind> DepGraph<K> {
&self,
tcx: Ctxt,
dep_node: &DepNode<K>,
- ) -> Option<(SerializedDepNodeIndex, DepNodeIndex)> {
- self.try_mark_green(tcx, dep_node).map(|(prev_index, dep_node_index)| {
+ ) -> Option<DepNodeIndex> {
+ self.try_mark_green(tcx, dep_node).map(|prev_index| {
debug_assert!(self.is_green(&dep_node));
- self.read_index(dep_node_index);
- (prev_index, dep_node_index)
+ self.read_index(prev_index);
+ prev_index
})
}
@@ -512,7 +568,7 @@ impl<K: DepKind> DepGraph<K> {
&self,
tcx: Ctxt,
dep_node: &DepNode<K>,
- ) -> Option<(SerializedDepNodeIndex, DepNodeIndex)> {
+ ) -> Option<DepNodeIndex> {
debug_assert!(!dep_node.kind.is_eval_always());
// Return None if the dep graph is disabled
@@ -522,16 +578,80 @@ impl<K: DepKind> DepGraph<K> {
let prev_index = data.previous.node_to_index_opt(dep_node)?;
match data.colors.get(prev_index) {
- Some(DepNodeColor::Green(dep_node_index)) => Some((prev_index, dep_node_index)),
- Some(DepNodeColor::Red) => None,
- None => {
+ DepNodeState::Invalid => panic!(),
+ DepNodeState::Green => Some(prev_index),
+ DepNodeState::Red => None,
+ DepNodeState::Unknown => {
// This DepNode and the corresponding query invocation existed
// in the previous compilation session too, so we can try to
// mark it as green by recursively marking all of its
// dependencies green.
- self.try_mark_previous_green(tcx, data, prev_index, &dep_node)
- .map(|dep_node_index| (prev_index, dep_node_index))
+ if self.try_mark_previous_green(tcx, data, prev_index, &dep_node) {
+ Some(prev_index)
+ } else {
+ None
+ }
+ }
+ }
+ }
+
+ /// Try to force a dep node to execute and see if it's green
+ fn try_force_previous_green<Ctxt: DepContext<DepKind = K>>(
+ &self,
+ tcx: Ctxt,
+ data: &DepGraphData<K>,
+ dep_node_index: DepNodeIndex,
+ ) -> bool {
+ let dep_node = &data.previous.index_to_node(dep_node_index);
+
+ debug!("try_force_previous_green({:?}) --- trying to force", dep_node);
+ if tcx.try_force_from_dep_node(dep_node) {
+ match data.colors.get(dep_node_index) {
+ DepNodeState::Green => {
+ debug!(
+ "try_force_previous_green({:?}) --- managed to \
+ FORCE to green",
+ dep_node
+ );
+ true
+ }
+ DepNodeState::Red => {
+ debug!(
+ "try_force_previous_green({:?}) - END - was red after forcing",
+ dep_node
+ );
+ false
+ }
+ DepNodeState::Invalid => panic!(),
+ DepNodeState::Unknown => {
+ if !tcx.has_errors_or_delayed_span_bugs() {
+ panic!(
+ "try_force_previous_green() - Forcing the DepNode \
+ should have set its color"
+ )
+ } else {
+ // If the query we just forced has resulted in
+ // some kind of compilation error, we cannot rely on
+ // the dep-node color having been properly updated.
+ // This means that the query system has reached an
+ // invalid state. We let the compiler continue (by
+ // returning `None`) so it can emit error messages
+ // and wind down, but rely on the fact that this
+ // invalid state will not be persisted to the
+ // incremental compilation cache because of
+ // compilation errors being present.
+ debug!(
+ "try_force_previous_green({:?}) resulted in compilation error",
+ dep_node
+ );
+ false
+ }
+ }
}
+ } else {
+ // The DepNode could not be forced.
+ debug!("try_force_previous_green({:?}) - END - could not be forced", dep_node);
+ false
}
}
@@ -540,38 +660,28 @@ impl<K: DepKind> DepGraph<K> {
&self,
tcx: Ctxt,
data: &DepGraphData<K>,
- prev_dep_node_index: SerializedDepNodeIndex,
+ dep_node_index: DepNodeIndex,
+ // FIXME: Remove this, only used in debug statements
dep_node: &DepNode<K>,
- ) -> Option<DepNodeIndex> {
+ ) -> bool {
debug!("try_mark_previous_green({:?}) - BEGIN", dep_node);
#[cfg(not(parallel_compiler))]
- {
- debug_assert!(
- !data
- .current
- .node_to_node_index
- .get_shard_by_value(dep_node)
- .lock()
- .contains_key(dep_node)
- );
- debug_assert!(data.colors.get(prev_dep_node_index).is_none());
- }
+ debug_assert!(data.colors.get(dep_node_index).color().is_none());
// We never try to mark eval_always nodes as green
debug_assert!(!dep_node.kind.is_eval_always());
- debug_assert_eq!(data.previous.index_to_node(prev_dep_node_index), *dep_node);
+ debug_assert_eq!(data.previous.index_to_node(dep_node_index), *dep_node);
- let prev_deps = data.previous.edge_targets_from(prev_dep_node_index);
-
- let mut current_deps = SmallVec::new();
+ let prev_deps = data.previous.edge_targets_from(dep_node_index);
for &dep_dep_node_index in prev_deps {
let dep_dep_node_color = data.colors.get(dep_dep_node_index);
match dep_dep_node_color {
- Some(DepNodeColor::Green(node_index)) => {
+ DepNodeState::Invalid => panic!(),
+ DepNodeState::Green => {
// This dependency has been marked as green before, we are
// still fine and can continue with checking the other
// dependencies.
@@ -581,9 +691,8 @@ impl<K: DepKind> DepGraph<K> {
dep_node,
data.previous.index_to_node(dep_dep_node_index)
);
- current_deps.push(node_index);
}
- Some(DepNodeColor::Red) => {
+ DepNodeState::Red => {
// We found a dependency the value of which has changed
// compared to the previous compilation session. We cannot
// mark the DepNode as green and also don't need to bother
@@ -594,97 +703,31 @@ impl<K: DepKind> DepGraph<K> {
dep_node,
data.previous.index_to_node(dep_dep_node_index)
);
- return None;
+ return false;
}
- None => {
+ DepNodeState::Unknown => {
let dep_dep_node = &data.previous.index_to_node(dep_dep_node_index);
// We don't know the state of this dependency. If it isn't
// an eval_always node, let's try to mark it green recursively.
if !dep_dep_node.kind.is_eval_always() {
- debug!(
- "try_mark_previous_green({:?}) --- state of dependency {:?} \
- is unknown, trying to mark it green",
- dep_node, dep_dep_node
- );
-
- let node_index = self.try_mark_previous_green(
- tcx,
- data,
- dep_dep_node_index,
- dep_dep_node,
- );
- if let Some(node_index) = node_index {
+ // We don't know the state of this dependency.
+ // We known it is not an eval_always node, since those get marked as `Invalid`.
+ // Let's try to mark it green recursively.
+ if self.try_mark_previous_green(tcx, data, dep_dep_node_index, dep_dep_node)
+ {
debug!(
"try_mark_previous_green({:?}) --- managed to MARK \
- dependency {:?} as green",
+ dependency {:?} as green",
dep_node, dep_dep_node
);
- current_deps.push(node_index);
continue;
}
}
// We failed to mark it green, so we try to force the query.
- debug!(
- "try_mark_previous_green({:?}) --- trying to force \
- dependency {:?}",
- dep_node, dep_dep_node
- );
- if tcx.try_force_from_dep_node(dep_dep_node) {
- let dep_dep_node_color = data.colors.get(dep_dep_node_index);
-
- match dep_dep_node_color {
- Some(DepNodeColor::Green(node_index)) => {
- debug!(
- "try_mark_previous_green({:?}) --- managed to \
- FORCE dependency {:?} to green",
- dep_node, dep_dep_node
- );
- current_deps.push(node_index);
- }
- Some(DepNodeColor::Red) => {
- debug!(
- "try_mark_previous_green({:?}) - END - \
- dependency {:?} was red after forcing",
- dep_node, dep_dep_node
- );
- return None;
- }
- None => {
- if !tcx.has_errors_or_delayed_span_bugs() {
- panic!(
- "try_mark_previous_green() - Forcing the DepNode \
- should have set its color"
- )
- } else {
- // If the query we just forced has resulted in
- // some kind of compilation error, we cannot rely on
- // the dep-node color having been properly updated.
- // This means that the query system has reached an
- // invalid state. We let the compiler continue (by
- // returning `None`) so it can emit error messages
- // and wind down, but rely on the fact that this
- // invalid state will not be persisted to the
- // incremental compilation cache because of
- // compilation errors being present.
- debug!(
- "try_mark_previous_green({:?}) - END - \
- dependency {:?} resulted in compilation error",
- dep_node, dep_dep_node
- );
- return None;
- }
- }
- }
- } else {
- // The DepNode could not be forced.
- debug!(
- "try_mark_previous_green({:?}) - END - dependency {:?} \
- could not be forced",
- dep_node, dep_dep_node
- );
- return None;
+ if !self.try_force_previous_green(tcx, data, dep_dep_node_index) {
+ return false;
}
}
}
@@ -696,40 +739,42 @@ impl<K: DepKind> DepGraph<K> {
// There may be multiple threads trying to mark the same dep node green concurrently
- let dep_node_index = {
- // Copy the fingerprint from the previous graph,
- // so we don't have to recompute it
- let fingerprint = data.previous.fingerprint_by_index(prev_dep_node_index);
-
- // We allocating an entry for the node in the current dependency graph and
- // adding all the appropriate edges imported from the previous graph
- data.current.intern_node(*dep_node, current_deps, fingerprint)
- };
+ #[cfg(not(parallel_compiler))]
+ debug_assert_eq!(
+ data.colors.get(dep_node_index).color(),
+ None,
+ "DepGraph::try_mark_previous_green() - Duplicate DepNodeState \
+ insertion for {:?}",
+ dep_node
+ );
// ... emitting any stored diagnostic ...
// FIXME: Store the fact that a node has diagnostics in a bit in the dep graph somewhere
// Maybe store a list on disk and encode this fact in the DepNodeState
- let diagnostics = tcx.load_diagnostics(prev_dep_node_index);
+ let diagnostics = tcx.load_diagnostics(dep_node_index);
#[cfg(not(parallel_compiler))]
- debug_assert!(
- data.colors.get(prev_dep_node_index).is_none(),
+ debug_assert_eq!(
+ data.colors.get(dep_node_index).color(),
+ None,
"DepGraph::try_mark_previous_green() - Duplicate DepNodeColor \
- insertion for {:?}",
+ insertion for {:?}",
dep_node
);
if unlikely!(!diagnostics.is_empty()) {
- self.emit_diagnostics(tcx, data, dep_node_index, prev_dep_node_index, diagnostics);
+ self.emit_diagnostics(tcx, data, dep_node_index, diagnostics);
}
// ... and finally storing a "Green" entry in the color map.
// Multiple threads can all write the same color here
- data.colors.insert(prev_dep_node_index, DepNodeColor::Green(dep_node_index));
+
+ data.colors.insert(dep_node_index, DepNodeState::Green);
debug!("try_mark_previous_green({:?}) - END - successfully marked as green", dep_node);
- Some(dep_node_index)
+
+ true
}
/// Atomically emits some loaded diagnostics.
@@ -741,12 +786,11 @@ impl<K: DepKind> DepGraph<K> {
tcx: Ctxt,
data: &DepGraphData<K>,
dep_node_index: DepNodeIndex,
- prev_dep_node_index: SerializedDepNodeIndex,
diagnostics: Vec<Diagnostic>,
) {
let mut emitting = data.emitting_diagnostics.lock();
- if data.colors.get(prev_dep_node_index) == Some(DepNodeColor::Green(dep_node_index)) {
+ if data.colors.get(dep_node_index) == DepNodeState::Green {
// The node is already green so diagnostics must have been emitted already
return;
}
@@ -767,7 +811,7 @@ impl<K: DepKind> DepGraph<K> {
}
// Mark the node as green now that diagnostics are emitted
- data.colors.insert(prev_dep_node_index, DepNodeColor::Green(dep_node_index));
+ data.colors.insert(dep_node_index, DepNodeState::Green);
// Remove the node from the set
data.emitting_diagnostics.lock().remove(&dep_node_index);
@@ -779,8 +823,7 @@ impl<K: DepKind> DepGraph<K> {
loop {
data.emitting_diagnostics_cond_var.wait(&mut emitting);
- if data.colors.get(prev_dep_node_index) == Some(DepNodeColor::Green(dep_node_index))
- {
+ if data.colors.get(dep_node_index) == DepNodeState::Green {
break;
}
}
@@ -807,11 +850,19 @@ impl<K: DepKind> DepGraph<K> {
let data = self.data.as_ref().unwrap();
for prev_index in data.colors.values.indices() {
match data.colors.get(prev_index) {
- Some(DepNodeColor::Green(_)) => {
+ DepNodeState::Green => {
let dep_node = data.previous.index_to_node(prev_index);
tcx.try_load_from_on_disk_cache(&dep_node);
}
- None | Some(DepNodeColor::Red) => {
+
+ // There cannot be results stored for invalid indices.
+ DepNodeState::Invalid |
+
+ // Unknown nodes are unused, so we don't want to promote these and we would
+ // not to mark their colors in order to do so anyway.
+ DepNodeState::Unknown |
+
+ DepNodeState::Red => {
// We can skip red nodes because a node can only be marked
// as red if the query result was recomputed and thus is
// already in memory.
@@ -893,7 +944,7 @@ struct DepNodeData<K> {
/// The only operation that must manipulate both locks is adding new nodes, in which case
/// we first acquire the `node_to_node_index` lock and then, once a new node is to be inserted,
/// acquire the lock on `data.`
-pub(super) struct CurrentDepGraph<K> {
+pub struct CurrentDepGraph<K> {
data: Lock<IndexVec<DepNodeIndex, DepNodeData<K>>>,
node_to_node_index: Sharded<FxHashMap<DepNode<K>, DepNodeIndex>>,
@@ -922,7 +973,7 @@ pub(super) struct CurrentDepGraph<K> {
}
impl<K: DepKind> CurrentDepGraph<K> {
- fn new(prev_graph_node_count: usize) -> CurrentDepGraph<K> {
+ pub fn new(prev_graph: &PreviousDepGraph<K>) -> Self {
use std::time::{SystemTime, UNIX_EPOCH};
let duration = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
@@ -946,11 +997,11 @@ impl<K: DepKind> CurrentDepGraph<K> {
// that we hopefully don't have to re-allocate during this compilation
// session. The over-allocation is 2% plus a small constant to account
// for the fact that in very small crates 2% might not be enough.
- let new_node_count_estimate = (prev_graph_node_count * 102) / 100 + 200;
+ let new_node_count_estimate = (prev_graph.node_count() * 102) / 100 + 200;
- CurrentDepGraph {
+ let mut current = CurrentDepGraph {
data: Lock::new(IndexVec::with_capacity(new_node_count_estimate)),
- node_to_node_index: Sharded::new(|| {
+ node_to_node_index: Sharded::new(|_| {
FxHashMap::with_capacity_and_hasher(
new_node_count_estimate / sharded::SHARDS,
Default::default(),
@@ -960,16 +1011,23 @@ impl<K: DepKind> CurrentDepGraph<K> {
forbidden_edge,
total_read_count: AtomicU64::new(0),
total_duplicate_read_count: AtomicU64::new(0),
+ };
+
+ // Fill in with the nodes from the previous session
+ for (idx, &node) in prev_graph.data.nodes.iter_enumerated() {
+ current.data.get_mut().push(DepNodeData {
+ node,
+ fingerprint: prev_graph.fingerprint_by_index(idx),
+ edges: prev_graph.edge_targets_from(idx).iter().copied().collect(),
+ });
}
- }
- fn complete_task(
- &self,
- node: DepNode<K>,
- task_deps: TaskDeps<K>,
- fingerprint: Fingerprint,
- ) -> DepNodeIndex {
- self.alloc_node(node, task_deps.reads, fingerprint)
+ // Fill in with the node to index map from the previous session
+ for (node, idx) in prev_graph.index.iter() {
+ current.node_to_node_index.get_shard_by_value_mut(node).insert(*node, *idx);
+ }
+
+ current
}
fn complete_anon_task(&self, kind: K, task_deps: TaskDeps<K>) -> DepNodeIndex {
@@ -996,18 +1054,31 @@ impl<K: DepKind> CurrentDepGraph<K> {
self.intern_node(target_dep_node, task_deps.reads, Fingerprint::ZERO)
}
- fn alloc_node(
+ fn new_node(
&self,
dep_node: DepNode<K>,
edges: EdgesVec,
fingerprint: Fingerprint,
+ previous: &PreviousDepGraph<K>,
) -> DepNodeIndex {
+ debug_assert!(previous.node_to_index_opt(&dep_node).is_none());
debug_assert!(
!self.node_to_node_index.get_shard_by_value(&dep_node).lock().contains_key(&dep_node)
);
self.intern_node(dep_node, edges, fingerprint)
}
+ fn update_node(
+ &self,
+ index: DepNodeIndex,
+ dep_node: DepNode<K>,
+ edges: SmallVec<[DepNodeIndex; 8]>,
+ fingerprint: Fingerprint,
+ ) {
+ // Update the edges and fingerprint for this dep node
+ self.data.lock()[index] = DepNodeData { node: dep_node, edges, fingerprint };
+ }
+
fn intern_node(
&self,
dep_node: DepNode<K>,
@@ -1028,6 +1099,16 @@ impl<K: DepKind> CurrentDepGraph<K> {
}
impl<K: DepKind> DepGraphData<K> {
+ fn current(&self, dep_node_index: DepNodeIndex) -> bool {
+ if dep_node_index.as_usize() < self.previous.node_count() {
+ // From this session if it's colored
+ self.colors.get(dep_node_index).color().is_some()
+ } else {
+ // From this session
+ true
+ }
+ }
+
#[inline(never)]
fn read_index(&self, source: DepNodeIndex) {
K::read_deps(|task_deps| {
@@ -1097,38 +1178,18 @@ impl<K> Default for TaskDeps<K> {
}
}
-// A data structure that stores Option<DepNodeColor> values as a contiguous
-// array, using one u32 per entry.
struct DepNodeColorMap {
- values: IndexVec<SerializedDepNodeIndex, AtomicU32>,
+ values: IndexVec<DepNodeIndex, AtomicCell<DepNodeState>>,
}
-const COMPRESSED_NONE: u32 = 0;
-const COMPRESSED_RED: u32 = 1;
-const COMPRESSED_FIRST_GREEN: u32 = 2;
-
impl DepNodeColorMap {
- fn new(size: usize) -> DepNodeColorMap {
- DepNodeColorMap { values: (0..size).map(|_| AtomicU32::new(COMPRESSED_NONE)).collect() }
- }
-
- fn get(&self, index: SerializedDepNodeIndex) -> Option<DepNodeColor> {
- match self.values[index].load(Ordering::Acquire) {
- COMPRESSED_NONE => None,
- COMPRESSED_RED => Some(DepNodeColor::Red),
- value => {
- Some(DepNodeColor::Green(DepNodeIndex::from_u32(value - COMPRESSED_FIRST_GREEN)))
- }
- }
+ #[inline]
+ fn get(&self, index: DepNodeIndex) -> DepNodeState {
+ self.values[index].load()
}
- fn insert(&self, index: SerializedDepNodeIndex, color: DepNodeColor) {
- self.values[index].store(
- match color {
- DepNodeColor::Red => COMPRESSED_RED,
- DepNodeColor::Green(index) => index.as_u32() + COMPRESSED_FIRST_GREEN,
- },
- Ordering::Release,
- )
+ #[inline]
+ fn insert(&self, index: DepNodeIndex, state: DepNodeState) {
+ self.values[index].store(state)
}
}
diff --git a/src/librustc_query_system/dep_graph/mod.rs b/src/librustc_query_system/dep_graph/mod.rs
index fbc91575ede41..0dff6bd62375e 100644
--- a/src/librustc_query_system/dep_graph/mod.rs
+++ b/src/librustc_query_system/dep_graph/mod.rs
@@ -7,10 +7,13 @@ mod serialized;
pub use dep_node::{DepNode, DepNodeParams, WorkProductId};
pub use graph::WorkProductFileKind;
-pub use graph::{hash_result, DepGraph, DepNodeColor, DepNodeIndex, TaskDeps, WorkProduct};
+pub use graph::{
+ hash_result, CurrentDepGraph, DepGraph, DepGraphArgs, DepNodeColor, DepNodeIndex, DepNodeState,
+ TaskDeps, WorkProduct,
+};
pub use prev::PreviousDepGraph;
pub use query::DepGraphQuery;
-pub use serialized::{SerializedDepGraph, SerializedDepNodeIndex};
+pub use serialized::SerializedDepGraph;
use rustc_data_structures::profiling::SelfProfilerRef;
use rustc_data_structures::sync::Lock;
@@ -42,7 +45,7 @@ pub trait DepContext: Copy {
fn try_load_from_on_disk_cache(&self, dep_node: &DepNode<Self::DepKind>);
/// Load diagnostics associated to the node in the previous session.
- fn load_diagnostics(&self, prev_dep_node_index: SerializedDepNodeIndex) -> Vec<Diagnostic>;
+ fn load_diagnostics(&self, dep_node_index: DepNodeIndex) -> Vec<Diagnostic>;
/// Register diagnostics for the given node, for use in next session.
fn store_diagnostics(&self, dep_node_index: DepNodeIndex, diagnostics: ThinVec<Diagnostic>);
@@ -60,6 +63,7 @@ pub trait DepContext: Copy {
/// Describe the different families of dependency nodes.
pub trait DepKind: Copy + fmt::Debug + Eq + Ord + Hash {
+ /// An unused dep kind.
const NULL: Self;
/// Return whether this kind always require evaluation.
diff --git a/src/librustc_query_system/dep_graph/prev.rs b/src/librustc_query_system/dep_graph/prev.rs
index 5cba64cac4b34..9a6ec3ee48470 100644
--- a/src/librustc_query_system/dep_graph/prev.rs
+++ b/src/librustc_query_system/dep_graph/prev.rs
@@ -1,47 +1,73 @@
-use super::serialized::{SerializedDepGraph, SerializedDepNodeIndex};
-use super::{DepKind, DepNode};
+use super::serialized::SerializedDepGraph;
+use super::{DepKind, DepNode, DepNodeIndex, DepNodeState};
use rustc_data_structures::fingerprint::Fingerprint;
use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::sync::AtomicCell;
+use rustc_index::vec::IndexVec;
-#[derive(Debug, RustcEncodable, RustcDecodable)]
+#[derive(Debug)]
pub struct PreviousDepGraph<K: DepKind> {
- data: SerializedDepGraph<K>,
- index: FxHashMap<DepNode<K>, SerializedDepNodeIndex>,
+ pub(super) data: SerializedDepGraph<K>,
+ pub(super) index: FxHashMap<DepNode<K>, DepNodeIndex>,
+ pub(super) unused: Vec<DepNodeIndex>,
}
impl<K: DepKind> Default for PreviousDepGraph<K> {
fn default() -> Self {
- PreviousDepGraph { data: Default::default(), index: Default::default() }
+ Self { data: Default::default(), index: Default::default(), unused: Default::default() }
}
}
impl<K: DepKind> PreviousDepGraph<K> {
- pub fn new(data: SerializedDepGraph<K>) -> PreviousDepGraph<K> {
+ pub fn new_and_state(
+ data: SerializedDepGraph<K>,
+ ) -> (Self, IndexVec<DepNodeIndex, AtomicCell<DepNodeState>>) {
+ let mut unused = Vec::new();
+
+ let state: IndexVec<_, _> = data
+ .nodes
+ .iter_enumerated()
+ .map(|(index, node)| {
+ if node.kind == DepKind::NULL {
+ // There might be `DepKind::NULL` nodes due to thread-local dep node indices
+ // that didn't get assigned anything.
+ // We also changed outdated nodes to `DepKind::NULL`.
+ unused.push(index);
+ AtomicCell::new(DepNodeState::Invalid)
+ } else {
+ AtomicCell::new(DepNodeState::Unknown)
+ }
+ })
+ .collect();
+
let index: FxHashMap<_, _> =
- data.nodes.iter_enumerated().map(|(idx, &dep_node)| (dep_node, idx)).collect();
- PreviousDepGraph { data, index }
+ data.nodes
+ .iter_enumerated()
+ .filter_map(|(idx, &dep_node)| {
+ if dep_node.kind == DepKind::NULL { None } else { Some((dep_node, idx)) }
+ })
+ .collect();
+
+ (PreviousDepGraph { data, index, unused }, state)
}
#[inline]
- pub fn edge_targets_from(
- &self,
- dep_node_index: SerializedDepNodeIndex,
- ) -> &[SerializedDepNodeIndex] {
+ pub fn edge_targets_from(&self, dep_node_index: DepNodeIndex) -> &[DepNodeIndex] {
self.data.edge_targets_from(dep_node_index)
}
#[inline]
- pub fn index_to_node(&self, dep_node_index: SerializedDepNodeIndex) -> DepNode<K> {
+ pub fn index_to_node(&self, dep_node_index: DepNodeIndex) -> DepNode<K> {
self.data.nodes[dep_node_index]
}
#[inline]
- pub fn node_to_index(&self, dep_node: &DepNode<K>) -> SerializedDepNodeIndex {
+ pub fn node_to_index(&self, dep_node: &DepNode<K>) -> DepNodeIndex {
self.index[dep_node]
}
#[inline]
- pub fn node_to_index_opt(&self, dep_node: &DepNode<K>) -> Option<SerializedDepNodeIndex> {
+ pub fn node_to_index_opt(&self, dep_node: &DepNode<K>) -> Option<DepNodeIndex> {
self.index.get(dep_node).cloned()
}
@@ -51,11 +77,11 @@ impl<K: DepKind> PreviousDepGraph<K> {
}
#[inline]
- pub fn fingerprint_by_index(&self, dep_node_index: SerializedDepNodeIndex) -> Fingerprint {
+ pub fn fingerprint_by_index(&self, dep_node_index: DepNodeIndex) -> Fingerprint {
self.data.fingerprints[dep_node_index]
}
pub fn node_count(&self) -> usize {
- self.index.len()
+ self.data.nodes.len()
}
}
diff --git a/src/librustc_query_system/dep_graph/serialized.rs b/src/librustc_query_system/dep_graph/serialized.rs
index 4a89da23ea6a5..ffe11230ae6b2 100644
--- a/src/librustc_query_system/dep_graph/serialized.rs
+++ b/src/librustc_query_system/dep_graph/serialized.rs
@@ -1,28 +1,24 @@
//! The data that we will serialize and deserialize.
-use super::{DepKind, DepNode};
+use super::{DepKind, DepNode, DepNodeIndex};
use rustc_data_structures::fingerprint::Fingerprint;
use rustc_index::vec::IndexVec;
-rustc_index::newtype_index! {
- pub struct SerializedDepNodeIndex { .. }
-}
-
/// Data for use when recompiling the **current crate**.
#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct SerializedDepGraph<K: DepKind> {
/// The set of all DepNodes in the graph
- pub nodes: IndexVec<SerializedDepNodeIndex, DepNode<K>>,
+ pub nodes: IndexVec<DepNodeIndex, DepNode<K>>,
/// The set of all Fingerprints in the graph. Each Fingerprint corresponds to
/// the DepNode at the same index in the nodes vector.
- pub fingerprints: IndexVec<SerializedDepNodeIndex, Fingerprint>,
+ pub fingerprints: IndexVec<DepNodeIndex, Fingerprint>,
/// For each DepNode, stores the list of edges originating from that
/// DepNode. Encoded as a [start, end) pair indexing into edge_list_data,
/// which holds the actual DepNodeIndices of the target nodes.
- pub edge_list_indices: IndexVec<SerializedDepNodeIndex, (u32, u32)>,
+ pub edge_list_indices: IndexVec<DepNodeIndex, (u32, u32)>,
/// A flattened list of all edge targets in the graph. Edge sources are
/// implicit in edge_list_indices.
- pub edge_list_data: Vec<SerializedDepNodeIndex>,
+ pub edge_list_data: Vec<DepNodeIndex>,
}
impl<K: DepKind> Default for SerializedDepGraph<K> {
@@ -38,7 +34,7 @@ impl<K: DepKind> Default for SerializedDepGraph<K> {
impl<K: DepKind> SerializedDepGraph<K> {
#[inline]
- pub fn edge_targets_from(&self, source: SerializedDepNodeIndex) -> &[SerializedDepNodeIndex] {
+ pub fn edge_targets_from(&self, source: DepNodeIndex) -> &[DepNodeIndex] {
let targets = self.edge_list_indices[source];
&self.edge_list_data[targets.0 as usize..targets.1 as usize]
}
diff --git a/src/librustc_query_system/query/config.rs b/src/librustc_query_system/query/config.rs
index 20dad0bd47ebc..99373035f4c65 100644
--- a/src/librustc_query_system/query/config.rs
+++ b/src/librustc_query_system/query/config.rs
@@ -1,7 +1,7 @@
//! Query configuration and description traits.
use crate::dep_graph::DepNode;
-use crate::dep_graph::SerializedDepNodeIndex;
+use crate::dep_graph::DepNodeIndex;
use crate::query::caches::QueryCache;
use crate::query::plumbing::CycleError;
use crate::query::{QueryContext, QueryState};
@@ -54,7 +54,7 @@ pub trait QueryDescription<CTX: QueryContext>: QueryAccessors<CTX> {
false
}
- fn try_load_from_disk(_: CTX, _: SerializedDepNodeIndex) -> Option<Self::Value> {
+ fn try_load_from_disk(_: CTX, _: DepNodeIndex) -> Option<Self::Value> {
panic!("QueryDescription::load_from_disk() called for an unsupported query.")
}
}
@@ -76,7 +76,7 @@ where
false
}
- default fn try_load_from_disk(_: CTX, _: SerializedDepNodeIndex) -> Option<Self::Value> {
+ default fn try_load_from_disk(_: CTX, _: DepNodeIndex) -> Option<Self::Value> {
panic!("QueryDescription::load_from_disk() called for an unsupported query.")
}
}
diff --git a/src/librustc_query_system/query/plumbing.rs b/src/librustc_query_system/query/plumbing.rs
index b371a914c6fce..c35f59766d880 100644
--- a/src/librustc_query_system/query/plumbing.rs
+++ b/src/librustc_query_system/query/plumbing.rs
@@ -2,8 +2,7 @@
//! generate the actual methods on tcx which find and execute the provider,
//! manage the caches, and so forth.
-use crate::dep_graph::{DepKind, DepNode};
-use crate::dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
+use crate::dep_graph::{DepKind, DepNode, DepNodeIndex};
use crate::query::caches::QueryCache;
use crate::query::config::QueryDescription;
use crate::query::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryShardJobId};
@@ -439,12 +438,11 @@ where
// `try_mark_green()`, so we can ignore them here.
let loaded = tcx.start_query(job.id, None, |tcx| {
let marked = tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node);
- marked.map(|(prev_dep_node_index, dep_node_index)| {
+ marked.map(|dep_node_index| {
(
load_from_disk_and_cache_in_memory::<Q, _>(
tcx,
key.clone(),
- prev_dep_node_index,
dep_node_index,
&dep_node,
),
@@ -466,7 +464,6 @@ where
fn load_from_disk_and_cache_in_memory<Q, CTX>(
tcx: CTX,
key: Q::Key,
- prev_dep_node_index: SerializedDepNodeIndex,
dep_node_index: DepNodeIndex,
dep_node: &DepNode<CTX::DepKind>,
) -> Q::Value
@@ -482,7 +479,7 @@ where
// First we try to load the result from the on-disk cache.
let result = if Q::cache_on_disk(tcx, key.clone(), None) {
let prof_timer = tcx.profiler().incr_cache_loading();
- let result = Q::try_load_from_disk(tcx, prev_dep_node_index);
+ let result = Q::try_load_from_disk(tcx, dep_node_index);
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
// We always expect to find a cached result for things that
@@ -516,7 +513,7 @@ where
// If `-Zincremental-verify-ich` is specified, re-hash results from
// the cache and make sure that they have the expected fingerprint.
if unlikely!(tcx.incremental_verify_ich()) {
- incremental_verify_ich::<Q, _>(tcx, &result, dep_node, dep_node_index);
+ incremental_verify_ich::<Q, _>(tcx, &result, dep_node);
}
result
@@ -524,31 +521,20 @@ where
#[inline(never)]
#[cold]
-fn incremental_verify_ich<Q, CTX>(
- tcx: CTX,
- result: &Q::Value,
- dep_node: &DepNode<CTX::DepKind>,
- dep_node_index: DepNodeIndex,
-) where
+fn incremental_verify_ich<Q, CTX>(tcx: CTX, result: &Q::Value, dep_node: &DepNode<CTX::DepKind>)
+where
CTX: QueryContext,
Q: QueryDescription<CTX>,
{
- assert!(
- Some(tcx.dep_graph().fingerprint_of(dep_node_index))
- == tcx.dep_graph().prev_fingerprint_of(dep_node),
- "fingerprint for green query instance not loaded from cache: {:?}",
- dep_node,
- );
-
debug!("BEGIN verify_ich({:?})", dep_node);
let mut hcx = tcx.create_stable_hashing_context();
let new_hash = Q::hash_result(&mut hcx, result).unwrap_or(Fingerprint::ZERO);
debug!("END verify_ich({:?})", dep_node);
- let old_hash = tcx.dep_graph().fingerprint_of(dep_node_index);
+ let old_hash = tcx.dep_graph().prev_fingerprint_of(dep_node);
- assert!(new_hash == old_hash, "found unstable fingerprints for {:?}", dep_node,);
+ assert!(Some(new_hash) == old_hash, "found unstable fingerprints for {:?}", dep_node,);
}
#[inline(always)]
@@ -659,7 +645,7 @@ where
// in-memory cache, or another query down the line will.
let _ = get_query::<Q, _>(tcx, DUMMY_SP, key);
}
- Some((_, dep_node_index)) => {
+ Some(dep_node_index) => {
tcx.profiler().query_cache_hit(dep_node_index.into());
}
}