From 6119763e199b1cf92a1a43d3511028f67e68986f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Sun, 24 Sep 2023 01:34:45 +0200 Subject: [PATCH 1/2] Encode dep graph edges directly from the previous graph when promoting --- .../rustc_incremental/src/persist/load.rs | 3 +- .../rustc_incremental/src/persist/save.rs | 3 +- .../rustc_query_system/src/dep_graph/graph.rs | 22 +- .../src/dep_graph/serialized.rs | 194 ++++++++++++++---- 4 files changed, 171 insertions(+), 51 deletions(-) diff --git a/compiler/rustc_incremental/src/persist/load.rs b/compiler/rustc_incremental/src/persist/load.rs index 357f2ae92d4..26aaa24771f 100644 --- a/compiler/rustc_incremental/src/persist/load.rs +++ b/compiler/rustc_incremental/src/persist/load.rs @@ -11,6 +11,7 @@ use rustc_session::Session; use rustc_span::ErrorGuaranteed; use std::path::{Path, PathBuf}; +use std::sync::Arc; use super::data::*; use super::file_format; @@ -88,7 +89,7 @@ fn delete_dirty_work_product(sess: &Session, swp: SerializedWorkProduct) { work_product::delete_workproduct_files(sess, &swp.work_product); } -fn load_dep_graph(sess: &Session) -> LoadResult<(SerializedDepGraph, WorkProductMap)> { +fn load_dep_graph(sess: &Session) -> LoadResult<(Arc, WorkProductMap)> { let prof = sess.prof.clone(); if sess.opts.incremental.is_none() { diff --git a/compiler/rustc_incremental/src/persist/save.rs b/compiler/rustc_incremental/src/persist/save.rs index 32759f5284a..9777f769280 100644 --- a/compiler/rustc_incremental/src/persist/save.rs +++ b/compiler/rustc_incremental/src/persist/save.rs @@ -10,6 +10,7 @@ use rustc_serialize::Encodable as RustcEncodable; use rustc_session::Session; use std::fs; +use std::sync::Arc; use super::data::*; use super::dirty_clean; @@ -147,7 +148,7 @@ fn encode_query_cache(tcx: TyCtxt<'_>, encoder: FileEncoder) -> FileEncodeResult /// and moves it to the permanent dep-graph path pub(crate) fn build_dep_graph( sess: &Session, - prev_graph: SerializedDepGraph, + prev_graph: Arc, prev_work_products: WorkProductMap, ) -> Option { if sess.opts.incremental.is_none() { diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index 9f067273f35..4cf8ead8ef8 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -13,6 +13,7 @@ use std::hash::Hash; use std::marker::PhantomData; use std::sync::atomic::Ordering; +use std::sync::Arc; use super::query::DepGraphQuery; use super::serialized::{GraphEncoder, SerializedDepGraph, SerializedDepNodeIndex}; @@ -81,7 +82,7 @@ pub(crate) struct DepGraphData { /// The dep-graph from the previous compilation session. It contains all /// nodes and edges as well as all fingerprints of nodes that have them. - previous: SerializedDepGraph, + previous: Arc, colors: DepNodeColorMap, @@ -113,7 +114,7 @@ pub fn hash_result(hcx: &mut StableHashingContext<'_>, result: &R) -> Fingerp impl DepGraph { pub fn new( profiler: &SelfProfilerRef, - prev_graph: SerializedDepGraph, + prev_graph: Arc, prev_work_products: WorkProductMap, encoder: FileEncoder, record_graph: bool, @@ -127,6 +128,7 @@ pub fn new( encoder, record_graph, record_stats, + prev_graph.clone(), ); let colors = DepNodeColorMap::new(prev_graph_node_count); @@ -1084,6 +1086,7 @@ fn new( encoder: FileEncoder, record_graph: bool, record_stats: bool, + previous: Arc, ) -> Self { use std::time::{SystemTime, UNIX_EPOCH}; @@ -1116,6 +1119,7 @@ fn new( record_graph, record_stats, profiler, + previous, ), new_node_to_index: Sharded::new(|| { FxHashMap::with_capacity_and_hasher( @@ -1236,16 +1240,14 @@ fn promote_node_and_deps_to_current( match prev_index_to_index[prev_index] { Some(dep_node_index) => dep_node_index, None => { - let key = prev_graph.index_to_node(prev_index); - let edges = prev_graph - .edge_targets_from(prev_index) - .map(|i| prev_index_to_index[i].unwrap()) - .collect(); - let fingerprint = prev_graph.fingerprint_by_index(prev_index); - let dep_node_index = self.encoder.send(key, fingerprint, edges); + let dep_node_index = self.encoder.promote(prev_index, &*prev_index_to_index); prev_index_to_index[prev_index] = Some(dep_node_index); #[cfg(debug_assertions)] - self.record_edge(dep_node_index, key, fingerprint); + self.record_edge( + dep_node_index, + prev_graph.index_to_node(prev_index), + prev_graph.fingerprint_by_index(prev_index), + ); dep_node_index } } diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs index 0c6a6358293..b5c610f817d 100644 --- a/compiler/rustc_query_system/src/dep_graph/serialized.rs +++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs @@ -41,6 +41,7 @@ use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fingerprint::PackedFingerprint; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::outline; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::sync::Lock; use rustc_data_structures::unhash::UnhashMap; @@ -49,6 +50,7 @@ use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; use std::iter; use std::marker::PhantomData; +use std::sync::Arc; // The maximum value of `SerializedDepNodeIndex` leaves the upper two bits // unused so that we can store multiple index types in `CompressedHybridIndex`, @@ -94,7 +96,7 @@ impl SerializedDepGraph { pub fn edge_targets_from( &self, source: SerializedDepNodeIndex, - ) -> impl Iterator + '_ { + ) -> impl Iterator + Clone + '_ { let header = self.edge_list_indices[source]; let mut raw = &self.edge_list_data[header.start()..]; // Figure out where the edge list for `source` ends by getting the start index of the next @@ -176,7 +178,7 @@ fn mask(bits: usize) -> usize { impl SerializedDepGraph { #[instrument(level = "debug", skip(d))] - pub fn decode(d: &mut MemDecoder<'_>) -> SerializedDepGraph { + pub fn decode(d: &mut MemDecoder<'_>) -> Arc { // The last 16 bytes are the node count and edge count. debug!("position: {:?}", d.position()); let (node_count, edge_count, graph_size) = @@ -254,7 +256,13 @@ pub fn decode(d: &mut MemDecoder<'_>) -> SerializedDepGraph { index[node.kind.as_usize()].insert(node.hash, idx); } - SerializedDepGraph { nodes, fingerprints, edge_list_indices, edge_list_data, index } + Arc::new(SerializedDepGraph { + nodes, + fingerprints, + edge_list_indices, + edge_list_data, + index, + }) } } @@ -299,21 +307,24 @@ impl SerializedNodeHeader { const MAX_INLINE_LEN: usize = (u16::MAX as usize >> (Self::TOTAL_BITS - Self::LEN_BITS)) - 1; #[inline] - fn new(node_info: &NodeInfo) -> Self { + fn new( + node: DepNode, + fingerprint: Fingerprint, + edge_max_index: u32, + edge_count: usize, + ) -> Self { debug_assert_eq!(Self::TOTAL_BITS, Self::LEN_BITS + Self::WIDTH_BITS + Self::KIND_BITS); - let NodeInfo { node, fingerprint, edges } = node_info; - let mut head = node.kind.as_inner(); - let free_bytes = edges.max_index().leading_zeros() as usize / 8; + let free_bytes = edge_max_index.leading_zeros() as usize / 8; let bytes_per_index = (DEP_NODE_SIZE - free_bytes).saturating_sub(1); head |= (bytes_per_index as u16) << Self::KIND_BITS; // Encode number of edges + 1 so that we can reserve 0 to indicate that the len doesn't fit // in this bitfield. - if edges.len() <= Self::MAX_INLINE_LEN { - head |= (edges.len() as u16 + 1) << (Self::KIND_BITS + Self::WIDTH_BITS); + if edge_count <= Self::MAX_INLINE_LEN { + head |= (edge_count as u16 + 1) << (Self::KIND_BITS + Self::WIDTH_BITS); } let hash: Fingerprint = node.hash.into(); @@ -327,10 +338,10 @@ fn new(node_info: &NodeInfo) -> Self { #[cfg(debug_assertions)] { let res = Self { bytes, _marker: PhantomData }; - assert_eq!(node_info.fingerprint, res.fingerprint()); - assert_eq!(node_info.node, res.node()); + assert_eq!(fingerprint, res.fingerprint()); + assert_eq!(node, res.node()); if let Some(len) = res.len() { - assert_eq!(node_info.edges.len(), len); + assert_eq!(edge_count, len); } } Self { bytes, _marker: PhantomData } @@ -393,21 +404,56 @@ struct NodeInfo { impl NodeInfo { fn encode(&self, e: &mut FileEncoder) { - let header = SerializedNodeHeader::::new(self); + let NodeInfo { node, fingerprint, ref edges } = *self; + let header = + SerializedNodeHeader::::new(node, fingerprint, edges.max_index(), edges.len()); e.write_array(header.bytes); if header.len().is_none() { - e.emit_usize(self.edges.len()); + e.emit_usize(edges.len()); } let bytes_per_index = header.bytes_per_index(); - for node_index in self.edges.iter() { + for node_index in edges.iter() { e.write_with(|dest| { *dest = node_index.as_u32().to_le_bytes(); bytes_per_index }); } } + + #[inline] + fn encode_promoted( + e: &mut FileEncoder, + node: DepNode, + fingerprint: Fingerprint, + prev_index: SerializedDepNodeIndex, + prev_index_to_index: &IndexVec>, + previous: &SerializedDepGraph, + ) -> usize { + let edges = previous.edge_targets_from(prev_index); + let edge_count = edges.size_hint().0; + let edge_max = + edges.clone().map(|i| prev_index_to_index[i].unwrap().as_u32()).max().unwrap_or(0); + + let header = SerializedNodeHeader::::new(node, fingerprint, edge_max, edge_count); + e.write_array(header.bytes); + + if header.len().is_none() { + e.emit_usize(edge_count); + } + + let bytes_per_index = header.bytes_per_index(); + for node_index in edges { + let node_index = prev_index_to_index[node_index].unwrap(); + e.write_with(|dest| { + *dest = node_index.as_u32().to_le_bytes(); + bytes_per_index + }); + } + + edge_count + } } struct Stat { @@ -417,6 +463,7 @@ struct Stat { } struct EncoderState { + previous: Arc, encoder: FileEncoder, total_node_count: usize, total_edge_count: usize, @@ -428,8 +475,9 @@ struct EncoderState { } impl EncoderState { - fn new(encoder: FileEncoder, record_stats: bool) -> Self { + fn new(encoder: FileEncoder, record_stats: bool, previous: Arc) -> Self { Self { + previous, encoder, total_edge_count: 0, total_node_count: 0, @@ -439,36 +487,88 @@ fn new(encoder: FileEncoder, record_stats: bool) -> Self { } } + #[inline] + fn record( + &mut self, + node: DepNode, + edge_count: usize, + edges: impl FnOnce(&mut Self) -> Vec, + record_graph: &Option>, + ) -> DepNodeIndex { + let index = DepNodeIndex::new(self.total_node_count); + + self.total_node_count += 1; + self.kind_stats[node.kind.as_usize()] += 1; + self.total_edge_count += edge_count; + + if let Some(record_graph) = &record_graph { + let edges = edges(self); + outline(move || { + // Do not ICE when a query is called from within `with_query`. + if let Some(record_graph) = &mut record_graph.try_lock() { + record_graph.push(index, node, &edges); + } + }); + } + + if let Some(stats) = &mut self.stats { + let kind = node.kind; + + outline(move || { + let stat = + stats.entry(kind).or_insert(Stat { kind, node_counter: 0, edge_counter: 0 }); + stat.node_counter += 1; + stat.edge_counter += edge_count as u64; + }); + } + + index + } + fn encode_node( &mut self, node: &NodeInfo, record_graph: &Option>, ) -> DepNodeIndex { - let index = DepNodeIndex::new(self.total_node_count); - self.total_node_count += 1; - self.kind_stats[node.node.kind.as_usize()] += 1; + node.encode::(&mut self.encoder); + self.record( + node.node, + node.edges.len(), + |_| node.edges[..].iter().copied().collect(), + record_graph, + ) + } - let edge_count = node.edges.len(); - self.total_edge_count += edge_count; + #[inline] + fn promote_node( + &mut self, + prev_index: SerializedDepNodeIndex, + record_graph: &Option>, + prev_index_to_index: &IndexVec>, + ) -> DepNodeIndex { + let node = self.previous.index_to_node(prev_index); - if let Some(record_graph) = &record_graph { - // Do not ICE when a query is called from within `with_query`. - if let Some(record_graph) = &mut record_graph.try_lock() { - record_graph.push(index, node.node, &node.edges); - } - } + let fingerprint = self.previous.fingerprint_by_index(prev_index); + let edge_count = NodeInfo::encode_promoted::( + &mut self.encoder, + node, + fingerprint, + prev_index, + prev_index_to_index, + &self.previous, + ); - if let Some(stats) = &mut self.stats { - let kind = node.node.kind; - - let stat = stats.entry(kind).or_insert(Stat { kind, node_counter: 0, edge_counter: 0 }); - stat.node_counter += 1; - stat.edge_counter += edge_count as u64; - } - - let encoder = &mut self.encoder; - node.encode::(encoder); - index + self.record( + node, + edge_count, + |this| { + this.previous + .edge_targets_from(prev_index) + .map(|i| prev_index_to_index[i].unwrap()) + .collect() + }, + record_graph, + ) } fn finish(self, profiler: &SelfProfilerRef) -> FileEncodeResult { @@ -479,6 +579,7 @@ fn finish(self, profiler: &SelfProfilerRef) -> FileEncodeResult { stats: _, kind_stats, marker: _, + previous: _, } = self; let node_count = total_node_count.try_into().unwrap(); @@ -520,9 +621,10 @@ pub fn new( record_graph: bool, record_stats: bool, profiler: &SelfProfilerRef, + previous: Arc, ) -> Self { let record_graph = record_graph.then(|| Lock::new(DepGraphQuery::new(prev_node_count))); - let status = Lock::new(Some(EncoderState::new(encoder, record_stats))); + let status = Lock::new(Some(EncoderState::new(encoder, record_stats, previous))); GraphEncoder { status, record_graph, profiler: profiler.clone() } } @@ -596,6 +698,20 @@ pub(crate) fn send( self.status.lock().as_mut().unwrap().encode_node(&node, &self.record_graph) } + #[inline] + pub(crate) fn promote( + &self, + prev_index: SerializedDepNodeIndex, + prev_index_to_index: &IndexVec>, + ) -> DepNodeIndex { + let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph"); + self.status.lock().as_mut().unwrap().promote_node( + prev_index, + &self.record_graph, + prev_index_to_index, + ) + } + pub fn finish(&self) -> FileEncodeResult { let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph_finish"); From aa9c9a36c06ac33275be60b840d1b954f8b1eac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Sat, 23 Mar 2024 20:23:25 +0100 Subject: [PATCH 2/2] Add some comments and do some renames --- .../rustc_query_system/src/dep_graph/graph.rs | 2 +- .../src/dep_graph/serialized.rs | 24 ++++++++++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index 4cf8ead8ef8..7be4ee16bd4 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -1240,7 +1240,7 @@ fn promote_node_and_deps_to_current( match prev_index_to_index[prev_index] { Some(dep_node_index) => dep_node_index, None => { - let dep_node_index = self.encoder.promote(prev_index, &*prev_index_to_index); + let dep_node_index = self.encoder.send_promoted(prev_index, &*prev_index_to_index); prev_index_to_index[prev_index] = Some(dep_node_index); #[cfg(debug_assertions)] self.record_edge( diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs index b5c610f817d..2bc7cb99547 100644 --- a/compiler/rustc_query_system/src/dep_graph/serialized.rs +++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs @@ -422,6 +422,9 @@ fn encode(&self, e: &mut FileEncoder) { } } + /// Encode a node that was promoted from the previous graph. It reads the edges directly from + /// the previous dep graph and expects all edges to already have a new dep node index assigned. + /// This avoids the overhead of constructing `EdgesVec`, which would be needed to call `encode`. #[inline] fn encode_promoted( e: &mut FileEncoder, @@ -433,6 +436,8 @@ fn encode_promoted( ) -> usize { let edges = previous.edge_targets_from(prev_index); let edge_count = edges.size_hint().0; + + // Find the highest edge in the new dep node indices let edge_max = edges.clone().map(|i| prev_index_to_index[i].unwrap().as_u32()).max().unwrap_or(0); @@ -502,7 +507,10 @@ fn record( self.total_edge_count += edge_count; if let Some(record_graph) = &record_graph { + // Call `edges` before the outlined code to allow the closure to be optimized out. let edges = edges(self); + + // Outline the build of the full dep graph as it's typically disabled and cold. outline(move || { // Do not ICE when a query is called from within `with_query`. if let Some(record_graph) = &mut record_graph.try_lock() { @@ -514,6 +522,7 @@ fn record( if let Some(stats) = &mut self.stats { let kind = node.kind; + // Outline the stats code as it's typically disabled and cold. outline(move || { let stat = stats.entry(kind).or_insert(Stat { kind, node_counter: 0, edge_counter: 0 }); @@ -525,6 +534,7 @@ fn record( index } + /// Encodes a node to the current graph. fn encode_node( &mut self, node: &NodeInfo, @@ -539,8 +549,14 @@ fn encode_node( ) } + /// Encodes a node that was promoted from the previous graph. It reads the information directly from + /// the previous dep graph for performance reasons. + /// + /// This differs from `encode_node` where you have to explictly provide the relevant `NodeInfo`. + /// + /// It expects all edges to already have a new dep node index assigned. #[inline] - fn promote_node( + fn encode_promoted_node( &mut self, prev_index: SerializedDepNodeIndex, record_graph: &Option>, @@ -698,14 +714,16 @@ pub(crate) fn send( self.status.lock().as_mut().unwrap().encode_node(&node, &self.record_graph) } + /// Encodes a node that was promoted from the previous graph. It reads the information directly from + /// the previous dep graph and expects all edges to already have a new dep node index assigned. #[inline] - pub(crate) fn promote( + pub(crate) fn send_promoted( &self, prev_index: SerializedDepNodeIndex, prev_index_to_index: &IndexVec>, ) -> DepNodeIndex { let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph"); - self.status.lock().as_mut().unwrap().promote_node( + self.status.lock().as_mut().unwrap().encode_promoted_node( prev_index, &self.record_graph, prev_index_to_index,