Use internal mutability for predecessor cache
This commit is contained in:
Dylan MacKenzie
parent
00f677d897
commit
884ba4f11f
@ -1,276 +0,0 @@
|
||||
use crate::ich::StableHashingContext;
|
||||
use crate::mir::{BasicBlock, BasicBlockData, Body, LocalDecls, Location, Successors};
|
||||
use rustc_data_structures::graph::dominators::{dominators, Dominators};
|
||||
use rustc_data_structures::graph::{self, GraphPredecessors, GraphSuccessors};
|
||||
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
|
||||
use rustc_index::vec::IndexVec;
|
||||
use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
|
||||
use smallvec::SmallVec;
|
||||
use std::iter;
|
||||
use std::ops::{Deref, DerefMut, Index, IndexMut};
|
||||
use std::vec::IntoIter;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Cache {
|
||||
// Typically 95%+ of the inner vectors have 4 or fewer elements.
|
||||
predecessors: Option<IndexVec<BasicBlock, SmallVec<[BasicBlock; 4]>>>,
|
||||
}
|
||||
|
||||
impl rustc_serialize::Encodable for Cache {
|
||||
fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> {
|
||||
Encodable::encode(&(), s)
|
||||
}
|
||||
}
|
||||
|
||||
impl rustc_serialize::Decodable for Cache {
|
||||
fn decode<D: Decoder>(d: &mut D) -> Result<Self, D::Error> {
|
||||
Decodable::decode(d).map(|_v: ()| Self::new())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> HashStable<StableHashingContext<'a>> for Cache {
|
||||
fn hash_stable(&self, _: &mut StableHashingContext<'a>, _: &mut StableHasher) {
|
||||
// Do nothing.
|
||||
}
|
||||
}
|
||||
|
||||
impl Cache {
|
||||
pub fn new() -> Self {
|
||||
Self { predecessors: None }
|
||||
}
|
||||
|
||||
pub fn invalidate_predecessors(&mut self) {
|
||||
// FIXME: consider being more fine-grained
|
||||
self.predecessors = None;
|
||||
}
|
||||
|
||||
pub fn ensure_predecessors(&mut self, body: &Body<'_>) {
|
||||
if self.predecessors.is_none() {
|
||||
let mut result = IndexVec::from_elem(smallvec![], body.basic_blocks());
|
||||
for (bb, data) in body.basic_blocks().iter_enumerated() {
|
||||
if let Some(ref term) = data.terminator {
|
||||
for &tgt in term.successors() {
|
||||
result[tgt].push(bb);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.predecessors = Some(result)
|
||||
}
|
||||
}
|
||||
|
||||
/// This will recompute the predecessors cache if it is not available
|
||||
fn predecessors(
|
||||
&mut self,
|
||||
body: &Body<'_>,
|
||||
) -> &IndexVec<BasicBlock, SmallVec<[BasicBlock; 4]>> {
|
||||
self.ensure_predecessors(body);
|
||||
self.predecessors.as_ref().unwrap()
|
||||
}
|
||||
|
||||
fn unwrap_predecessors_for(&self, bb: BasicBlock) -> &[BasicBlock] {
|
||||
&self.predecessors.as_ref().unwrap()[bb]
|
||||
}
|
||||
|
||||
fn unwrap_predecessor_locations<'a>(
|
||||
&'a self,
|
||||
loc: Location,
|
||||
body: &'a Body<'a>,
|
||||
) -> impl Iterator<Item = Location> + 'a {
|
||||
let if_zero_locations = if loc.statement_index == 0 {
|
||||
let predecessor_blocks = self.unwrap_predecessors_for(loc.block);
|
||||
let num_predecessor_blocks = predecessor_blocks.len();
|
||||
Some(
|
||||
(0..num_predecessor_blocks)
|
||||
.map(move |i| predecessor_blocks[i])
|
||||
.map(move |bb| body.terminator_loc(bb)),
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let if_not_zero_locations = if loc.statement_index == 0 {
|
||||
None
|
||||
} else {
|
||||
Some(Location { block: loc.block, statement_index: loc.statement_index - 1 })
|
||||
};
|
||||
|
||||
if_zero_locations.into_iter().flatten().chain(if_not_zero_locations)
|
||||
}
|
||||
|
||||
pub fn basic_blocks_mut<'a, 'tcx>(
|
||||
&mut self,
|
||||
body: &'a mut Body<'tcx>,
|
||||
) -> &'a mut IndexVec<BasicBlock, BasicBlockData<'tcx>> {
|
||||
debug!("bbm: Clearing predecessors cache for body at: {:?}", body.span.data());
|
||||
self.invalidate_predecessors();
|
||||
&mut body.basic_blocks
|
||||
}
|
||||
|
||||
pub fn basic_blocks_and_local_decls_mut<'a, 'tcx>(
|
||||
&mut self,
|
||||
body: &'a mut Body<'tcx>,
|
||||
) -> (&'a mut IndexVec<BasicBlock, BasicBlockData<'tcx>>, &'a mut LocalDecls<'tcx>) {
|
||||
debug!("bbaldm: Clearing predecessors cache for body at: {:?}", body.span.data());
|
||||
self.invalidate_predecessors();
|
||||
(&mut body.basic_blocks, &mut body.local_decls)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, HashStable, RustcEncodable, RustcDecodable, TypeFoldable)]
|
||||
pub struct BodyAndCache<'tcx> {
|
||||
body: Body<'tcx>,
|
||||
cache: Cache,
|
||||
}
|
||||
|
||||
impl BodyAndCache<'tcx> {
|
||||
pub fn new(body: Body<'tcx>) -> Self {
|
||||
Self { body, cache: Cache::new() }
|
||||
}
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! read_only {
|
||||
($body:expr) => {{
|
||||
$body.ensure_predecessors();
|
||||
$body.unwrap_read_only()
|
||||
}};
|
||||
}
|
||||
|
||||
impl BodyAndCache<'tcx> {
|
||||
pub fn ensure_predecessors(&mut self) {
|
||||
self.cache.ensure_predecessors(&self.body);
|
||||
}
|
||||
|
||||
pub fn predecessors(&mut self) -> &IndexVec<BasicBlock, SmallVec<[BasicBlock; 4]>> {
|
||||
self.cache.predecessors(&self.body)
|
||||
}
|
||||
|
||||
pub fn unwrap_read_only(&self) -> ReadOnlyBodyAndCache<'_, 'tcx> {
|
||||
ReadOnlyBodyAndCache::new(&self.body, &self.cache)
|
||||
}
|
||||
|
||||
pub fn basic_blocks_mut(&mut self) -> &mut IndexVec<BasicBlock, BasicBlockData<'tcx>> {
|
||||
self.cache.basic_blocks_mut(&mut self.body)
|
||||
}
|
||||
|
||||
pub fn basic_blocks_and_local_decls_mut(
|
||||
&mut self,
|
||||
) -> (&mut IndexVec<BasicBlock, BasicBlockData<'tcx>>, &mut LocalDecls<'tcx>) {
|
||||
self.cache.basic_blocks_and_local_decls_mut(&mut self.body)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'tcx> Index<BasicBlock> for BodyAndCache<'tcx> {
|
||||
type Output = BasicBlockData<'tcx>;
|
||||
|
||||
fn index(&self, index: BasicBlock) -> &BasicBlockData<'tcx> {
|
||||
&self.body[index]
|
||||
}
|
||||
}
|
||||
|
||||
impl<'tcx> IndexMut<BasicBlock> for BodyAndCache<'tcx> {
|
||||
fn index_mut(&mut self, index: BasicBlock) -> &mut Self::Output {
|
||||
&mut self.basic_blocks_mut()[index]
|
||||
}
|
||||
}
|
||||
|
||||
impl<'tcx> Deref for BodyAndCache<'tcx> {
|
||||
type Target = Body<'tcx>;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.body
|
||||
}
|
||||
}
|
||||
|
||||
impl<'tcx> DerefMut for BodyAndCache<'tcx> {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.body
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
pub struct ReadOnlyBodyAndCache<'a, 'tcx> {
|
||||
body: &'a Body<'tcx>,
|
||||
cache: &'a Cache,
|
||||
}
|
||||
|
||||
impl ReadOnlyBodyAndCache<'a, 'tcx> {
|
||||
fn new(body: &'a Body<'tcx>, cache: &'a Cache) -> Self {
|
||||
assert!(
|
||||
cache.predecessors.is_some(),
|
||||
"Cannot construct ReadOnlyBodyAndCache without computed predecessors"
|
||||
);
|
||||
Self { body, cache }
|
||||
}
|
||||
|
||||
pub fn predecessors(&self) -> &IndexVec<BasicBlock, SmallVec<[BasicBlock; 4]>> {
|
||||
self.cache.predecessors.as_ref().unwrap()
|
||||
}
|
||||
|
||||
pub fn predecessors_for(&self, bb: BasicBlock) -> &[BasicBlock] {
|
||||
self.cache.unwrap_predecessors_for(bb)
|
||||
}
|
||||
|
||||
pub fn predecessor_locations(&self, loc: Location) -> impl Iterator<Item = Location> + '_ {
|
||||
self.cache.unwrap_predecessor_locations(loc, self.body)
|
||||
}
|
||||
|
||||
pub fn basic_blocks(&self) -> &IndexVec<BasicBlock, BasicBlockData<'tcx>> {
|
||||
&self.body.basic_blocks
|
||||
}
|
||||
|
||||
pub fn dominators(&self) -> Dominators<BasicBlock> {
|
||||
dominators(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl graph::DirectedGraph for ReadOnlyBodyAndCache<'a, 'tcx> {
|
||||
type Node = BasicBlock;
|
||||
}
|
||||
|
||||
impl graph::GraphPredecessors<'graph> for ReadOnlyBodyAndCache<'a, 'tcx> {
|
||||
type Item = BasicBlock;
|
||||
type Iter = IntoIter<BasicBlock>;
|
||||
}
|
||||
|
||||
impl graph::WithPredecessors for ReadOnlyBodyAndCache<'a, 'tcx> {
|
||||
fn predecessors(&self, node: Self::Node) -> <Self as GraphPredecessors<'_>>::Iter {
|
||||
self.cache.unwrap_predecessors_for(node).to_vec().into_iter()
|
||||
}
|
||||
}
|
||||
|
||||
impl graph::WithNumNodes for ReadOnlyBodyAndCache<'a, 'tcx> {
|
||||
fn num_nodes(&self) -> usize {
|
||||
self.body.num_nodes()
|
||||
}
|
||||
}
|
||||
|
||||
impl graph::WithStartNode for ReadOnlyBodyAndCache<'a, 'tcx> {
|
||||
fn start_node(&self) -> Self::Node {
|
||||
self.body.start_node()
|
||||
}
|
||||
}
|
||||
|
||||
impl graph::WithSuccessors for ReadOnlyBodyAndCache<'a, 'tcx> {
|
||||
fn successors(&self, node: Self::Node) -> <Self as GraphSuccessors<'_>>::Iter {
|
||||
self.body.successors(node)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, 'b, 'tcx> graph::GraphSuccessors<'b> for ReadOnlyBodyAndCache<'a, 'tcx> {
|
||||
type Item = BasicBlock;
|
||||
type Iter = iter::Cloned<Successors<'b>>;
|
||||
}
|
||||
|
||||
impl Deref for ReadOnlyBodyAndCache<'a, 'tcx> {
|
||||
type Target = &'a Body<'tcx>;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.body
|
||||
}
|
||||
}
|
||||
|
||||
CloneTypeFoldableAndLiftImpls! {
|
||||
Cache,
|
||||
}
|
||||
@ -21,27 +21,28 @@ use polonius_engine::Atom;
|
||||
pub use rustc_ast::ast::Mutability;
|
||||
use rustc_ast::ast::Name;
|
||||
use rustc_data_structures::fx::FxHashSet;
|
||||
use rustc_data_structures::graph::dominators::Dominators;
|
||||
use rustc_data_structures::graph::dominators::{dominators, Dominators};
|
||||
use rustc_data_structures::graph::{self, GraphSuccessors};
|
||||
use rustc_data_structures::sync::MappedLockGuard;
|
||||
use rustc_index::bit_set::BitMatrix;
|
||||
use rustc_index::vec::{Idx, IndexVec};
|
||||
use rustc_macros::HashStable;
|
||||
use rustc_serialize::{Decodable, Encodable};
|
||||
use rustc_span::symbol::Symbol;
|
||||
use rustc_span::{Span, DUMMY_SP};
|
||||
use smallvec::SmallVec;
|
||||
use std::borrow::Cow;
|
||||
use std::fmt::{self, Debug, Display, Formatter, Write};
|
||||
use std::ops::Index;
|
||||
use std::ops::{Index, IndexMut};
|
||||
use std::slice;
|
||||
use std::{iter, mem, option};
|
||||
|
||||
pub use self::cache::{BodyAndCache, ReadOnlyBodyAndCache};
|
||||
use self::predecessors::{PredecessorCache, Predecessors};
|
||||
pub use self::query::*;
|
||||
pub use crate::read_only;
|
||||
|
||||
mod cache;
|
||||
pub mod interpret;
|
||||
pub mod mono;
|
||||
mod predecessors;
|
||||
mod query;
|
||||
pub mod tcx;
|
||||
pub mod traversal;
|
||||
@ -108,7 +109,7 @@ pub struct Body<'tcx> {
|
||||
pub yield_ty: Option<Ty<'tcx>>,
|
||||
|
||||
/// Generator drop glue.
|
||||
pub generator_drop: Option<Box<BodyAndCache<'tcx>>>,
|
||||
pub generator_drop: Option<Box<Body<'tcx>>>,
|
||||
|
||||
/// The layout of a generator. Produced by the state transformation.
|
||||
pub generator_layout: Option<GeneratorLayout<'tcx>>,
|
||||
@ -164,6 +165,8 @@ pub struct Body<'tcx> {
|
||||
/// implementation without the flag hid this situation silently.
|
||||
/// FIXME(oli-obk): rewrite the promoted during promotion to eliminate the cell components.
|
||||
pub ignore_interior_mut_in_const_validation: bool,
|
||||
|
||||
pub predecessor_cache: PredecessorCache,
|
||||
}
|
||||
|
||||
impl<'tcx> Body<'tcx> {
|
||||
@ -202,6 +205,7 @@ impl<'tcx> Body<'tcx> {
|
||||
span,
|
||||
ignore_interior_mut_in_const_validation: false,
|
||||
control_flow_destroyed,
|
||||
predecessor_cache: PredecessorCache::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@ -227,6 +231,7 @@ impl<'tcx> Body<'tcx> {
|
||||
generator_kind: None,
|
||||
var_debug_info: Vec::new(),
|
||||
ignore_interior_mut_in_const_validation: false,
|
||||
predecessor_cache: PredecessorCache::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@ -235,6 +240,25 @@ impl<'tcx> Body<'tcx> {
|
||||
&self.basic_blocks
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn basic_blocks_mut(&mut self) -> &mut IndexVec<BasicBlock, BasicBlockData<'tcx>> {
|
||||
// Because the user could mutate basic block terminators via this reference, we need to
|
||||
// invalidate the predecessor cache.
|
||||
//
|
||||
// FIXME: Use a finer-grained API for this, so only transformations that alter terminators
|
||||
// invalidate the predecessor cache.
|
||||
self.predecessor_cache.invalidate();
|
||||
&mut self.basic_blocks
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn basic_blocks_and_local_decls_mut(
|
||||
&mut self,
|
||||
) -> (&mut IndexVec<BasicBlock, BasicBlockData<'tcx>>, &mut LocalDecls<'tcx>) {
|
||||
self.predecessor_cache.invalidate();
|
||||
(&mut self.basic_blocks, &mut self.local_decls)
|
||||
}
|
||||
|
||||
/// Returns `true` if a cycle exists in the control-flow graph that is reachable from the
|
||||
/// `START_BLOCK`.
|
||||
pub fn is_cfg_cyclic(&self) -> bool {
|
||||
@ -365,6 +389,23 @@ impl<'tcx> Body<'tcx> {
|
||||
pub fn terminator_loc(&self, bb: BasicBlock) -> Location {
|
||||
Location { block: bb, statement_index: self[bb].statements.len() }
|
||||
}
|
||||
|
||||
pub fn predecessors_for(
|
||||
&self,
|
||||
bb: BasicBlock,
|
||||
) -> impl std::ops::Deref<Target = SmallVec<[BasicBlock; 4]>> + '_ {
|
||||
let predecessors = self.predecessor_cache.compute(&self.basic_blocks);
|
||||
MappedLockGuard::map(predecessors, |preds| &mut preds[bb])
|
||||
}
|
||||
|
||||
pub fn predecessors(&self) -> impl std::ops::Deref<Target = Predecessors> + '_ {
|
||||
self.predecessor_cache.compute(&self.basic_blocks)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn dominators(&self) -> Dominators<BasicBlock> {
|
||||
dominators(self)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, RustcEncodable, RustcDecodable, HashStable)]
|
||||
@ -387,6 +428,13 @@ impl<'tcx> Index<BasicBlock> for Body<'tcx> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<'tcx> IndexMut<BasicBlock> for Body<'tcx> {
|
||||
#[inline]
|
||||
fn index_mut(&mut self, index: BasicBlock) -> &mut BasicBlockData<'tcx> {
|
||||
&mut self.basic_blocks_mut()[index]
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, HashStable, TypeFoldable)]
|
||||
pub enum ClearCrossCrate<T> {
|
||||
Clear,
|
||||
@ -2613,6 +2661,17 @@ impl<'a, 'b> graph::GraphSuccessors<'b> for Body<'a> {
|
||||
type Iter = iter::Cloned<Successors<'b>>;
|
||||
}
|
||||
|
||||
impl graph::GraphPredecessors<'graph> for Body<'tcx> {
|
||||
type Item = BasicBlock;
|
||||
type Iter = smallvec::IntoIter<[BasicBlock; 4]>;
|
||||
}
|
||||
|
||||
impl graph::WithPredecessors for Body<'tcx> {
|
||||
fn predecessors(&self, node: Self::Node) -> <Self as graph::GraphPredecessors<'_>>::Iter {
|
||||
self.predecessors_for(node).clone().into_iter()
|
||||
}
|
||||
}
|
||||
|
||||
/// `Location` represents the position of the start of the statement; or, if
|
||||
/// `statement_index` equals the number of statements, then the start of the
|
||||
/// terminator.
|
||||
@ -2642,11 +2701,7 @@ impl Location {
|
||||
}
|
||||
|
||||
/// Returns `true` if `other` is earlier in the control flow graph than `self`.
|
||||
pub fn is_predecessor_of<'tcx>(
|
||||
&self,
|
||||
other: Location,
|
||||
body: ReadOnlyBodyAndCache<'_, 'tcx>,
|
||||
) -> bool {
|
||||
pub fn is_predecessor_of<'tcx>(&self, other: Location, body: &Body<'tcx>) -> bool {
|
||||
// If we are in the same block as the other location and are an earlier statement
|
||||
// then we are a predecessor of `other`.
|
||||
if self.block == other.block && self.statement_index < other.statement_index {
|
||||
|
||||
67
src/librustc_middle/mir/predecessors.rs
Normal file
67
src/librustc_middle/mir/predecessors.rs
Normal file
@ -0,0 +1,67 @@
|
||||
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
|
||||
use rustc_data_structures::sync::{Lock, LockGuard, MappedLockGuard};
|
||||
use rustc_index::vec::IndexVec;
|
||||
use rustc_serialize as serialize;
|
||||
use smallvec::SmallVec;
|
||||
|
||||
use crate::mir::{BasicBlock, BasicBlockData};
|
||||
|
||||
// Typically 95%+ of basic blocks have 4 or fewer predecessors.
|
||||
pub type Predecessors = IndexVec<BasicBlock, SmallVec<[BasicBlock; 4]>>;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PredecessorCache {
|
||||
cache: Lock<Option<Predecessors>>,
|
||||
}
|
||||
|
||||
impl PredecessorCache {
|
||||
pub fn new() -> Self {
|
||||
PredecessorCache { cache: Lock::new(None) }
|
||||
}
|
||||
|
||||
pub fn invalidate(&mut self) {
|
||||
*self.cache.get_mut() = None;
|
||||
}
|
||||
|
||||
pub fn compute(
|
||||
&self,
|
||||
basic_blocks: &IndexVec<BasicBlock, BasicBlockData<'_>>,
|
||||
) -> MappedLockGuard<'_, Predecessors> {
|
||||
LockGuard::map(self.cache.lock(), |cache| {
|
||||
cache.get_or_insert_with(|| {
|
||||
let mut preds = IndexVec::from_elem(SmallVec::new(), basic_blocks);
|
||||
for (bb, data) in basic_blocks.iter_enumerated() {
|
||||
if let Some(term) = &data.terminator {
|
||||
for &succ in term.successors() {
|
||||
preds[succ].push(bb);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
preds
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl serialize::Encodable for PredecessorCache {
|
||||
fn encode<S: serialize::Encoder>(&self, s: &mut S) -> Result<(), S::Error> {
|
||||
serialize::Encodable::encode(&(), s)
|
||||
}
|
||||
}
|
||||
|
||||
impl serialize::Decodable for PredecessorCache {
|
||||
fn decode<D: serialize::Decoder>(d: &mut D) -> Result<Self, D::Error> {
|
||||
serialize::Decodable::decode(d).map(|_v: ()| Self::new())
|
||||
}
|
||||
}
|
||||
|
||||
impl<CTX> HashStable<CTX> for PredecessorCache {
|
||||
fn hash_stable(&self, _: &mut CTX, _: &mut StableHasher) {
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
|
||||
CloneTypeFoldableAndLiftImpls! {
|
||||
PredecessorCache,
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user