From e596579066e611a09557421c348560a89819ad80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Wed, 14 Jun 2023 19:52:49 +0000 Subject: [PATCH 1/8] inline explicit rpo access in promote consts --- compiler/rustc_const_eval/src/transform/promote_consts.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/compiler/rustc_const_eval/src/transform/promote_consts.rs b/compiler/rustc_const_eval/src/transform/promote_consts.rs index 0e2d9ee8fb2..44b143c77f3 100644 --- a/compiler/rustc_const_eval/src/transform/promote_consts.rs +++ b/compiler/rustc_const_eval/src/transform/promote_consts.rs @@ -14,7 +14,6 @@ use rustc_hir as hir; use rustc_middle::mir; -use rustc_middle::mir::traversal::ReversePostorderIter; use rustc_middle::mir::visit::{MutVisitor, MutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::subst::InternalSubsts; @@ -53,9 +52,8 @@ fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { return; } - let mut rpo = traversal::reverse_postorder(body); let ccx = ConstCx::new(tcx, body); - let (mut temps, all_candidates) = collect_temps_and_candidates(&ccx, &mut rpo); + let (mut temps, all_candidates) = collect_temps_and_candidates(&ccx); let promotable_candidates = validate_candidates(&ccx, &mut temps, &all_candidates); @@ -166,14 +164,13 @@ fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) { pub fn collect_temps_and_candidates<'tcx>( ccx: &ConstCx<'_, 'tcx>, - rpo: &mut ReversePostorderIter<'_, 'tcx>, ) -> (IndexVec, Vec) { let mut collector = Collector { temps: IndexVec::from_elem(TempState::Undefined, &ccx.body.local_decls), candidates: vec![], ccx, }; - for (bb, data) in rpo { + for (bb, data) in traversal::reverse_postorder(ccx.body) { collector.visit_basic_block_data(bb, data); } (collector.temps, collector.candidates) From 066d38190bb5e086d7ab565bc46d71b49c6d5217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Wed, 14 Jun 2023 19:56:35 +0000 Subject: [PATCH 2/8] add RPO to BB CFG cache --- compiler/rustc_middle/src/mir/basic_blocks.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/compiler/rustc_middle/src/mir/basic_blocks.rs b/compiler/rustc_middle/src/mir/basic_blocks.rs index 9d70dbfa072..29873a93fe6 100644 --- a/compiler/rustc_middle/src/mir/basic_blocks.rs +++ b/compiler/rustc_middle/src/mir/basic_blocks.rs @@ -27,6 +27,7 @@ struct Cache { switch_sources: OnceCell, is_cyclic: OnceCell, postorder: OnceCell>, + reverse_postorder: OnceCell>, dominators: OnceCell>, } @@ -70,6 +71,17 @@ pub fn postorder(&self) -> &[BasicBlock] { }) } + /// Returns basic blocks in a reverse postorder. + #[inline] + pub fn reverse_postorder(&self) -> &[BasicBlock] { + self.cache.reverse_postorder.get_or_init(|| { + let mut rpo: Vec<_> = + Postorder::new(&self.basic_blocks, START_BLOCK).map(|(bb, _)| bb).collect(); + rpo.reverse(); + rpo + }) + } + /// `switch_sources()[&(target, switch)]` returns a list of switch /// values that lead to a `target` block from a `switch` block. #[inline] From 4215859617a1037955db1150719c6faed2ce32a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Wed, 14 Jun 2023 19:57:23 +0000 Subject: [PATCH 3/8] make const-prop use cached RPO --- compiler/rustc_mir_transform/src/const_prop.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs index 1d43dbda0aa..2f2c7357b00 100644 --- a/compiler/rustc_mir_transform/src/const_prop.rs +++ b/compiler/rustc_mir_transform/src/const_prop.rs @@ -118,8 +118,8 @@ fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // Traverse the body in reverse post-order, to ensure that `FullConstProp` locals are // assigned before being read. - let postorder = body.basic_blocks.postorder().to_vec(); - for bb in postorder.into_iter().rev() { + let rpo = body.basic_blocks.reverse_postorder().to_vec(); + for bb in rpo { let data = &mut body.basic_blocks.as_mut_preserves_cfg()[bb]; optimization_finder.visit_basic_block_data(bb, data); } From 8f6e65136c94956fc172f73b31be282b65afb0c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Wed, 14 Jun 2023 19:57:55 +0000 Subject: [PATCH 4/8] make reorder BB pass use cached RPO --- compiler/rustc_mir_transform/src/prettify.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_mir_transform/src/prettify.rs b/compiler/rustc_mir_transform/src/prettify.rs index 6f46974ea00..745fa30841c 100644 --- a/compiler/rustc_mir_transform/src/prettify.rs +++ b/compiler/rustc_mir_transform/src/prettify.rs @@ -24,7 +24,7 @@ fn is_enabled(&self, _session: &Session) -> bool { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let rpo: IndexVec = - body.basic_blocks.postorder().iter().copied().rev().collect(); + body.basic_blocks.reverse_postorder().iter().copied().collect(); if rpo.iter().is_sorted() { return; } From 0eec39b67d12d121095a1da4e24109ce4dc41054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Wed, 14 Jun 2023 19:59:41 +0000 Subject: [PATCH 5/8] make `traversal::reverse_postorder` use RPO cache --- compiler/rustc_middle/src/mir/traversal.rs | 35 ++++------------------ 1 file changed, 6 insertions(+), 29 deletions(-) diff --git a/compiler/rustc_middle/src/mir/traversal.rs b/compiler/rustc_middle/src/mir/traversal.rs index 99ead14139a..780003f5fa0 100644 --- a/compiler/rustc_middle/src/mir/traversal.rs +++ b/compiler/rustc_middle/src/mir/traversal.rs @@ -295,34 +295,11 @@ pub fn reachable_as_bitset(body: &Body<'_>) -> BitSet { iter.visited } -#[derive(Clone)] -pub struct ReversePostorderIter<'a, 'tcx> { +/// Creates an iterator over the `Body`'s basic blocks, that: +/// - returns basic blocks in a reverse postorder, +/// - makes use of the `BasicBlocks` CFG cache's reverse postorder. +pub fn reverse_postorder<'a, 'tcx>( body: &'a Body<'tcx>, - blocks: &'a [BasicBlock], - idx: usize, -} - -impl<'a, 'tcx> Iterator for ReversePostorderIter<'a, 'tcx> { - type Item = (BasicBlock, &'a BasicBlockData<'tcx>); - - fn next(&mut self) -> Option<(BasicBlock, &'a BasicBlockData<'tcx>)> { - if self.idx == 0 { - return None; - } - self.idx -= 1; - - self.blocks.get(self.idx).map(|&bb| (bb, &self.body[bb])) - } - - fn size_hint(&self) -> (usize, Option) { - (self.idx, Some(self.idx)) - } -} - -impl<'a, 'tcx> ExactSizeIterator for ReversePostorderIter<'a, 'tcx> {} - -pub fn reverse_postorder<'a, 'tcx>(body: &'a Body<'tcx>) -> ReversePostorderIter<'a, 'tcx> { - let blocks = body.basic_blocks.postorder(); - let len = blocks.len(); - ReversePostorderIter { body, blocks, idx: len } +) -> impl Iterator)> + ExactSizeIterator { + body.basic_blocks.reverse_postorder().iter().map(|&bb| (bb, &body.basic_blocks[bb])) } From 0b4b0869a7968a2612c9bd93019a021ce168c557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Wed, 14 Jun 2023 20:00:23 +0000 Subject: [PATCH 6/8] make `traversal::postorder` traverse RPO cache backwards --- compiler/rustc_middle/src/mir/traversal.rs | 40 +++++++++++++++++++--- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_middle/src/mir/traversal.rs b/compiler/rustc_middle/src/mir/traversal.rs index 780003f5fa0..6d44234d11f 100644 --- a/compiler/rustc_middle/src/mir/traversal.rs +++ b/compiler/rustc_middle/src/mir/traversal.rs @@ -188,10 +188,6 @@ fn traverse_successor(&mut self) { } } -pub fn postorder<'a, 'tcx>(body: &'a Body<'tcx>) -> Postorder<'a, 'tcx> { - Postorder::new(&body.basic_blocks, START_BLOCK) -} - impl<'a, 'tcx> Iterator for Postorder<'a, 'tcx> { type Item = (BasicBlock, &'a BasicBlockData<'tcx>); @@ -219,6 +215,42 @@ fn size_hint(&self) -> (usize, Option) { } } +/// Creates an iterator over the `Body`'s basic blocks, that: +/// - returns basic blocks in a postorder, +/// - traverses the `BasicBlocks` CFG cache's reverse postorder backwards, and does not cache the +/// postorder itself. +pub fn postorder<'a, 'tcx>(body: &'a Body<'tcx>) -> PostorderIter<'a, 'tcx> { + let blocks = body.basic_blocks.reverse_postorder(); + let len = blocks.len(); + PostorderIter { body, blocks, idx: len } +} + +#[derive(Clone)] +pub struct PostorderIter<'a, 'tcx> { + body: &'a Body<'tcx>, + blocks: &'a [BasicBlock], + idx: usize, +} + +impl<'a, 'tcx> Iterator for PostorderIter<'a, 'tcx> { + type Item = (BasicBlock, &'a BasicBlockData<'tcx>); + + fn next(&mut self) -> Option<(BasicBlock, &'a BasicBlockData<'tcx>)> { + if self.idx == 0 { + return None; + } + self.idx -= 1; + + self.blocks.get(self.idx).map(|&bb| (bb, &self.body[bb])) + } + + fn size_hint(&self) -> (usize, Option) { + (self.idx, Some(self.idx)) + } +} + +impl<'a, 'tcx> ExactSizeIterator for PostorderIter<'a, 'tcx> {} + /// Reverse postorder traversal of a graph /// /// Reverse postorder is the reverse order of a postorder traversal. From f134101e69bcf52ffa36b9a713a38df80a3a9676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Wed, 14 Jun 2023 20:04:51 +0000 Subject: [PATCH 7/8] remove unused postorder CFG cache --- compiler/rustc_middle/src/mir/basic_blocks.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/compiler/rustc_middle/src/mir/basic_blocks.rs b/compiler/rustc_middle/src/mir/basic_blocks.rs index 29873a93fe6..7722e7b47cf 100644 --- a/compiler/rustc_middle/src/mir/basic_blocks.rs +++ b/compiler/rustc_middle/src/mir/basic_blocks.rs @@ -26,7 +26,6 @@ struct Cache { predecessors: OnceCell, switch_sources: OnceCell, is_cyclic: OnceCell, - postorder: OnceCell>, reverse_postorder: OnceCell>, dominators: OnceCell>, } @@ -63,14 +62,6 @@ pub fn predecessors(&self) -> &Predecessors { }) } - /// Returns basic blocks in a postorder. - #[inline] - pub fn postorder(&self) -> &[BasicBlock] { - self.cache.postorder.get_or_init(|| { - Postorder::new(&self.basic_blocks, START_BLOCK).map(|(bb, _)| bb).collect() - }) - } - /// Returns basic blocks in a reverse postorder. #[inline] pub fn reverse_postorder(&self) -> &[BasicBlock] { From 08a9f25245eb7cbc4f0952a8497808f94999b6b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Sun, 18 Jun 2023 09:16:40 +0000 Subject: [PATCH 8/8] remove redundant combinators between PO and RPO --- compiler/rustc_middle/src/mir/traversal.rs | 36 ++++------------------ 1 file changed, 6 insertions(+), 30 deletions(-) diff --git a/compiler/rustc_middle/src/mir/traversal.rs b/compiler/rustc_middle/src/mir/traversal.rs index 6d44234d11f..ec16a8470c4 100644 --- a/compiler/rustc_middle/src/mir/traversal.rs +++ b/compiler/rustc_middle/src/mir/traversal.rs @@ -219,38 +219,13 @@ fn size_hint(&self) -> (usize, Option) { /// - returns basic blocks in a postorder, /// - traverses the `BasicBlocks` CFG cache's reverse postorder backwards, and does not cache the /// postorder itself. -pub fn postorder<'a, 'tcx>(body: &'a Body<'tcx>) -> PostorderIter<'a, 'tcx> { - let blocks = body.basic_blocks.reverse_postorder(); - let len = blocks.len(); - PostorderIter { body, blocks, idx: len } -} - -#[derive(Clone)] -pub struct PostorderIter<'a, 'tcx> { +pub fn postorder<'a, 'tcx>( body: &'a Body<'tcx>, - blocks: &'a [BasicBlock], - idx: usize, +) -> impl Iterator)> + ExactSizeIterator + DoubleEndedIterator +{ + reverse_postorder(body).rev() } -impl<'a, 'tcx> Iterator for PostorderIter<'a, 'tcx> { - type Item = (BasicBlock, &'a BasicBlockData<'tcx>); - - fn next(&mut self) -> Option<(BasicBlock, &'a BasicBlockData<'tcx>)> { - if self.idx == 0 { - return None; - } - self.idx -= 1; - - self.blocks.get(self.idx).map(|&bb| (bb, &self.body[bb])) - } - - fn size_hint(&self) -> (usize, Option) { - (self.idx, Some(self.idx)) - } -} - -impl<'a, 'tcx> ExactSizeIterator for PostorderIter<'a, 'tcx> {} - /// Reverse postorder traversal of a graph /// /// Reverse postorder is the reverse order of a postorder traversal. @@ -332,6 +307,7 @@ pub fn reachable_as_bitset(body: &Body<'_>) -> BitSet { /// - makes use of the `BasicBlocks` CFG cache's reverse postorder. pub fn reverse_postorder<'a, 'tcx>( body: &'a Body<'tcx>, -) -> impl Iterator)> + ExactSizeIterator { +) -> impl Iterator)> + ExactSizeIterator + DoubleEndedIterator +{ body.basic_blocks.reverse_postorder().iter().map(|&bb| (bb, &body.basic_blocks[bb])) }