From cc21d9aa522cf07148e154ce9aec5f5f7a67c86c Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 31 May 2023 14:42:54 +1000 Subject: [PATCH] Don't compute inlining status of mono items in advance. We record inlining status for mono items in `MonoItems`, and then transfer it to `InliningMap`, for later use in `InliningMap::with_inlining_candidates`. But we can just compute inlining status directly in `InliningMap::with_inlining_candidates`, because the mono item is right there. There's no need to compute it in advance. This commit changes the code to do that, removing the need for `MonoItems` and `InliningMap::inlines`. This does result in more calls to `instantiation_mode` (one per static occurrence) but the performance effect is negligible. --- compiler/rustc_monomorphize/src/collector.rs | 76 ++++--------------- .../rustc_monomorphize/src/partitioning.rs | 7 +- 2 files changed, 17 insertions(+), 66 deletions(-) diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index cefa64d27ac..09025c1ee33 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -179,7 +179,6 @@ use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, DefIdMap, LocalDefId}; use rustc_hir::lang_items::LangItem; -use rustc_index::bit_set::GrowableBitSet; use rustc_middle::mir::interpret::{AllocId, ConstValue}; use rustc_middle::mir::interpret::{ErrorHandled, GlobalAlloc, Scalar}; use rustc_middle::mir::mono::{InstantiationMode, MonoItem}; @@ -220,78 +219,29 @@ pub struct InliningMap<'tcx> { // The range selects elements within the `targets` vecs. index: FxHashMap, Range>, targets: Vec>, - - // Contains one bit per mono item in the `targets` field. That bit - // is true if that mono item needs to be inlined into every CGU. - inlines: GrowableBitSet, } -/// Struct to store mono items in each collecting and if they should -/// be inlined. We call `instantiation_mode` to get their inlining -/// status when inserting new elements, which avoids calling it in -/// `inlining_map.lock_mut()`. See the `collect_items_rec` implementation -/// below. -struct MonoItems<'tcx> { - // If this is false, we do not need to compute whether items - // will need to be inlined. - compute_inlining: bool, - - // The TyCtxt used to determine whether the a item should - // be inlined. - tcx: TyCtxt<'tcx>, - - // The collected mono items. The bool field in each element - // indicates whether this element should be inlined. - items: Vec<(Spanned>, bool /*inlined*/)>, -} - -impl<'tcx> MonoItems<'tcx> { - #[inline] - fn push(&mut self, item: Spanned>) { - self.extend([item]); - } - - #[inline] - fn extend>>>(&mut self, iter: T) { - self.items.extend(iter.into_iter().map(|mono_item| { - let inlined = if !self.compute_inlining { - false - } else { - mono_item.node.instantiation_mode(self.tcx) == InstantiationMode::LocalCopy - }; - (mono_item, inlined) - })) - } -} +type MonoItems<'tcx> = Vec>>; impl<'tcx> InliningMap<'tcx> { fn new() -> InliningMap<'tcx> { - InliningMap { - index: FxHashMap::default(), - targets: Vec::new(), - inlines: GrowableBitSet::with_capacity(1024), - } + InliningMap { index: FxHashMap::default(), targets: Vec::new() } } fn record_accesses<'a>( &mut self, source: MonoItem<'tcx>, - new_targets: &'a [(Spanned>, bool)], + new_targets: &'a [Spanned>], ) where 'tcx: 'a, { let start_index = self.targets.len(); let new_items_count = new_targets.len(); - let new_items_count_total = new_items_count + self.targets.len(); self.targets.reserve(new_items_count); - self.inlines.ensure(new_items_count_total); - for (i, (Spanned { node: mono_item, .. }, inlined)) in new_targets.into_iter().enumerate() { + for Spanned { node: mono_item, .. } in new_targets.into_iter() { self.targets.push(*mono_item); - if *inlined { - self.inlines.insert(i + start_index); - } } let end_index = self.targets.len(); @@ -300,13 +250,14 @@ fn record_accesses<'a>( /// Internally iterate over all items referenced by `source` which will be /// made available for inlining. - pub fn with_inlining_candidates(&self, source: MonoItem<'tcx>, mut f: F) + pub fn with_inlining_candidates(&self, tcx: TyCtxt<'tcx>, source: MonoItem<'tcx>, mut f: F) where F: FnMut(MonoItem<'tcx>), { if let Some(range) = self.index.get(&source) { - for (i, candidate) in self.targets[range.clone()].iter().enumerate() { - if self.inlines.contains(range.start + i) { + for candidate in self.targets[range.clone()].iter() { + let is_inlined = candidate.instantiation_mode(tcx) == InstantiationMode::LocalCopy; + if is_inlined { f(*candidate); } } @@ -367,7 +318,7 @@ pub fn collect_crate_mono_items( #[instrument(skip(tcx, mode), level = "debug")] fn collect_roots(tcx: TyCtxt<'_>, mode: MonoItemCollectionMode) -> Vec> { debug!("collecting roots"); - let mut roots = MonoItems { compute_inlining: false, tcx, items: Vec::new() }; + let mut roots = Vec::new(); { let entry_fn = tcx.entry_fn(()); @@ -393,9 +344,8 @@ fn collect_roots(tcx: TyCtxt<'_>, mode: MonoItemCollectionMode) -> Vec( return; } - let mut neighbors = MonoItems { compute_inlining: true, tcx, items: Vec::new() }; + let mut neighbors = Vec::new(); let recursion_depth_reset; // @@ -542,9 +492,9 @@ fn collect_items_rec<'tcx>( formatted_item, }); } - inlining_map.lock_mut().record_accesses(starting_point.node, &neighbors.items); + inlining_map.lock_mut().record_accesses(starting_point.node, &neighbors); - for (neighbour, _) in neighbors.items { + for neighbour in neighbors { collect_items_rec(tcx, neighbour, visited, recursion_depths, recursion_limit, inlining_map); } diff --git a/compiler/rustc_monomorphize/src/partitioning.rs b/compiler/rustc_monomorphize/src/partitioning.rs index be9c349c384..0bacb58383d 100644 --- a/compiler/rustc_monomorphize/src/partitioning.rs +++ b/compiler/rustc_monomorphize/src/partitioning.rs @@ -424,7 +424,7 @@ fn place_inlined_mono_items<'tcx>( // Collect all items that need to be available in this codegen unit. let mut reachable = FxHashSet::default(); for root in old_codegen_unit.items().keys() { - follow_inlining(*root, cx.inlining_map, &mut reachable); + follow_inlining(cx.tcx, *root, cx.inlining_map, &mut reachable); } let mut new_codegen_unit = CodegenUnit::new(old_codegen_unit.name()); @@ -478,6 +478,7 @@ fn place_inlined_mono_items<'tcx>( return mono_item_placements; fn follow_inlining<'tcx>( + tcx: TyCtxt<'tcx>, mono_item: MonoItem<'tcx>, inlining_map: &InliningMap<'tcx>, visited: &mut FxHashSet>, @@ -486,8 +487,8 @@ fn follow_inlining<'tcx>( return; } - inlining_map.with_inlining_candidates(mono_item, |target| { - follow_inlining(target, inlining_map, visited); + inlining_map.with_inlining_candidates(tcx, mono_item, |target| { + follow_inlining(tcx, target, inlining_map, visited); }); } }