Refactor cost computation as a visitor.
This commit is contained in:
Camille GILLOT
parent
bb99e6fdd9
commit
e4fd2bd721
@ -409,118 +409,56 @@ impl<'tcx> Inliner<'tcx> {
|
|||||||
debug!(" final inline threshold = {}", threshold);
|
debug!(" final inline threshold = {}", threshold);
|
||||||
|
|
||||||
// FIXME: Give a bonus to functions with only a single caller
|
// FIXME: Give a bonus to functions with only a single caller
|
||||||
let mut first_block = true;
|
let diverges = matches!(
|
||||||
let mut cost = 0;
|
callee_body.basic_blocks()[START_BLOCK].terminator().kind,
|
||||||
|
TerminatorKind::Unreachable | TerminatorKind::Call { target: None, .. }
|
||||||
|
);
|
||||||
|
if diverges && !matches!(callee_attrs.inline, InlineAttr::Always) {
|
||||||
|
return Err("callee diverges unconditionally");
|
||||||
|
}
|
||||||
|
|
||||||
// Traverse the MIR manually so we can account for the effects of
|
let mut checker = CostChecker {
|
||||||
// inlining on the CFG.
|
tcx: self.tcx,
|
||||||
|
param_env: self.param_env,
|
||||||
|
instance: callsite.callee,
|
||||||
|
callee_body,
|
||||||
|
cost: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Traverse the MIR manually so we can account for the effects of inlining on the CFG.
|
||||||
let mut work_list = vec![START_BLOCK];
|
let mut work_list = vec![START_BLOCK];
|
||||||
let mut visited = BitSet::new_empty(callee_body.basic_blocks().len());
|
let mut visited = BitSet::new_empty(callee_body.basic_blocks().len());
|
||||||
while let Some(bb) = work_list.pop() {
|
while let Some(bb) = work_list.pop() {
|
||||||
if !visited.insert(bb.index()) {
|
if !visited.insert(bb.index()) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
let blk = &callee_body.basic_blocks()[bb];
|
let blk = &callee_body.basic_blocks()[bb];
|
||||||
|
checker.visit_basic_block_data(bb, blk);
|
||||||
|
|
||||||
for stmt in &blk.statements {
|
|
||||||
// Don't count StorageLive/StorageDead in the inlining cost.
|
|
||||||
match stmt.kind {
|
|
||||||
StatementKind::StorageLive(_)
|
|
||||||
| StatementKind::StorageDead(_)
|
|
||||||
| StatementKind::Deinit(_)
|
|
||||||
| StatementKind::Nop => {}
|
|
||||||
_ => cost += INSTR_COST,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let term = blk.terminator();
|
let term = blk.terminator();
|
||||||
let mut is_drop = false;
|
if let TerminatorKind::Drop { ref place, target, unwind }
|
||||||
match term.kind {
|
| TerminatorKind::DropAndReplace { ref place, target, unwind, .. } = term.kind
|
||||||
TerminatorKind::Drop { ref place, target, unwind }
|
{
|
||||||
| TerminatorKind::DropAndReplace { ref place, target, unwind, .. } => {
|
work_list.push(target);
|
||||||
is_drop = true;
|
|
||||||
work_list.push(target);
|
|
||||||
// If the place doesn't actually need dropping, treat it like
|
|
||||||
// a regular goto.
|
|
||||||
let ty = callsite.callee.subst_mir(self.tcx, &place.ty(callee_body, tcx).ty);
|
|
||||||
if ty.needs_drop(tcx, self.param_env) {
|
|
||||||
cost += CALL_PENALTY;
|
|
||||||
if let Some(unwind) = unwind {
|
|
||||||
cost += LANDINGPAD_PENALTY;
|
|
||||||
work_list.push(unwind);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
cost += INSTR_COST;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
TerminatorKind::Unreachable | TerminatorKind::Call { target: None, .. }
|
// If the place doesn't actually need dropping, treat it like a regular goto.
|
||||||
if first_block =>
|
let ty = callsite.callee.subst_mir(self.tcx, &place.ty(callee_body, tcx).ty);
|
||||||
{
|
if ty.needs_drop(tcx, self.param_env) && let Some(unwind) = unwind {
|
||||||
// If the function always diverges, don't inline
|
work_list.push(unwind);
|
||||||
// unless the cost is zero
|
|
||||||
threshold = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
TerminatorKind::Call { func: Operand::Constant(ref f), cleanup, .. } => {
|
|
||||||
if let ty::FnDef(def_id, _) =
|
|
||||||
*callsite.callee.subst_mir(self.tcx, &f.literal.ty()).kind()
|
|
||||||
{
|
|
||||||
// Don't give intrinsics the extra penalty for calls
|
|
||||||
if tcx.is_intrinsic(def_id) {
|
|
||||||
cost += INSTR_COST;
|
|
||||||
} else {
|
|
||||||
cost += CALL_PENALTY;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
cost += CALL_PENALTY;
|
|
||||||
}
|
}
|
||||||
if cleanup.is_some() {
|
} else {
|
||||||
cost += LANDINGPAD_PENALTY;
|
work_list.extend(term.successors())
|
||||||
}
|
|
||||||
}
|
|
||||||
TerminatorKind::Assert { cleanup, .. } => {
|
|
||||||
cost += CALL_PENALTY;
|
|
||||||
|
|
||||||
if cleanup.is_some() {
|
|
||||||
cost += LANDINGPAD_PENALTY;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
TerminatorKind::Resume => cost += RESUME_PENALTY,
|
|
||||||
TerminatorKind::InlineAsm { cleanup, .. } => {
|
|
||||||
cost += INSTR_COST;
|
|
||||||
|
|
||||||
if cleanup.is_some() {
|
|
||||||
cost += LANDINGPAD_PENALTY;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ => cost += INSTR_COST,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if !is_drop {
|
|
||||||
for succ in term.successors() {
|
|
||||||
work_list.push(succ);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
first_block = false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Count up the cost of local variables and temps, if we know the size
|
// Count up the cost of local variables and temps, if we know the size
|
||||||
// use that, otherwise we use a moderately-large dummy cost.
|
// use that, otherwise we use a moderately-large dummy cost.
|
||||||
|
|
||||||
let ptr_size = tcx.data_layout.pointer_size.bytes();
|
|
||||||
|
|
||||||
for v in callee_body.vars_and_temps_iter() {
|
for v in callee_body.vars_and_temps_iter() {
|
||||||
let ty = callsite.callee.subst_mir(self.tcx, &callee_body.local_decls[v].ty);
|
checker.visit_local_decl(v, &callee_body.local_decls[v]);
|
||||||
// Cost of the var is the size in machine-words, if we know
|
|
||||||
// it.
|
|
||||||
if let Some(size) = type_size_of(tcx, self.param_env, ty) {
|
|
||||||
cost += ((size + ptr_size - 1) / ptr_size) as usize;
|
|
||||||
} else {
|
|
||||||
cost += UNKNOWN_SIZE_COST;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let cost = checker.cost;
|
||||||
if let InlineAttr::Always = callee_attrs.inline {
|
if let InlineAttr::Always = callee_attrs.inline {
|
||||||
debug!("INLINING {:?} because inline(always) [cost={}]", callsite, cost);
|
debug!("INLINING {:?} because inline(always) [cost={}]", callsite, cost);
|
||||||
Ok(())
|
Ok(())
|
||||||
@ -790,6 +728,98 @@ fn type_size_of<'tcx>(
|
|||||||
tcx.layout_of(param_env.and(ty)).ok().map(|layout| layout.size.bytes())
|
tcx.layout_of(param_env.and(ty)).ok().map(|layout| layout.size.bytes())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Verify that the callee body is compatible with the caller.
|
||||||
|
///
|
||||||
|
/// This visitor mostly computes the inlining cost,
|
||||||
|
/// but also needs to verify that types match because of normalization failure.
|
||||||
|
struct CostChecker<'b, 'tcx> {
|
||||||
|
tcx: TyCtxt<'tcx>,
|
||||||
|
param_env: ParamEnv<'tcx>,
|
||||||
|
cost: usize,
|
||||||
|
callee_body: &'b Body<'tcx>,
|
||||||
|
instance: ty::Instance<'tcx>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
|
||||||
|
fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) {
|
||||||
|
// Don't count StorageLive/StorageDead in the inlining cost.
|
||||||
|
match statement.kind {
|
||||||
|
StatementKind::StorageLive(_)
|
||||||
|
| StatementKind::StorageDead(_)
|
||||||
|
| StatementKind::Deinit(_)
|
||||||
|
| StatementKind::Nop => {}
|
||||||
|
_ => self.cost += INSTR_COST,
|
||||||
|
}
|
||||||
|
|
||||||
|
self.super_statement(statement, location);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
|
||||||
|
let tcx = self.tcx;
|
||||||
|
match terminator.kind {
|
||||||
|
TerminatorKind::Drop { ref place, unwind, .. }
|
||||||
|
| TerminatorKind::DropAndReplace { ref place, unwind, .. } => {
|
||||||
|
// If the place doesn't actually need dropping, treat it like a regular goto.
|
||||||
|
let ty = self.instance.subst_mir(tcx, &place.ty(self.callee_body, tcx).ty);
|
||||||
|
if ty.needs_drop(tcx, self.param_env) {
|
||||||
|
self.cost += CALL_PENALTY;
|
||||||
|
if unwind.is_some() {
|
||||||
|
self.cost += LANDINGPAD_PENALTY;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
self.cost += INSTR_COST;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
TerminatorKind::Call { func: Operand::Constant(ref f), cleanup, .. } => {
|
||||||
|
let fn_ty = self.instance.subst_mir(tcx, &f.literal.ty());
|
||||||
|
self.cost += if let ty::FnDef(def_id, _) = *fn_ty.kind() && tcx.is_intrinsic(def_id) {
|
||||||
|
// Don't give intrinsics the extra penalty for calls
|
||||||
|
INSTR_COST
|
||||||
|
} else {
|
||||||
|
CALL_PENALTY
|
||||||
|
};
|
||||||
|
if cleanup.is_some() {
|
||||||
|
self.cost += LANDINGPAD_PENALTY;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
TerminatorKind::Assert { cleanup, .. } => {
|
||||||
|
self.cost += CALL_PENALTY;
|
||||||
|
if cleanup.is_some() {
|
||||||
|
self.cost += LANDINGPAD_PENALTY;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
TerminatorKind::Resume => self.cost += RESUME_PENALTY,
|
||||||
|
TerminatorKind::InlineAsm { cleanup, .. } => {
|
||||||
|
self.cost += INSTR_COST;
|
||||||
|
if cleanup.is_some() {
|
||||||
|
self.cost += LANDINGPAD_PENALTY;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => self.cost += INSTR_COST,
|
||||||
|
}
|
||||||
|
|
||||||
|
self.super_terminator(terminator, location);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Count up the cost of local variables and temps, if we know the size
|
||||||
|
/// use that, otherwise we use a moderately-large dummy cost.
|
||||||
|
fn visit_local_decl(&mut self, local: Local, local_decl: &LocalDecl<'tcx>) {
|
||||||
|
let tcx = self.tcx;
|
||||||
|
let ptr_size = tcx.data_layout.pointer_size.bytes();
|
||||||
|
|
||||||
|
let ty = self.instance.subst_mir(tcx, &local_decl.ty);
|
||||||
|
// Cost of the var is the size in machine-words, if we know
|
||||||
|
// it.
|
||||||
|
if let Some(size) = type_size_of(tcx, self.param_env, ty) {
|
||||||
|
self.cost += ((size + ptr_size - 1) / ptr_size) as usize;
|
||||||
|
} else {
|
||||||
|
self.cost += UNKNOWN_SIZE_COST;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.super_local_decl(local, local_decl)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Integrator.
|
* Integrator.
|
||||||
*
|
*
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user