From bf7dbff9210497ca3db0b19b5ca0c6daed47e64e Mon Sep 17 00:00:00 2001 From: lcnr Date: Tue, 17 Jan 2023 10:21:30 +0100 Subject: [PATCH] instantiate canonical vars eagerly --- compiler/rustc_middle/src/ty/sty.rs | 11 - .../src/solve/assembly.rs | 78 ++--- .../rustc_trait_selection/src/solve/cache.rs | 267 ------------------ .../src/solve/fulfill.rs | 26 +- .../src/solve/infcx_ext.rs | 37 --- .../rustc_trait_selection/src/solve/mod.rs | 212 ++++++++------ .../src/solve/project_goals.rs | 36 +-- .../src/solve/search_graph/cache.rs | 115 ++++++++ .../src/solve/search_graph/mod.rs | 172 +++++++++++ .../src/solve/{ => search_graph}/overflow.rs | 29 +- .../src/solve/trait_goals.rs | 22 +- 11 files changed, 509 insertions(+), 496 deletions(-) delete mode 100644 compiler/rustc_trait_selection/src/solve/cache.rs create mode 100644 compiler/rustc_trait_selection/src/solve/search_graph/cache.rs create mode 100644 compiler/rustc_trait_selection/src/solve/search_graph/mod.rs rename compiler/rustc_trait_selection/src/solve/{ => search_graph}/overflow.rs (74%) diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index 3f8252aefdc..4166f79c96c 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -1106,17 +1106,6 @@ pub fn no_bound_vars(self) -> Option if self.0.has_escaping_bound_vars() { None } else { Some(self.skip_binder()) } } - pub fn no_bound_vars_ignoring_escaping(self, tcx: TyCtxt<'tcx>) -> Option - where - T: TypeFoldable<'tcx>, - { - if !self.0.has_escaping_bound_vars() { - Some(self.skip_binder()) - } else { - self.0.try_fold_with(&mut SkipBindersAt { index: ty::INNERMOST, tcx }).ok() - } - } - /// Splits the contents into two things that share the same binder /// level as the original, returning two distinct binders. /// diff --git a/compiler/rustc_trait_selection/src/solve/assembly.rs b/compiler/rustc_trait_selection/src/solve/assembly.rs index ba68da0686f..da6bb844a0f 100644 --- a/compiler/rustc_trait_selection/src/solve/assembly.rs +++ b/compiler/rustc_trait_selection/src/solve/assembly.rs @@ -1,20 +1,11 @@ //! Code shared by trait and projection goals for candidate assembly. use super::infcx_ext::InferCtxtExt; -use super::{ - instantiate_canonical_query_response, CanonicalGoal, CanonicalResponse, Certainty, EvalCtxt, - Goal, -}; +use super::{CanonicalResponse, Certainty, EvalCtxt, Goal}; use rustc_hir::def_id::DefId; -use rustc_infer::infer::TyCtxtInferExt; -use rustc_infer::infer::{ - canonical::{CanonicalVarValues, OriginalQueryValues}, - InferCtxt, -}; use rustc_infer::traits::query::NoSolution; use rustc_middle::ty::TypeFoldable; use rustc_middle::ty::{self, Ty, TyCtxt}; -use rustc_span::DUMMY_SP; use std::fmt::Debug; /// A candidate is a possible way to prove a goal. @@ -40,7 +31,7 @@ pub(super) trait GoalKind<'tcx>: TypeFoldable<'tcx> + Copy { fn trait_def_id(self, tcx: TyCtxt<'tcx>) -> DefId; fn consider_impl_candidate( - acx: &mut AssemblyCtxt<'_, 'tcx, Self>, + acx: &mut AssemblyCtxt<'_, '_, 'tcx, Self>, goal: Goal<'tcx, Self>, impl_def_id: DefId, ); @@ -49,21 +40,17 @@ fn consider_impl_candidate( /// An abstraction which correctly deals with the canonical results for candidates. /// /// It also deduplicates the behavior between trait and projection predicates. -pub(super) struct AssemblyCtxt<'a, 'tcx, G: GoalKind<'tcx>> { - pub(super) cx: &'a mut EvalCtxt<'tcx>, - pub(super) infcx: &'a InferCtxt<'tcx>, - var_values: CanonicalVarValues<'tcx>, +pub(super) struct AssemblyCtxt<'a, 'b, 'tcx, G: GoalKind<'tcx>> { + pub(super) cx: &'a mut EvalCtxt<'b, 'tcx>, candidates: Vec>, } -impl<'a, 'tcx, G: GoalKind<'tcx>> AssemblyCtxt<'a, 'tcx, G> { +impl<'a, 'b, 'tcx, G: GoalKind<'tcx>> AssemblyCtxt<'a, 'b, 'tcx, G> { pub(super) fn assemble_and_evaluate_candidates( - cx: &'a mut EvalCtxt<'tcx>, - goal: CanonicalGoal<'tcx, G>, + cx: &'a mut EvalCtxt<'b, 'tcx>, + goal: Goal<'tcx, G>, ) -> Vec> { - let (ref infcx, goal, var_values) = - cx.tcx.infer_ctxt().build_with_canonical(DUMMY_SP, &goal); - let mut acx = AssemblyCtxt { cx, infcx, var_values, candidates: Vec::new() }; + let mut acx = AssemblyCtxt { cx, candidates: Vec::new() }; acx.assemble_candidates_after_normalizing_self_ty(goal); @@ -77,7 +64,7 @@ pub(super) fn try_insert_candidate( source: G::CandidateSource, certainty: Certainty, ) { - match self.infcx.make_canonical_response(self.var_values.clone(), certainty) { + match self.cx.make_canonical_response(certainty) { Ok(result) => self.candidates.push(Candidate { source, result }), Err(NoSolution) => debug!(?source, ?certainty, "failed leakcheck"), } @@ -89,13 +76,14 @@ pub(super) fn try_insert_candidate( /// self type to the list of candidates in case that succeeds. Note that we can't just eagerly return in /// this case as projections as self types add ` fn assemble_candidates_after_normalizing_self_ty(&mut self, goal: Goal<'tcx, G>) { - let tcx = self.cx.tcx; + let tcx = self.cx.tcx(); + let infcx = self.cx.infcx; // FIXME: We also have to normalize opaque types, not sure where to best fit that in. let &ty::Alias(ty::Projection, projection_ty) = goal.predicate.self_ty().kind() else { return }; - self.infcx.probe(|_| { - let normalized_ty = self.infcx.next_ty_infer(); + infcx.probe(|_| { + let normalized_ty = infcx.next_ty_infer(); let normalizes_to_goal = goal.with( tcx, ty::Binder::dummy(ty::ProjectionPredicate { @@ -103,43 +91,31 @@ fn assemble_candidates_after_normalizing_self_ty(&mut self, goal: Goal<'tcx, G>) term: normalized_ty.into(), }), ); - let normalization_certainty = - match self.cx.evaluate_goal(&self.infcx, normalizes_to_goal) { - Ok((_, certainty)) => certainty, - Err(NoSolution) => return, - }; + let normalization_certainty = match self.cx.evaluate_goal(normalizes_to_goal) { + Ok((_, certainty)) => certainty, + Err(NoSolution) => return, + }; // NOTE: Alternatively we could call `evaluate_goal` here and only have a `Normalized` candidate. // This doesn't work as long as we use `CandidateSource` in both winnowing and to resolve associated items. let goal = goal.with(tcx, goal.predicate.with_self_ty(tcx, normalized_ty)); - let mut orig_values = OriginalQueryValues::default(); - let goal = self.infcx.canonicalize_query(goal, &mut orig_values); let normalized_candidates = AssemblyCtxt::assemble_and_evaluate_candidates(self.cx, goal); - - // Map each candidate from being canonical wrt the current inference context to being - // canonical wrt the caller. - for Candidate { source, result } in normalized_candidates { - self.infcx.probe(|_| { - let candidate_certainty = - instantiate_canonical_query_response(&self.infcx, &orig_values, result); - - // FIXME: This is a bit scary if the `normalizes_to_goal` overflows. - // - // If we have an ambiguous candidate it hides that normalization - // caused an overflow which may cause issues. - self.try_insert_candidate( - source, - normalization_certainty.unify_and(candidate_certainty), - ) - }) + for mut normalized_candidate in normalized_candidates { + normalized_candidate.result = + normalized_candidate.result.unchecked_map(|mut response| { + response.certainty = response.certainty.unify_and(normalization_certainty); + response + }); + self.candidates.push(normalized_candidate); } }) } fn assemble_impl_candidates(&mut self, goal: Goal<'tcx, G>) { - self.cx.tcx.for_each_relevant_impl( - goal.predicate.trait_def_id(self.cx.tcx), + let tcx = self.cx.tcx(); + tcx.for_each_relevant_impl( + goal.predicate.trait_def_id(tcx), goal.predicate.self_ty(), |impl_def_id| G::consider_impl_candidate(self, goal, impl_def_id), ); diff --git a/compiler/rustc_trait_selection/src/solve/cache.rs b/compiler/rustc_trait_selection/src/solve/cache.rs deleted file mode 100644 index 9ac629980eb..00000000000 --- a/compiler/rustc_trait_selection/src/solve/cache.rs +++ /dev/null @@ -1,267 +0,0 @@ -//! This module both handles the global cache which stores "finished" goals, -//! and the provisional cache which contains partially computed goals. -//! -//! The provisional cache is necessary when dealing with coinductive cycles. -//! -//! For more information about the provisional cache and coinduction in general, -//! check out the relevant section of the rustc-dev-guide. -//! -//! FIXME(@lcnr): Write that section, feel free to ping me if you need help here -//! before then or if I still haven't done that before January 2023. -use super::overflow::OverflowData; -use super::{CanonicalGoal, Certainty, MaybeCause, Response}; -use super::{EvalCtxt, QueryResult}; -use rustc_data_structures::fx::FxHashMap; -use rustc_index::vec::IndexVec; -use rustc_infer::infer::canonical::{Canonical, CanonicalVarKind, CanonicalVarValues}; -use rustc_middle::ty::{self, TyCtxt}; -use std::collections::hash_map::Entry; - -rustc_index::newtype_index! { - pub struct StackDepth {} -} -rustc_index::newtype_index! { - pub struct EntryIndex {} -} - -#[derive(Debug, Clone)] -struct ProvisionalEntry<'tcx> { - // In case we have a coinductive cycle, this is the - // the currently least restrictive result of this goal. - response: QueryResult<'tcx>, - // In case of a cycle, the depth of lowest stack entry involved - // in that cycle. This is monotonically decreasing in the stack as all - // elements between the current stack element in the lowest stack entry - // involved have to also be involved in that cycle. - // - // We can only move entries to the global cache once we're complete done - // with the cycle. If this entry has not been involved in a cycle, - // this is just its own depth. - depth: StackDepth, - - // The goal for this entry. Should always be equal to the corresponding goal - // in the lookup table. - goal: CanonicalGoal<'tcx>, -} - -struct StackElem<'tcx> { - goal: CanonicalGoal<'tcx>, - has_been_used: bool, -} - -pub(super) struct ProvisionalCache<'tcx> { - stack: IndexVec>, - entries: IndexVec>, - // FIXME: This is only used to quickly check whether a given goal - // is in the cache. We should experiment with using something like - // `SsoHashSet` here because in most cases there are only a few entries. - lookup_table: FxHashMap, EntryIndex>, -} - -impl<'tcx> ProvisionalCache<'tcx> { - pub(super) fn empty() -> ProvisionalCache<'tcx> { - ProvisionalCache { - stack: Default::default(), - entries: Default::default(), - lookup_table: Default::default(), - } - } - - pub(super) fn current_depth(&self) -> usize { - self.stack.len() - } -} - -impl<'tcx> EvalCtxt<'tcx> { - /// Tries putting the new goal on the stack, returning an error if it is already cached. - /// - /// This correctly updates the provisional cache if there is a cycle. - pub(super) fn try_push_stack( - &mut self, - goal: CanonicalGoal<'tcx>, - ) -> Result<(), QueryResult<'tcx>> { - // FIXME: start by checking the global cache - - // Look at the provisional cache to check for cycles. - let cache = &mut self.provisional_cache; - match cache.lookup_table.entry(goal) { - // No entry, simply push this goal on the stack after dealing with overflow. - Entry::Vacant(v) => { - if self.overflow_data.has_overflow(cache.stack.len()) { - return Err(self.deal_with_overflow(goal)); - } - - let depth = cache.stack.push(StackElem { goal, has_been_used: false }); - let response = response_no_constraints(self.tcx, goal, Certainty::Yes); - let entry_index = cache.entries.push(ProvisionalEntry { response, depth, goal }); - v.insert(entry_index); - Ok(()) - } - // We have a nested goal which relies on a goal `root` deeper in the stack. - // - // We first store that we may have to rerun `evaluate_goal` for `root` in case the - // provisional response is not equal to the final response. We also update the depth - // of all goals which recursively depend on our current goal to depend on `root` - // instead. - // - // Finally we can return either the provisional response for that goal if we have a - // coinductive cycle or an ambiguous result if the cycle is inductive. - Entry::Occupied(entry_index) => { - let entry_index = *entry_index.get(); - // FIXME `ProvisionalEntry` should be `Copy`. - let entry = cache.entries.get(entry_index).unwrap().clone(); - cache.stack[entry.depth].has_been_used = true; - for provisional_entry in cache.entries.iter_mut().skip(entry_index.index()) { - provisional_entry.depth = provisional_entry.depth.min(entry.depth); - } - - // NOTE: The goals on the stack aren't the only goals involved in this cycle. - // We can also depend on goals which aren't part of the stack but coinductively - // depend on the stack themselves. We already checked whether all the goals - // between these goals and their root on the stack. This means that as long as - // each goal in a cycle is checked for coinductivity by itself, simply checking - // the stack is enough. - if cache.stack.raw[entry.depth.index()..] - .iter() - .all(|g| g.goal.value.predicate.is_coinductive(self.tcx)) - { - Err(entry.response) - } else { - Err(response_no_constraints( - self.tcx, - goal, - Certainty::Maybe(MaybeCause::Overflow), - )) - } - } - } - } - - /// We cannot simply store the result of [EvalCtxt::compute_goal] as we have to deal with - /// coinductive cycles. - /// - /// When we encounter a coinductive cycle, we have to prove the final result of that cycle - /// while we are still computing that result. Because of this we continously recompute the - /// cycle until the result of the previous iteration is equal to the final result, at which - /// point we are done. - /// - /// This function returns `true` if we were able to finalize the goal and `false` if it has - /// updated the provisional cache and we have to recompute the current goal. - /// - /// FIXME: Refer to the rustc-dev-guide entry once it exists. - pub(super) fn try_finalize_goal( - &mut self, - actual_goal: CanonicalGoal<'tcx>, - response: QueryResult<'tcx>, - ) -> bool { - let cache = &mut self.provisional_cache; - let StackElem { goal, has_been_used } = cache.stack.pop().unwrap(); - assert_eq!(goal, actual_goal); - - let provisional_entry_index = *cache.lookup_table.get(&goal).unwrap(); - let provisional_entry = &mut cache.entries[provisional_entry_index]; - // Was the current goal the root of a cycle and was the provisional response - // different from the final one. - if has_been_used && provisional_entry.response != response { - // If so, update the provisional reponse for this goal... - provisional_entry.response = response; - // ...remove all entries whose result depends on this goal - // from the provisional cache... - // - // That's not completely correct, as a nested goal can also - // depend on a goal which is lower in the stack so it doesn't - // actually depend on the current goal. This should be fairly - // rare and is hopefully not relevant for performance. - #[allow(rustc::potential_query_instability)] - cache.lookup_table.retain(|_key, index| *index <= provisional_entry_index); - cache.entries.truncate(provisional_entry_index.index() + 1); - - // ...and finally push our goal back on the stack and reevaluate it. - cache.stack.push(StackElem { goal, has_been_used: false }); - false - } else { - // If not, we're done with this goal. - // - // Check whether that this goal doesn't depend on a goal deeper on the stack - // and if so, move it and all nested goals to the global cache. - // - // Note that if any nested goal were to depend on something deeper on the stack, - // this would have also updated the depth of this goal. - if provisional_entry.depth == cache.stack.next_index() { - for (i, entry) in cache.entries.drain_enumerated(provisional_entry_index.index()..) - { - let actual_index = cache.lookup_table.remove(&entry.goal); - debug_assert_eq!(Some(i), actual_index); - Self::try_move_finished_goal_to_global_cache( - self.tcx, - &mut self.overflow_data, - &cache.stack, - entry.goal, - entry.response, - ); - } - } - true - } - } - - fn try_move_finished_goal_to_global_cache( - tcx: TyCtxt<'tcx>, - overflow_data: &mut OverflowData, - stack: &IndexVec>, - goal: CanonicalGoal<'tcx>, - response: QueryResult<'tcx>, - ) { - // We move goals to the global cache if we either did not hit an overflow or if it's - // the root goal as that will now always hit the same overflow limit. - // - // NOTE: We cannot move any non-root goals to the global cache even if their final result - // isn't impacted by the overflow as that goal still has unstable query dependencies - // because it didn't go its full depth. - // - // FIXME(@lcnr): We could still cache subtrees which are not impacted by overflow though. - // Tracking that info correctly isn't trivial, so I haven't implemented it for now. - let should_cache_globally = !overflow_data.did_overflow() || stack.is_empty(); - if should_cache_globally { - // FIXME: move the provisional entry to the global cache. - let _ = (tcx, goal, response); - } - } -} - -pub(super) fn response_no_constraints<'tcx>( - tcx: TyCtxt<'tcx>, - goal: Canonical<'tcx, impl Sized>, - certainty: Certainty, -) -> QueryResult<'tcx> { - let var_values = goal - .variables - .iter() - .enumerate() - .map(|(i, info)| match info.kind { - CanonicalVarKind::Ty(_) | CanonicalVarKind::PlaceholderTy(_) => { - tcx.mk_ty(ty::Bound(ty::INNERMOST, ty::BoundVar::from_usize(i).into())).into() - } - CanonicalVarKind::Region(_) | CanonicalVarKind::PlaceholderRegion(_) => { - let br = ty::BoundRegion { - var: ty::BoundVar::from_usize(i), - kind: ty::BrAnon(i as u32, None), - }; - tcx.mk_region(ty::ReLateBound(ty::INNERMOST, br)).into() - } - CanonicalVarKind::Const(_, ty) | CanonicalVarKind::PlaceholderConst(_, ty) => tcx - .mk_const(ty::ConstKind::Bound(ty::INNERMOST, ty::BoundVar::from_usize(i)), ty) - .into(), - }) - .collect(); - - Ok(Canonical { - max_universe: goal.max_universe, - variables: goal.variables, - value: Response { - var_values: CanonicalVarValues { var_values }, - external_constraints: Default::default(), - certainty, - }, - }) -} diff --git a/compiler/rustc_trait_selection/src/solve/fulfill.rs b/compiler/rustc_trait_selection/src/solve/fulfill.rs index dfc2b5ed329..3146f468f7d 100644 --- a/compiler/rustc_trait_selection/src/solve/fulfill.rs +++ b/compiler/rustc_trait_selection/src/solve/fulfill.rs @@ -2,7 +2,7 @@ use rustc_data_structures::fx::FxHashMap; use rustc_infer::{ - infer::InferCtxt, + infer::{canonical::OriginalQueryValues, InferCtxt}, traits::{ query::NoSolution, FulfillmentError, FulfillmentErrorCode, PredicateObligation, SelectionError, TraitEngine, @@ -67,10 +67,26 @@ fn select_where_possible(&mut self, infcx: &InferCtxt<'tcx>) -> Vec result, + let goal = obligation.clone().into(); + + // FIXME: Add a better API for that '^^ + let mut orig_values = OriginalQueryValues::default(); + let canonical_goal = infcx.canonicalize_query(goal, &mut orig_values); + let (changed, certainty) = match EvalCtxt::evaluate_canonical_goal( + infcx.tcx, + &mut super::search_graph::SearchGraph::new(infcx.tcx), + canonical_goal, + ) { + Ok(canonical_response) => { + ( + true, // FIXME: check whether `var_values` are an identity substitution. + super::instantiate_canonical_query_response( + infcx, + &orig_values, + canonical_response, + ), + ) + } Err(NoSolution) => { errors.push(FulfillmentError { obligation: obligation.clone(), diff --git a/compiler/rustc_trait_selection/src/solve/infcx_ext.rs b/compiler/rustc_trait_selection/src/solve/infcx_ext.rs index 436f4eea662..8a8c3091d54 100644 --- a/compiler/rustc_trait_selection/src/solve/infcx_ext.rs +++ b/compiler/rustc_trait_selection/src/solve/infcx_ext.rs @@ -1,23 +1,11 @@ -use rustc_infer::infer::canonical::CanonicalVarValues; use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind}; use rustc_infer::infer::InferCtxt; -use rustc_infer::traits::query::NoSolution; use rustc_middle::ty::Ty; use rustc_span::DUMMY_SP; -use crate::solve::ExternalConstraints; - -use super::{Certainty, QueryResult, Response}; - /// Methods used inside of the canonical queries of the solver. pub(super) trait InferCtxtExt<'tcx> { fn next_ty_infer(&self) -> Ty<'tcx>; - - fn make_canonical_response( - &self, - var_values: CanonicalVarValues<'tcx>, - certainty: Certainty, - ) -> QueryResult<'tcx>; } impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> { @@ -27,29 +15,4 @@ fn next_ty_infer(&self) -> Ty<'tcx> { span: DUMMY_SP, }) } - - fn make_canonical_response( - &self, - var_values: CanonicalVarValues<'tcx>, - certainty: Certainty, - ) -> QueryResult<'tcx> { - let external_constraints = take_external_constraints(self)?; - - Ok(self.canonicalize_response(Response { var_values, external_constraints, certainty })) - } -} - -#[instrument(level = "debug", skip(infcx), ret)] -fn take_external_constraints<'tcx>( - infcx: &InferCtxt<'tcx>, -) -> Result, NoSolution> { - let region_obligations = infcx.take_registered_region_obligations(); - let opaque_types = infcx.take_opaque_types_for_query_response(); - Ok(ExternalConstraints { - // FIXME: Now that's definitely wrong :) - // - // Should also do the leak check here I think - regions: drop(region_obligations), - opaque_types, - }) } diff --git a/compiler/rustc_trait_selection/src/solve/mod.rs b/compiler/rustc_trait_selection/src/solve/mod.rs index 80775b7aaf2..0e629664b2b 100644 --- a/compiler/rustc_trait_selection/src/solve/mod.rs +++ b/compiler/rustc_trait_selection/src/solve/mod.rs @@ -19,27 +19,23 @@ use std::mem; +use rustc_infer::infer::canonical::{Canonical, CanonicalVarKind, CanonicalVarValues}; use rustc_infer::infer::canonical::{OriginalQueryValues, QueryRegionConstraints, QueryResponse}; use rustc_infer::infer::{InferCtxt, InferOk, TyCtxtInferExt}; use rustc_infer::traits::query::NoSolution; use rustc_infer::traits::Obligation; use rustc_middle::infer::canonical::Certainty as OldCertainty; -use rustc_middle::infer::canonical::{Canonical, CanonicalVarValues}; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_middle::ty::{RegionOutlivesPredicate, ToPredicate, TypeOutlivesPredicate}; use rustc_span::DUMMY_SP; use crate::traits::ObligationCause; -use self::cache::response_no_constraints; -use self::infcx_ext::InferCtxtExt; - mod assembly; -mod cache; mod fulfill; mod infcx_ext; -mod overflow; mod project_goals; +mod search_graph; mod trait_goals; pub use fulfill::FulfillmentCtxt; @@ -146,45 +142,25 @@ pub trait TyCtxtExt<'tcx> { impl<'tcx> TyCtxtExt<'tcx> for TyCtxt<'tcx> { fn evaluate_goal(self, goal: CanonicalGoal<'tcx>) -> QueryResult<'tcx> { - let mut cx = EvalCtxt::new(self); - cx.evaluate_canonical_goal(goal) + let mut search_graph = search_graph::SearchGraph::new(self); + EvalCtxt::evaluate_canonical_goal(self, &mut search_graph, goal) } } -struct EvalCtxt<'tcx> { - tcx: TyCtxt<'tcx>, +struct EvalCtxt<'a, 'tcx> { + infcx: &'a InferCtxt<'tcx>, + var_values: CanonicalVarValues<'tcx>, - provisional_cache: cache::ProvisionalCache<'tcx>, - overflow_data: overflow::OverflowData, + search_graph: &'a mut search_graph::SearchGraph<'tcx>, } -impl<'tcx> EvalCtxt<'tcx> { - fn new(tcx: TyCtxt<'tcx>) -> EvalCtxt<'tcx> { - EvalCtxt { - tcx, - provisional_cache: cache::ProvisionalCache::empty(), - overflow_data: overflow::OverflowData::new(tcx), - } - } - - /// Recursively evaluates `goal`, returning whether any inference vars have - /// been constrained and the certainty of the result. - fn evaluate_goal( - &mut self, - infcx: &InferCtxt<'tcx>, - goal: Goal<'tcx, ty::Predicate<'tcx>>, - ) -> Result<(bool, Certainty), NoSolution> { - let mut orig_values = OriginalQueryValues::default(); - let canonical_goal = infcx.canonicalize_query(goal, &mut orig_values); - let canonical_response = self.evaluate_canonical_goal(canonical_goal)?; - Ok(( - !canonical_response.value.var_values.is_identity(), - instantiate_canonical_query_response(infcx, &orig_values, canonical_response), - )) - } - - fn evaluate_canonical_goal(&mut self, goal: CanonicalGoal<'tcx>) -> QueryResult<'tcx> { - match self.try_push_stack(goal) { +impl<'a, 'tcx> EvalCtxt<'a, 'tcx> { + fn evaluate_canonical_goal( + tcx: TyCtxt<'tcx>, + search_graph: &'a mut search_graph::SearchGraph<'tcx>, + canonical_goal: CanonicalGoal<'tcx>, + ) -> QueryResult<'tcx> { + match search_graph.try_push_stack(tcx, canonical_goal) { Ok(()) => {} // Our goal is already on the stack, eager return. Err(response) => return response, @@ -195,41 +171,65 @@ fn evaluate_canonical_goal(&mut self, goal: CanonicalGoal<'tcx>) -> QueryResult< // // FIXME: Similar to `evaluate_all`, this has to check for overflow. loop { - let result = self.compute_goal(goal); + let (ref infcx, goal, var_values) = + tcx.infer_ctxt().build_with_canonical(DUMMY_SP, &canonical_goal); + let mut ecx = EvalCtxt { infcx, var_values, search_graph }; + let result = ecx.compute_goal(goal); // FIXME: `Response` should be `Copy` - if self.try_finalize_goal(goal, result.clone()) { + if search_graph.try_finalize_goal(tcx, canonical_goal, result.clone()) { return result; } } } - fn compute_goal(&mut self, canonical_goal: CanonicalGoal<'tcx>) -> QueryResult<'tcx> { - // WARNING: We're looking at a canonical value without instantiating it here. - // - // We have to be incredibly careful to not change the order of bound variables or - // remove any. As we go from `Goal<'tcx, Predicate>` to `Goal` with the variants - // of `PredicateKind` this is the case and it is and faster than instantiating and - // recanonicalizing. - let Goal { param_env, predicate } = canonical_goal.value; + fn tcx(&self) -> TyCtxt<'tcx> { + self.infcx.tcx + } - if let Some(kind) = predicate.kind().no_bound_vars_ignoring_escaping(self.tcx) { + fn make_canonical_response(&self, certainty: Certainty) -> QueryResult<'tcx> { + let external_constraints = take_external_constraints(self.infcx)?; + + Ok(self.infcx.canonicalize_response(Response { + var_values: self.var_values.clone(), + external_constraints, + certainty, + })) + } + + /// Recursively evaluates `goal`, returning whether any inference vars have + /// been constrained and the certainty of the result. + fn evaluate_goal( + &mut self, + goal: Goal<'tcx, ty::Predicate<'tcx>>, + ) -> Result<(bool, Certainty), NoSolution> { + let mut orig_values = OriginalQueryValues::default(); + let canonical_goal = self.infcx.canonicalize_query(goal, &mut orig_values); + let canonical_response = + EvalCtxt::evaluate_canonical_goal(self.tcx(), self.search_graph, canonical_goal)?; + Ok(( + !canonical_response.value.var_values.is_identity(), + instantiate_canonical_query_response(self.infcx, &orig_values, canonical_response), + )) + } + + fn compute_goal(&mut self, goal: Goal<'tcx, ty::Predicate<'tcx>>) -> QueryResult<'tcx> { + let Goal { param_env, predicate } = goal; + let kind = predicate.kind(); + if let Some(kind) = kind.no_bound_vars() { match kind { - ty::PredicateKind::Clause(ty::Clause::Trait(predicate)) => self.compute_trait_goal( - canonical_goal.unchecked_rebind(Goal { param_env, predicate }), - ), - ty::PredicateKind::Clause(ty::Clause::Projection(predicate)) => self - .compute_projection_goal( - canonical_goal.unchecked_rebind(Goal { param_env, predicate }), - ), - ty::PredicateKind::Clause(ty::Clause::TypeOutlives(predicate)) => self - .compute_type_outlives_goal( - canonical_goal.unchecked_rebind(Goal { param_env, predicate }), - ), - ty::PredicateKind::Clause(ty::Clause::RegionOutlives(predicate)) => self - .compute_region_outlives_goal( - canonical_goal.unchecked_rebind(Goal { param_env, predicate }), - ), + ty::PredicateKind::Clause(ty::Clause::Trait(predicate)) => { + self.compute_trait_goal(Goal { param_env, predicate }) + } + ty::PredicateKind::Clause(ty::Clause::Projection(predicate)) => { + self.compute_projection_goal(Goal { param_env, predicate }) + } + ty::PredicateKind::Clause(ty::Clause::TypeOutlives(predicate)) => { + self.compute_type_outlives_goal(Goal { param_env, predicate }) + } + ty::PredicateKind::Clause(ty::Clause::RegionOutlives(predicate)) => { + self.compute_region_outlives_goal(Goal { param_env, predicate }) + } // FIXME: implement these predicates :) ty::PredicateKind::WellFormed(_) | ty::PredicateKind::ObjectSafe(_) @@ -239,49 +239,41 @@ fn compute_goal(&mut self, canonical_goal: CanonicalGoal<'tcx>) -> QueryResult<' | ty::PredicateKind::ConstEvaluatable(_) | ty::PredicateKind::ConstEquate(_, _) | ty::PredicateKind::TypeWellFormedFromEnv(_) - | ty::PredicateKind::Ambiguous => { - // FIXME - response_no_constraints(self.tcx, canonical_goal, Certainty::Yes) - } + | ty::PredicateKind::Ambiguous => self.make_canonical_response(Certainty::Yes), } } else { - let (infcx, goal, var_values) = - self.tcx.infer_ctxt().build_with_canonical(DUMMY_SP, &canonical_goal); - let kind = infcx.replace_bound_vars_with_placeholders(goal.predicate.kind()); - let goal = goal.with(self.tcx, ty::Binder::dummy(kind)); - let (_, certainty) = self.evaluate_goal(&infcx, goal)?; - infcx.make_canonical_response(var_values, certainty) + let kind = self.infcx.replace_bound_vars_with_placeholders(kind); + let goal = goal.with(self.tcx(), ty::Binder::dummy(kind)); + let (_, certainty) = self.evaluate_goal(goal)?; + self.make_canonical_response(certainty) } } fn compute_type_outlives_goal( &mut self, - goal: CanonicalGoal<'tcx, TypeOutlivesPredicate<'tcx>>, + _goal: Goal<'tcx, TypeOutlivesPredicate<'tcx>>, ) -> QueryResult<'tcx> { - // FIXME - response_no_constraints(self.tcx, goal, Certainty::Yes) + self.make_canonical_response(Certainty::Yes) } fn compute_region_outlives_goal( &mut self, - goal: CanonicalGoal<'tcx, RegionOutlivesPredicate<'tcx>>, + _goal: Goal<'tcx, RegionOutlivesPredicate<'tcx>>, ) -> QueryResult<'tcx> { - // FIXME - response_no_constraints(self.tcx, goal, Certainty::Yes) + self.make_canonical_response(Certainty::Yes) } } -impl<'tcx> EvalCtxt<'tcx> { +impl<'tcx> EvalCtxt<'_, 'tcx> { fn evaluate_all( &mut self, - infcx: &InferCtxt<'tcx>, mut goals: Vec>>, ) -> Result { let mut new_goals = Vec::new(); self.repeat_while_none(|this| { let mut has_changed = Err(Certainty::Yes); for goal in goals.drain(..) { - let (changed, certainty) = match this.evaluate_goal(infcx, goal) { + let (changed, certainty) = match this.evaluate_goal(goal) { Ok(result) => result, Err(NoSolution) => return Some(Err(NoSolution)), }; @@ -310,6 +302,21 @@ fn evaluate_all( } } +#[instrument(level = "debug", skip(infcx), ret)] +fn take_external_constraints<'tcx>( + infcx: &InferCtxt<'tcx>, +) -> Result, NoSolution> { + let region_obligations = infcx.take_registered_region_obligations(); + let opaque_types = infcx.take_opaque_types_for_query_response(); + Ok(ExternalConstraints { + // FIXME: Now that's definitely wrong :) + // + // Should also do the leak check here I think + regions: drop(region_obligations), + opaque_types, + }) +} + fn instantiate_canonical_query_response<'tcx>( infcx: &InferCtxt<'tcx>, original_values: &OriginalQueryValues<'tcx>, @@ -334,3 +341,40 @@ fn instantiate_canonical_query_response<'tcx>( assert!(obligations.is_empty()); value } + +pub(super) fn response_no_constraints<'tcx>( + tcx: TyCtxt<'tcx>, + goal: Canonical<'tcx, impl Sized>, + certainty: Certainty, +) -> QueryResult<'tcx> { + let var_values = goal + .variables + .iter() + .enumerate() + .map(|(i, info)| match info.kind { + CanonicalVarKind::Ty(_) | CanonicalVarKind::PlaceholderTy(_) => { + tcx.mk_ty(ty::Bound(ty::INNERMOST, ty::BoundVar::from_usize(i).into())).into() + } + CanonicalVarKind::Region(_) | CanonicalVarKind::PlaceholderRegion(_) => { + let br = ty::BoundRegion { + var: ty::BoundVar::from_usize(i), + kind: ty::BrAnon(i as u32, None), + }; + tcx.mk_region(ty::ReLateBound(ty::INNERMOST, br)).into() + } + CanonicalVarKind::Const(_, ty) | CanonicalVarKind::PlaceholderConst(_, ty) => tcx + .mk_const(ty::ConstKind::Bound(ty::INNERMOST, ty::BoundVar::from_usize(i)), ty) + .into(), + }) + .collect(); + + Ok(Canonical { + max_universe: goal.max_universe, + variables: goal.variables, + value: Response { + var_values: CanonicalVarValues { var_values }, + external_constraints: Default::default(), + certainty, + }, + }) +} diff --git a/compiler/rustc_trait_selection/src/solve/project_goals.rs b/compiler/rustc_trait_selection/src/solve/project_goals.rs index d2f2e78f555..1120dfb8098 100644 --- a/compiler/rustc_trait_selection/src/solve/project_goals.rs +++ b/compiler/rustc_trait_selection/src/solve/project_goals.rs @@ -1,7 +1,7 @@ use crate::traits::{specialization_graph, translate_substs}; use super::assembly::{self, AssemblyCtxt}; -use super::{CanonicalGoal, EvalCtxt, Goal, QueryResult}; +use super::{EvalCtxt, Goal, QueryResult}; use rustc_errors::ErrorGuaranteed; use rustc_hir::def::DefKind; use rustc_hir::def_id::DefId; @@ -26,10 +26,10 @@ pub(super) enum CandidateSource { type Candidate<'tcx> = assembly::Candidate<'tcx, ProjectionPredicate<'tcx>>; -impl<'tcx> EvalCtxt<'tcx> { +impl<'tcx> EvalCtxt<'_, 'tcx> { pub(super) fn compute_projection_goal( &mut self, - goal: CanonicalGoal<'tcx, ProjectionPredicate<'tcx>>, + goal: Goal<'tcx, ProjectionPredicate<'tcx>>, ) -> QueryResult<'tcx> { let candidates = AssemblyCtxt::assemble_and_evaluate_candidates(self, goal); self.merge_project_candidates(candidates) @@ -104,11 +104,13 @@ fn trait_def_id(self, tcx: TyCtxt<'tcx>) -> DefId { } fn consider_impl_candidate( - acx: &mut AssemblyCtxt<'_, 'tcx, ProjectionPredicate<'tcx>>, + acx: &mut AssemblyCtxt<'_, '_, 'tcx, ProjectionPredicate<'tcx>>, goal: Goal<'tcx, ProjectionPredicate<'tcx>>, impl_def_id: DefId, ) { - let tcx = acx.cx.tcx; + let tcx = acx.cx.tcx(); + let infcx = acx.cx.infcx; + let goal_trait_ref = goal.predicate.projection_ty.trait_ref(tcx); let impl_trait_ref = tcx.impl_trait_ref(impl_def_id).unwrap(); let drcx = DeepRejectCtxt { treat_obligation_params: TreatParams::AsPlaceholder }; @@ -118,12 +120,11 @@ fn consider_impl_candidate( return; } - acx.infcx.probe(|_| { - let impl_substs = acx.infcx.fresh_substs_for_item(DUMMY_SP, impl_def_id); + infcx.probe(|_| { + let impl_substs = infcx.fresh_substs_for_item(DUMMY_SP, impl_def_id); let impl_trait_ref = impl_trait_ref.subst(tcx, impl_substs); - let Ok(InferOk { obligations, .. }) = acx - .infcx + let Ok(InferOk { obligations, .. }) = infcx .at(&ObligationCause::dummy(), goal.param_env) .define_opaque_types(false) .eq(goal_trait_ref, impl_trait_ref) @@ -138,11 +139,12 @@ fn consider_impl_candidate( .into_iter() .map(|pred| goal.with(tcx, pred)); - let nested_goals = obligations.into_iter().map(|o| o.into()).chain(where_clause_bounds).collect(); - let Ok(trait_ref_certainty) = acx.cx.evaluate_all(acx.infcx, nested_goals) else { return }; + let nested_goals = + obligations.into_iter().map(|o| o.into()).chain(where_clause_bounds).collect(); + let Ok(trait_ref_certainty) = acx.cx.evaluate_all(nested_goals) else { return }; let Some(assoc_def) = fetch_eligible_assoc_item_def( - acx.infcx, + infcx, goal.param_env, goal_trait_ref, goal.predicate.def_id(), @@ -174,7 +176,7 @@ fn consider_impl_candidate( impl_substs, ); let substs = translate_substs( - acx.infcx, + infcx, goal.param_env, impl_def_id, impl_substs_with_gat, @@ -185,7 +187,8 @@ fn consider_impl_candidate( let is_const = matches!(tcx.def_kind(assoc_def.item.def_id), DefKind::AssocConst); let ty = tcx.bound_type_of(assoc_def.item.def_id); let term: ty::EarlyBinder> = if is_const { - let identity_substs = ty::InternalSubsts::identity_for_item(tcx, assoc_def.item.def_id); + let identity_substs = + ty::InternalSubsts::identity_for_item(tcx, assoc_def.item.def_id); let did = ty::WithOptConstParam::unknown(assoc_def.item.def_id); let kind = ty::ConstKind::Unevaluated(ty::UnevaluatedConst::new(did, identity_substs)); @@ -194,8 +197,7 @@ fn consider_impl_candidate( ty.map_bound(|ty| ty.into()) }; - let Ok(InferOk { obligations, .. }) = acx - .infcx + let Ok(InferOk { obligations, .. }) = infcx .at(&ObligationCause::dummy(), goal.param_env) .define_opaque_types(false) .eq(goal.predicate.term, term.subst(tcx, substs)) @@ -205,7 +207,7 @@ fn consider_impl_candidate( }; let nested_goals = obligations.into_iter().map(|o| o.into()).collect(); - let Ok(rhs_certainty) = acx.cx.evaluate_all(acx.infcx, nested_goals) else { return }; + let Ok(rhs_certainty) = acx.cx.evaluate_all(nested_goals) else { return }; let certainty = trait_ref_certainty.unify_and(rhs_certainty); acx.try_insert_candidate(CandidateSource::Impl(impl_def_id), certainty); diff --git a/compiler/rustc_trait_selection/src/solve/search_graph/cache.rs b/compiler/rustc_trait_selection/src/solve/search_graph/cache.rs new file mode 100644 index 00000000000..435e46f211a --- /dev/null +++ b/compiler/rustc_trait_selection/src/solve/search_graph/cache.rs @@ -0,0 +1,115 @@ +//! This module both handles the global cache which stores "finished" goals, +//! and the provisional cache which contains partially computed goals. +//! +//! The provisional cache is necessary when dealing with coinductive cycles. +//! +//! For more information about the provisional cache and coinduction in general, +//! check out the relevant section of the rustc-dev-guide. +//! +//! FIXME(@lcnr): Write that section, feel free to ping me if you need help here +//! before then or if I still haven't done that before January 2023. +use super::overflow::OverflowData; +use super::StackDepth; +use crate::solve::{CanonicalGoal, QueryResult}; +use rustc_data_structures::fx::FxHashMap; +use rustc_index::vec::IndexVec; +use rustc_middle::ty::TyCtxt; + +rustc_index::newtype_index! { + pub struct EntryIndex {} +} + +#[derive(Debug, Clone)] +pub(super) struct ProvisionalEntry<'tcx> { + // In case we have a coinductive cycle, this is the + // the currently least restrictive result of this goal. + pub(super) response: QueryResult<'tcx>, + // In case of a cycle, the position of deepest stack entry involved + // in that cycle. This is monotonically decreasing in the stack as all + // elements between the current stack element in the deepest stack entry + // involved have to also be involved in that cycle. + // + // We can only move entries to the global cache once we're complete done + // with the cycle. If this entry has not been involved in a cycle, + // this is just its own depth. + pub(super) depth: StackDepth, + + // The goal for this entry. Should always be equal to the corresponding goal + // in the lookup table. + pub(super) goal: CanonicalGoal<'tcx>, +} + +pub(super) struct ProvisionalCache<'tcx> { + pub(super) entries: IndexVec>, + // FIXME: This is only used to quickly check whether a given goal + // is in the cache. We should experiment with using something like + // `SsoHashSet` here because in most cases there are only a few entries. + pub(super) lookup_table: FxHashMap, EntryIndex>, +} + +impl<'tcx> ProvisionalCache<'tcx> { + pub(super) fn empty() -> ProvisionalCache<'tcx> { + ProvisionalCache { entries: Default::default(), lookup_table: Default::default() } + } + + /// Adds a dependency from the current leaf to `target` in the cache + /// to prevent us from moving any goals which depend on the current leaf + /// to the global cache while we're still computing `target`. + pub(super) fn add_dependency_of_leaf_on(&mut self, target: EntryIndex) { + let depth = self.entries[target].depth; + for provisional_entry in &mut self.entries.raw[target.index()..] { + // The depth of `target` is the position of the deepest goal in the stack + // on which `target` depends. That goal is the `root` of this cycle. + // + // Any entry which was added after `target` is either on the stack itself + // at which point its depth is definitely at least as high as the depth of + // `root`. If it's not on the stack itself it has to depend on a goal + // between `root` and `leaf`. If it were to depend on a goal deeper in the + // stack than `root`, then `root` would also depend on that goal, at which + // point `root` wouldn't be the root anymore. + debug_assert!(provisional_entry.depth >= depth); + provisional_entry.depth = depth; + } + + // We only update entries which were added after `target` as no other + // entry should have a higher depth. + // + // Any entry which previously had a higher depth than target has to + // be between `target` and `root`. Because of this we would have updated + // its depth when calling `add_dependency_of_leaf_on(root)` for `target`. + if cfg!(debug_assertions) { + self.entries.iter().all(|e| e.depth <= depth); + } + } + + pub(super) fn depth(&self, entry_index: EntryIndex) -> StackDepth { + self.entries[entry_index].depth + } + + pub(super) fn provisional_result(&self, entry_index: EntryIndex) -> QueryResult<'tcx> { + self.entries[entry_index].response.clone() + } +} + +pub(super) fn try_move_finished_goal_to_global_cache<'tcx>( + tcx: TyCtxt<'tcx>, + overflow_data: &mut OverflowData, + stack: &IndexVec>, + goal: CanonicalGoal<'tcx>, + response: QueryResult<'tcx>, +) { + // We move goals to the global cache if we either did not hit an overflow or if it's + // the root goal as that will now always hit the same overflow limit. + // + // NOTE: We cannot move any non-root goals to the global cache even if their final result + // isn't impacted by the overflow as that goal still has unstable query dependencies + // because it didn't go its full depth. + // + // FIXME(@lcnr): We could still cache subtrees which are not impacted by overflow though. + // Tracking that info correctly isn't trivial, so I haven't implemented it for now. + let should_cache_globally = !overflow_data.did_overflow() || stack.is_empty(); + if should_cache_globally { + // FIXME: move the provisional entry to the global cache. + let _ = (tcx, goal, response); + } +} diff --git a/compiler/rustc_trait_selection/src/solve/search_graph/mod.rs b/compiler/rustc_trait_selection/src/solve/search_graph/mod.rs new file mode 100644 index 00000000000..8d2a3a9cd81 --- /dev/null +++ b/compiler/rustc_trait_selection/src/solve/search_graph/mod.rs @@ -0,0 +1,172 @@ +mod cache; +mod overflow; + +use self::cache::ProvisionalEntry; +use super::{CanonicalGoal, Certainty, MaybeCause, QueryResult}; +use cache::ProvisionalCache; +use overflow::OverflowData; +use rustc_index::vec::IndexVec; +use rustc_middle::ty::TyCtxt; +use std::collections::hash_map::Entry; + +rustc_index::newtype_index! { + pub struct StackDepth {} +} + +struct StackElem<'tcx> { + goal: CanonicalGoal<'tcx>, + has_been_used: bool, +} + +pub(super) struct SearchGraph<'tcx> { + /// The stack of goals currently being computed. + /// + /// An element is *deeper* in the stack if its index is *lower*. + stack: IndexVec>, + overflow_data: OverflowData, + provisional_cache: ProvisionalCache<'tcx>, +} + +impl<'tcx> SearchGraph<'tcx> { + pub(super) fn new(tcx: TyCtxt<'tcx>) -> SearchGraph<'tcx> { + Self { + stack: Default::default(), + overflow_data: OverflowData::new(tcx), + provisional_cache: ProvisionalCache::empty(), + } + } + + /// Tries putting the new goal on the stack, returning an error if it is already cached. + /// + /// This correctly updates the provisional cache if there is a cycle. + pub(super) fn try_push_stack( + &mut self, + tcx: TyCtxt<'tcx>, + goal: CanonicalGoal<'tcx>, + ) -> Result<(), QueryResult<'tcx>> { + // FIXME: start by checking the global cache + + // Look at the provisional cache to check for cycles. + let cache = &mut self.provisional_cache; + match cache.lookup_table.entry(goal) { + // No entry, simply push this goal on the stack after dealing with overflow. + Entry::Vacant(v) => { + if self.overflow_data.has_overflow(self.stack.len()) { + return Err(self.deal_with_overflow(tcx, goal)); + } + + let depth = self.stack.push(StackElem { goal, has_been_used: false }); + let response = super::response_no_constraints(tcx, goal, Certainty::Yes); + let entry_index = cache.entries.push(ProvisionalEntry { response, depth, goal }); + v.insert(entry_index); + Ok(()) + } + // We have a nested goal which relies on a goal `root` deeper in the stack. + // + // We first store that we may have to rerun `evaluate_goal` for `root` in case the + // provisional response is not equal to the final response. We also update the depth + // of all goals which recursively depend on our current goal to depend on `root` + // instead. + // + // Finally we can return either the provisional response for that goal if we have a + // coinductive cycle or an ambiguous result if the cycle is inductive. + Entry::Occupied(entry_index) => { + let entry_index = *entry_index.get(); + + cache.add_dependency_of_leaf_on(entry_index); + let stack_depth = cache.depth(entry_index); + + self.stack[stack_depth].has_been_used = true; + // NOTE: The goals on the stack aren't the only goals involved in this cycle. + // We can also depend on goals which aren't part of the stack but coinductively + // depend on the stack themselves. We already checked whether all the goals + // between these goals and their root on the stack. This means that as long as + // each goal in a cycle is checked for coinductivity by itself, simply checking + // the stack is enough. + if self.stack.raw[stack_depth.index()..] + .iter() + .all(|g| g.goal.value.predicate.is_coinductive(tcx)) + { + Err(cache.provisional_result(entry_index)) + } else { + Err(super::response_no_constraints( + tcx, + goal, + Certainty::Maybe(MaybeCause::Overflow), + )) + } + } + } + } + + /// We cannot simply store the result of [EvalCtxt::compute_goal] as we have to deal with + /// coinductive cycles. + /// + /// When we encounter a coinductive cycle, we have to prove the final result of that cycle + /// while we are still computing that result. Because of this we continously recompute the + /// cycle until the result of the previous iteration is equal to the final result, at which + /// point we are done. + /// + /// This function returns `true` if we were able to finalize the goal and `false` if it has + /// updated the provisional cache and we have to recompute the current goal. + /// + /// FIXME: Refer to the rustc-dev-guide entry once it exists. + pub(super) fn try_finalize_goal( + &mut self, + tcx: TyCtxt<'tcx>, + actual_goal: CanonicalGoal<'tcx>, + response: QueryResult<'tcx>, + ) -> bool { + let StackElem { goal, has_been_used } = self.stack.pop().unwrap(); + assert_eq!(goal, actual_goal); + + let cache = &mut self.provisional_cache; + let provisional_entry_index = *cache.lookup_table.get(&goal).unwrap(); + let provisional_entry = &mut cache.entries[provisional_entry_index]; + let depth = provisional_entry.depth; + // Was the current goal the root of a cycle and was the provisional response + // different from the final one. + if has_been_used && provisional_entry.response != response { + // If so, update the provisional reponse for this goal... + provisional_entry.response = response; + // ...remove all entries whose result depends on this goal + // from the provisional cache... + // + // That's not completely correct, as a nested goal can also + // depend on a goal which is lower in the stack so it doesn't + // actually depend on the current goal. This should be fairly + // rare and is hopefully not relevant for performance. + #[allow(rustc::potential_query_instability)] + cache.lookup_table.retain(|_key, index| *index <= provisional_entry_index); + cache.entries.truncate(provisional_entry_index.index() + 1); + + // ...and finally push our goal back on the stack and reevaluate it. + self.stack.push(StackElem { goal, has_been_used: false }); + false + } else { + // If not, we're done with this goal. + // + // Check whether that this goal doesn't depend on a goal deeper on the stack + // and if so, move it and all nested goals to the global cache. + // + // Note that if any nested goal were to depend on something deeper on the stack, + // this would have also updated the depth of the current goal. + if depth == self.stack.next_index() { + for (i, entry) in cache.entries.drain_enumerated(provisional_entry_index.index()..) + { + let actual_index = cache.lookup_table.remove(&entry.goal); + debug_assert_eq!(Some(i), actual_index); + debug_assert!(entry.depth == depth); + cache::try_move_finished_goal_to_global_cache( + tcx, + &mut self.overflow_data, + &self.stack, + entry.goal, + entry.response, + ); + } + } + true + } + } +} diff --git a/compiler/rustc_trait_selection/src/solve/overflow.rs b/compiler/rustc_trait_selection/src/solve/search_graph/overflow.rs similarity index 74% rename from compiler/rustc_trait_selection/src/solve/overflow.rs rename to compiler/rustc_trait_selection/src/solve/search_graph/overflow.rs index 8bbb9f63e78..1dd3894c91a 100644 --- a/compiler/rustc_trait_selection/src/solve/overflow.rs +++ b/compiler/rustc_trait_selection/src/solve/search_graph/overflow.rs @@ -3,8 +3,8 @@ use rustc_middle::ty::TyCtxt; use rustc_session::Limit; -use super::cache::response_no_constraints; -use super::{Certainty, EvalCtxt, MaybeCause, QueryResult}; +use super::SearchGraph; +use crate::solve::{response_no_constraints, Certainty, EvalCtxt, MaybeCause, QueryResult}; /// When detecting a solver overflow, we return ambiguity. Overflow can be /// *hidden* by either a fatal error in an **AND** or a trivial success in an **OR**. @@ -50,32 +50,35 @@ fn deal_with_overflow(&mut self) { } } -impl<'tcx> EvalCtxt<'tcx> { - pub(super) fn deal_with_overflow( +impl<'tcx> SearchGraph<'tcx> { + pub fn deal_with_overflow( &mut self, + tcx: TyCtxt<'tcx>, goal: Canonical<'tcx, impl Sized>, ) -> QueryResult<'tcx> { self.overflow_data.deal_with_overflow(); - response_no_constraints(self.tcx, goal, Certainty::Maybe(MaybeCause::Overflow)) + response_no_constraints(tcx, goal, Certainty::Maybe(MaybeCause::Overflow)) } +} +impl<'tcx> EvalCtxt<'_, 'tcx> { /// A `while`-loop which tracks overflow. - pub(super) fn repeat_while_none( + pub fn repeat_while_none( &mut self, mut loop_body: impl FnMut(&mut Self) -> Option>, ) -> Result { - let start_depth = self.overflow_data.additional_depth; - let depth = self.provisional_cache.current_depth(); - while !self.overflow_data.has_overflow(depth) { + let start_depth = self.search_graph.overflow_data.additional_depth; + let depth = self.search_graph.stack.len(); + while !self.search_graph.overflow_data.has_overflow(depth) { if let Some(result) = loop_body(self) { - self.overflow_data.additional_depth = start_depth; + self.search_graph.overflow_data.additional_depth = start_depth; return result; } - self.overflow_data.additional_depth += 1; + self.search_graph.overflow_data.additional_depth += 1; } - self.overflow_data.additional_depth = start_depth; - self.overflow_data.deal_with_overflow(); + self.search_graph.overflow_data.additional_depth = start_depth; + self.search_graph.overflow_data.deal_with_overflow(); Ok(Certainty::Maybe(MaybeCause::Overflow)) } } diff --git a/compiler/rustc_trait_selection/src/solve/trait_goals.rs b/compiler/rustc_trait_selection/src/solve/trait_goals.rs index a43fef5cdb0..cc68cfeea1d 100644 --- a/compiler/rustc_trait_selection/src/solve/trait_goals.rs +++ b/compiler/rustc_trait_selection/src/solve/trait_goals.rs @@ -3,7 +3,7 @@ use std::iter; use super::assembly::{self, AssemblyCtxt}; -use super::{CanonicalGoal, EvalCtxt, Goal, QueryResult}; +use super::{EvalCtxt, Goal, QueryResult}; use rustc_hir::def_id::DefId; use rustc_infer::infer::InferOk; use rustc_infer::traits::query::NoSolution; @@ -67,11 +67,12 @@ fn trait_def_id(self, _: TyCtxt<'tcx>) -> DefId { } fn consider_impl_candidate( - acx: &mut AssemblyCtxt<'_, 'tcx, Self>, + acx: &mut AssemblyCtxt<'_, '_, 'tcx, Self>, goal: Goal<'tcx, TraitPredicate<'tcx>>, impl_def_id: DefId, ) { - let tcx = acx.cx.tcx; + let tcx = acx.cx.tcx(); + let infcx = acx.cx.infcx; let impl_trait_ref = tcx.impl_trait_ref(impl_def_id).unwrap(); let drcx = DeepRejectCtxt { treat_obligation_params: TreatParams::AsPlaceholder }; @@ -81,12 +82,11 @@ fn consider_impl_candidate( return; } - acx.infcx.probe(|_| { - let impl_substs = acx.infcx.fresh_substs_for_item(DUMMY_SP, impl_def_id); + infcx.probe(|_| { + let impl_substs = infcx.fresh_substs_for_item(DUMMY_SP, impl_def_id); let impl_trait_ref = impl_trait_ref.subst(tcx, impl_substs); - let Ok(InferOk { obligations, .. }) = acx - .infcx + let Ok(InferOk { obligations, .. }) = infcx .at(&ObligationCause::dummy(), goal.param_env) .define_opaque_types(false) .eq(goal.predicate.trait_ref, impl_trait_ref) @@ -104,16 +104,16 @@ fn consider_impl_candidate( let nested_goals = obligations.into_iter().map(|o| o.into()).chain(where_clause_bounds).collect(); - let Ok(certainty) = acx.cx.evaluate_all(acx.infcx, nested_goals) else { return }; + let Ok(certainty) = acx.cx.evaluate_all(nested_goals) else { return }; acx.try_insert_candidate(CandidateSource::Impl(impl_def_id), certainty); }) } } -impl<'tcx> EvalCtxt<'tcx> { +impl<'tcx> EvalCtxt<'_, 'tcx> { pub(super) fn compute_trait_goal( &mut self, - goal: CanonicalGoal<'tcx, TraitPredicate<'tcx>>, + goal: Goal<'tcx, TraitPredicate<'tcx>>, ) -> QueryResult<'tcx> { let candidates = AssemblyCtxt::assemble_and_evaluate_candidates(self, goal); self.merge_trait_candidates_discard_reservation_impls(candidates) @@ -176,7 +176,7 @@ fn trait_candidate_should_be_dropped_in_favor_of( fn discard_reservation_impl(&self, candidate: Candidate<'tcx>) -> Candidate<'tcx> { if let CandidateSource::Impl(def_id) = candidate.source { - if let ty::ImplPolarity::Reservation = self.tcx.impl_polarity(def_id) { + if let ty::ImplPolarity::Reservation = self.tcx().impl_polarity(def_id) { debug!("Selected reservation impl"); // FIXME: reduce candidate to ambiguous // FIXME: replace `var_values` with identity, yeet external constraints.