Auto merge of #128506 - compiler-errors:by-move-body, r=cjgillot

Stop storing a special inner body for the coroutine by-move body for async closures

...and instead, just synthesize an item which is treated mostly normally by the MIR pipeline.

This PR does a few things:
* We synthesize a new `DefId` for the by-move body of a closure, which has its `mir_built` fed with the output of the `ByMoveBody` MIR transformation, and some other relevant queries.
* This has the `DefKind::ByMoveBody`, which we use to distinguish it from "real" bodies (that come from HIR) which need to be borrowck'd. Introduce `TyCtxt::is_synthetic_mir` to skip over `mir_borrowck` which is called by `mir_promoted`; borrowck isn't really possible to make work ATM since it heavily relies being called on a body generated from HIR, and is redundant by the construction of the by-move-body.
* Remove the special `PassManager` hacks for handling the inner `by_move_body` stored within the coroutine's mir body. Instead, this body is fed like a regular MIR body, so it's goes through all of the `tcx.*_mir` stages normally (build -> promoted -> ...etc... -> optimized) ✨.
* Remove the `InstanceKind::ByMoveBody` shim, since now we have a "regular" def id, we can just use `InstanceKind::Item`. This also allows us to remove the corresponding hacks from codegen, such as in `fn_sig_for_fn_abi` ✨.

Notable remarks:
* ~~I know it's kind of weird to be using `DefKind::Closure` here, since it's not a distinct closure but just a new MIR body. I don't believe it really matters, but I could also use a different `DefKind`... maybe one that we could use for synthetic MIR bodies in general?~~ edit: We're doing this now.

compiler/rustc_const_eval/src/interpret/call.rs

@@ -576,7 +576,6 @@ pub(super) fn init_fn_call(
             | ty::InstanceKind::ReifyShim(..)
             | ty::InstanceKind::ClosureOnceShim { .. }
             | ty::InstanceKind::ConstructCoroutineInClosureShim { .. }
-            | ty::InstanceKind::CoroutineKindShim { .. }
             | ty::InstanceKind::FnPtrShim(..)
             | ty::InstanceKind::DropGlue(..)
             | ty::InstanceKind::CloneShim(..)

compiler/rustc_hir/src/def.rs

@@ -133,6 +133,9 @@ pub enum DefKind {
     /// we treat them all the same, and code which needs to distinguish them can match
     /// or `hir::ClosureKind` or `type_of`.
     Closure,
+    /// The definition of a synthetic coroutine body created by the lowering of a
+    /// coroutine-closure, such as an async closure.
+    SyntheticCoroutineBody,
 }
 
 impl DefKind {
@@ -177,6 +180,7 @@ pub fn descr(self, def_id: DefId) -> &'static str {
             DefKind::Closure => "closure",
             DefKind::ExternCrate => "extern crate",
             DefKind::GlobalAsm => "global assembly block",
+            DefKind::SyntheticCoroutineBody => "synthetic mir body",
         }
     }
 
@@ -236,7 +240,8 @@ pub fn ns(&self) -> Option<Namespace> {
             | DefKind::ForeignMod
             | DefKind::GlobalAsm
             | DefKind::Impl { .. }
-            | DefKind::OpaqueTy => None,
+            | DefKind::OpaqueTy
+            | DefKind::SyntheticCoroutineBody => None,
         }
     }
 
@@ -276,6 +281,7 @@ pub fn def_path_data(self, name: Symbol) -> DefPathData {
             DefKind::GlobalAsm => DefPathData::GlobalAsm,
             DefKind::Impl { .. } => DefPathData::Impl,
             DefKind::Closure => DefPathData::Closure,
+            DefKind::SyntheticCoroutineBody => DefPathData::Closure,
         }
     }
 
@@ -291,7 +297,8 @@ pub fn has_codegen_attrs(self) -> bool {
             | DefKind::AssocFn
             | DefKind::Ctor(..)
             | DefKind::Closure
-            | DefKind::Static { .. } => true,
+            | DefKind::Static { .. }
+            | DefKind::SyntheticCoroutineBody => true,
             DefKind::Mod
             | DefKind::Struct
             | DefKind::Union

compiler/rustc_hir_analysis/src/check/wfcheck.rs

@@ -2174,7 +2174,8 @@ fn lint_redundant_lifetimes<'tcx>(
         | DefKind::Field
         | DefKind::LifetimeParam
         | DefKind::GlobalAsm
-        | DefKind::Closure => return,
+        | DefKind::Closure
+        | DefKind::SyntheticCoroutineBody => return,
     }
 
     // The ordering of this lifetime map is a bit subtle.

compiler/rustc_hir_analysis/src/lib.rs

@@ -201,10 +201,6 @@ pub fn check_crate(tcx: TyCtxt<'_>) {
         }
     });
 
-    // Freeze definitions as we don't add new ones at this point. This improves performance by
-    // allowing lock-free access to them.
-    tcx.untracked().definitions.freeze();
-
     // FIXME: Remove this when we implement creating `DefId`s
     // for anon constants during their parents' typeck.
     // Typeck all body owners in parallel will produce queries

compiler/rustc_interface/src/passes.rs

@@ -784,7 +784,20 @@ fn run_required_analyses(tcx: TyCtxt<'_>) {
             }
         );
     });
+
     rustc_hir_analysis::check_crate(tcx);
+    sess.time("MIR_coroutine_by_move_body", || {
+        tcx.hir().par_body_owners(|def_id| {
+            if tcx.needs_coroutine_by_move_body_def_id(def_id) {
+                tcx.ensure_with_value().coroutine_by_move_body_def_id(def_id);
+            }
+        });
+    });
+    // Freeze definitions as we don't add new ones at this point.
+    // We need to wait until now since we synthesize a by-move body
+    // This improves performance by allowing lock-free access to them.
+    tcx.untracked().definitions.freeze();
+
     sess.time("MIR_borrow_checking", || {
         tcx.hir().par_body_owners(|def_id| {
             // Run unsafety check because it's responsible for stealing and
@@ -816,6 +829,7 @@ fn run_required_analyses(tcx: TyCtxt<'_>) {
             );
         }
     });
+
     sess.time("layout_testing", || layout_test::test_layout(tcx));
     sess.time("abi_testing", || abi_test::test_abi(tcx));
 

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -872,7 +872,8 @@ fn should_encode_span(def_kind: DefKind) -> bool {
         | DefKind::OpaqueTy
         | DefKind::Field
         | DefKind::Impl { .. }
-        | DefKind::Closure => true,
+        | DefKind::Closure
+        | DefKind::SyntheticCoroutineBody => true,
         DefKind::ForeignMod | DefKind::GlobalAsm => false,
     }
 }
@@ -902,6 +903,7 @@ fn should_encode_attrs(def_kind: DefKind) -> bool {
         // https://github.com/model-checking/kani and is not a performance
         // or maintenance issue for us.
         DefKind::Closure => true,
+        DefKind::SyntheticCoroutineBody => false,
         DefKind::TyParam
         | DefKind::ConstParam
         | DefKind::Ctor(..)
@@ -948,7 +950,8 @@ fn should_encode_expn_that_defined(def_kind: DefKind) -> bool {
         | DefKind::Field
         | DefKind::LifetimeParam
         | DefKind::GlobalAsm
-        | DefKind::Closure => false,
+        | DefKind::Closure
+        | DefKind::SyntheticCoroutineBody => false,
     }
 }
 
@@ -984,7 +987,8 @@ fn should_encode_visibility(def_kind: DefKind) -> bool {
         | DefKind::GlobalAsm
         | DefKind::Impl { .. }
         | DefKind::Closure
-        | DefKind::ExternCrate => false,
+        | DefKind::ExternCrate
+        | DefKind::SyntheticCoroutineBody => false,
     }
 }
 
@@ -1019,7 +1023,8 @@ fn should_encode_stability(def_kind: DefKind) -> bool {
         | DefKind::InlineConst
         | DefKind::GlobalAsm
         | DefKind::Closure
-        | DefKind::ExternCrate => false,
+        | DefKind::ExternCrate
+        | DefKind::SyntheticCoroutineBody => false,
     }
 }
 
@@ -1061,6 +1066,7 @@ fn should_encode_mir(
         }
         // Coroutines require optimized MIR to compute layout.
         DefKind::Closure if tcx.is_coroutine(def_id.to_def_id()) => (false, true),
+        DefKind::SyntheticCoroutineBody => (false, true),
         // Full-fledged functions + closures
         DefKind::AssocFn | DefKind::Fn | DefKind::Closure => {
             let generics = tcx.generics_of(def_id);
@@ -1109,7 +1115,8 @@ fn should_encode_variances<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, def_kind: Def
         | DefKind::InlineConst
         | DefKind::GlobalAsm
         | DefKind::Closure
-        | DefKind::ExternCrate => false,
+        | DefKind::ExternCrate
+        | DefKind::SyntheticCoroutineBody => false,
         DefKind::TyAlias => tcx.type_alias_is_lazy(def_id),
     }
 }
@@ -1137,7 +1144,8 @@ fn should_encode_generics(def_kind: DefKind) -> bool {
         | DefKind::Impl { .. }
         | DefKind::Field
         | DefKind::TyParam
-        | DefKind::Closure => true,
+        | DefKind::Closure
+        | DefKind::SyntheticCoroutineBody => true,
         DefKind::Mod
         | DefKind::ForeignMod
         | DefKind::ConstParam
@@ -1168,7 +1176,8 @@ fn should_encode_type(tcx: TyCtxt<'_>, def_id: LocalDefId, def_kind: DefKind) ->
         | DefKind::Closure
         | DefKind::ConstParam
         | DefKind::AnonConst
-        | DefKind::InlineConst => true,
+        | DefKind::InlineConst
+        | DefKind::SyntheticCoroutineBody => true,
 
         DefKind::OpaqueTy => {
             let origin = tcx.opaque_type_origin(def_id);
@@ -1240,7 +1249,8 @@ fn should_encode_fn_sig(def_kind: DefKind) -> bool {
         | DefKind::Use
         | DefKind::LifetimeParam
         | DefKind::GlobalAsm
-        | DefKind::ExternCrate => false,
+        | DefKind::ExternCrate
+        | DefKind::SyntheticCoroutineBody => false,
     }
 }
 
@@ -1277,7 +1287,8 @@ fn should_encode_constness(def_kind: DefKind) -> bool {
         | DefKind::Use
         | DefKind::LifetimeParam
         | DefKind::GlobalAsm
-        | DefKind::ExternCrate => false,
+        | DefKind::ExternCrate
+        | DefKind::SyntheticCoroutineBody => false,
     }
 }
 
@@ -1310,7 +1321,8 @@ fn should_encode_const(def_kind: DefKind) -> bool {
         | DefKind::Use
         | DefKind::LifetimeParam
         | DefKind::GlobalAsm
-        | DefKind::ExternCrate => false,
+        | DefKind::ExternCrate
+        | DefKind::SyntheticCoroutineBody => false,
     }
 }
 
@@ -1458,7 +1470,7 @@ fn encode_def_ids(&mut self) {
                     record!(self.tables.associated_type_for_effects[def_id] <- assoc_def_id);
                 }
             }
-            if def_kind == DefKind::Closure
+            if let DefKind::Closure | DefKind::SyntheticCoroutineBody = def_kind
                 && let Some(coroutine_kind) = self.tcx.coroutine_kind(def_id)
             {
                 self.tables.coroutine_kind.set(def_id.index, Some(coroutine_kind))

compiler/rustc_metadata/src/rmeta/table.rs

@@ -171,6 +171,7 @@ impl FixedSizeEncoding for Option<$ty> {
         ( Macro(MacroKind::Bang)                   )
         ( Macro(MacroKind::Attr)                   )
         ( Macro(MacroKind::Derive)                 )
+        ( SyntheticCoroutineBody                   )
     }
 }
 

compiler/rustc_middle/src/hir/map/mod.rs

@@ -307,7 +307,7 @@ pub fn body_owner_kind(self, def_id: impl Into<DefId>) -> BodyOwnerKind {
             }
             DefKind::InlineConst => BodyOwnerKind::Const { inline: true },
             DefKind::Ctor(..) | DefKind::Fn | DefKind::AssocFn => BodyOwnerKind::Fn,
-            DefKind::Closure => BodyOwnerKind::Closure,
+            DefKind::Closure | DefKind::SyntheticCoroutineBody => BodyOwnerKind::Closure,
             DefKind::Static { safety: _, mutability, nested: false } => {
                 BodyOwnerKind::Static(mutability)
             }

compiler/rustc_middle/src/mir/mod.rs

@@ -267,18 +267,6 @@ pub struct CoroutineInfo<'tcx> {
     /// Coroutine drop glue. This field is populated after the state transform pass.
     pub coroutine_drop: Option<Body<'tcx>>,
 
-    /// The body of the coroutine, modified to take its upvars by move rather than by ref.
-    ///
-    /// This is used by coroutine-closures, which must return a different flavor of coroutine
-    /// when called using `AsyncFnOnce::call_once`. It is produced by the `ByMoveBody` pass which
-    /// is run right after building the initial MIR, and will only be populated for coroutines
-    /// which come out of the async closure desugaring.
-    ///
-    /// This body should be processed in lockstep with the containing body -- any optimization
-    /// passes, etc, should be applied to this body as well. This is done automatically if
-    /// using `run_passes`.
-    pub by_move_body: Option<Body<'tcx>>,
-
     /// The layout of a coroutine. This field is populated after the state transform pass.
     pub coroutine_layout: Option<CoroutineLayout<'tcx>>,
 
@@ -298,7 +286,6 @@ pub fn initial(
             coroutine_kind,
             yield_ty: Some(yield_ty),
             resume_ty: Some(resume_ty),
-            by_move_body: None,
             coroutine_drop: None,
             coroutine_layout: None,
         }
@@ -665,10 +652,6 @@ pub fn coroutine_drop(&self) -> Option<&Body<'tcx>> {
         self.coroutine.as_ref().and_then(|coroutine| coroutine.coroutine_drop.as_ref())
     }
 
-    pub fn coroutine_by_move_body(&self) -> Option<&Body<'tcx>> {
-        self.coroutine.as_ref()?.by_move_body.as_ref()
-    }
-
     #[inline]
     pub fn coroutine_kind(&self) -> Option<CoroutineKind> {
         self.coroutine.as_ref().map(|coroutine| coroutine.coroutine_kind)

compiler/rustc_middle/src/mir/mono.rs

@@ -415,7 +415,6 @@ fn item_sort_key<'tcx>(tcx: TyCtxt<'tcx>, item: MonoItem<'tcx>) -> ItemSortKey<'
                             | InstanceKind::Virtual(..)
                             | InstanceKind::ClosureOnceShim { .. }
                             | InstanceKind::ConstructCoroutineInClosureShim { .. }
-                            | InstanceKind::CoroutineKindShim { .. }
                             | InstanceKind::DropGlue(..)
                             | InstanceKind::CloneShim(..)
                             | InstanceKind::ThreadLocalShim(..)

compiler/rustc_middle/src/mir/visit.rs

@@ -349,7 +349,6 @@ fn super_source_scope_data(
                             coroutine_closure_def_id: _def_id,
                             receiver_by_ref: _,
                         } |
-                        ty::InstanceKind::CoroutineKindShim { coroutine_def_id: _def_id } |
                         ty::InstanceKind::AsyncDropGlueCtorShim(_def_id, None) |
                         ty::InstanceKind::DropGlue(_def_id, None) => {}
 

compiler/rustc_middle/src/query/mod.rs

@@ -326,6 +326,7 @@
     query predicates_of(key: DefId) -> ty::GenericPredicates<'tcx> {
         desc { |tcx| "computing predicates of `{}`", tcx.def_path_str(key) }
         cache_on_disk_if { key.is_local() }
+        feedable
     }
 
     query opaque_types_defined_by(
@@ -498,6 +499,7 @@
     /// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/mir/construction.html
     query mir_built(key: LocalDefId) -> &'tcx Steal<mir::Body<'tcx>> {
         desc { |tcx| "building MIR for `{}`", tcx.def_path_str(key) }
+        feedable
     }
 
     /// Try to build an abstract representation of the given constant.
@@ -742,6 +744,7 @@
     query constness(key: DefId) -> hir::Constness {
         desc { |tcx| "checking if item is const: `{}`", tcx.def_path_str(key) }
         separate_provide_extern
+        feedable
     }
 
     query asyncness(key: DefId) -> ty::Asyncness {
@@ -760,10 +763,22 @@
         desc { |tcx| "checking if item is promotable: `{}`", tcx.def_path_str(key) }
     }
 
+    /// The body of the coroutine, modified to take its upvars by move rather than by ref.
+    ///
+    /// This is used by coroutine-closures, which must return a different flavor of coroutine
+    /// when called using `AsyncFnOnce::call_once`. It is produced by the `ByMoveBody` pass which
+    /// is run right after building the initial MIR, and will only be populated for coroutines
+    /// which come out of the async closure desugaring.
+    query coroutine_by_move_body_def_id(def_id: DefId) -> DefId {
+        desc { |tcx| "looking up the coroutine by-move body for `{}`", tcx.def_path_str(def_id) }
+        separate_provide_extern
+    }
+
     /// Returns `Some(coroutine_kind)` if the node pointed to by `def_id` is a coroutine.
     query coroutine_kind(def_id: DefId) -> Option<hir::CoroutineKind> {
         desc { |tcx| "looking up coroutine kind of `{}`", tcx.def_path_str(def_id) }
         separate_provide_extern
+        feedable
     }
 
     query coroutine_for_closure(def_id: DefId) -> DefId {

compiler/rustc_middle/src/ty/context.rs

@@ -1579,6 +1579,12 @@ pub fn coroutine_is_async(self, def_id: DefId) -> bool {
         )
     }
 
+    // Whether the body owner is synthetic, which in this case means it does not correspond to
+    // meaningful HIR. This is currently used to skip over MIR borrowck.
+    pub fn is_synthetic_mir(self, def_id: impl Into<DefId>) -> bool {
+        matches!(self.def_kind(def_id.into()), DefKind::SyntheticCoroutineBody)
+    }
+
     /// Returns `true` if the node pointed to by `def_id` is a general coroutine that implements `Coroutine`.
     /// This means it is neither an `async` or `gen` construct.
     pub fn is_general_coroutine(self, def_id: DefId) -> bool {
@@ -3168,6 +3174,18 @@ pub fn impl_polarity(self, def_id: impl IntoQueryParam<DefId>) -> ty::ImplPolari
         self.impl_trait_header(def_id).map_or(ty::ImplPolarity::Positive, |h| h.polarity)
     }
 
+    pub fn needs_coroutine_by_move_body_def_id(self, def_id: LocalDefId) -> bool {
+        if let Some(hir::CoroutineKind::Desugared(_, hir::CoroutineSource::Closure)) =
+            self.coroutine_kind(def_id)
+            && let ty::Coroutine(_, args) = self.type_of(def_id).instantiate_identity().kind()
+            && args.as_coroutine().kind_ty().to_opt_closure_kind() != Some(ty::ClosureKind::FnOnce)
+        {
+            true
+        } else {
+            false
+        }
+    }
+
     /// Whether this is a trait implementation that has `#[diagnostic::do_not_recommend]`
     pub fn do_not_recommend_impl(self, def_id: DefId) -> bool {
         matches!(self.def_kind(def_id), DefKind::Impl { of_trait: true })

compiler/rustc_middle/src/ty/instance.rs

@@ -141,14 +141,6 @@ pub enum InstanceKind<'tcx> {
         receiver_by_ref: bool,
     },
 
-    /// `<[coroutine] as Future>::poll`, but for coroutines produced when `AsyncFnOnce`
-    /// is called on a coroutine-closure whose closure kind greater than `FnOnce`, or
-    /// similarly for `AsyncFnMut`.
-    ///
-    /// This will select the body that is produced by the `ByMoveBody` transform, and thus
-    /// take and use all of its upvars by-move rather than by-ref.
-    CoroutineKindShim { coroutine_def_id: DefId },
-
     /// Compiler-generated accessor for thread locals which returns a reference to the thread local
     /// the `DefId` defines. This is used to export thread locals from dylibs on platforms lacking
     /// native support.
@@ -248,7 +240,6 @@ pub fn def_id(self) -> DefId {
                 coroutine_closure_def_id: def_id,
                 receiver_by_ref: _,
             }
-            | ty::InstanceKind::CoroutineKindShim { coroutine_def_id: def_id }
             | InstanceKind::DropGlue(def_id, _)
             | InstanceKind::CloneShim(def_id, _)
             | InstanceKind::FnPtrAddrShim(def_id, _)
@@ -270,7 +261,6 @@ pub fn def_id_if_not_guaranteed_local_codegen(self) -> Option<DefId> {
             | InstanceKind::Intrinsic(..)
             | InstanceKind::ClosureOnceShim { .. }
             | ty::InstanceKind::ConstructCoroutineInClosureShim { .. }
-            | ty::InstanceKind::CoroutineKindShim { .. }
             | InstanceKind::DropGlue(..)
             | InstanceKind::AsyncDropGlueCtorShim(..)
             | InstanceKind::CloneShim(..)
@@ -377,7 +367,6 @@ pub fn has_polymorphic_mir_body(&self) -> bool {
             | InstanceKind::AsyncDropGlueCtorShim(_, Some(_)) => false,
             InstanceKind::ClosureOnceShim { .. }
             | InstanceKind::ConstructCoroutineInClosureShim { .. }
-            | InstanceKind::CoroutineKindShim { .. }
             | InstanceKind::DropGlue(..)
             | InstanceKind::AsyncDropGlueCtorShim(..)
             | InstanceKind::Item(_)
@@ -452,7 +441,6 @@ pub fn fmt_instance(
         InstanceKind::FnPtrShim(_, ty) => write!(f, " - shim({ty})"),
         InstanceKind::ClosureOnceShim { .. } => write!(f, " - shim"),
         InstanceKind::ConstructCoroutineInClosureShim { .. } => write!(f, " - shim"),
-        InstanceKind::CoroutineKindShim { .. } => write!(f, " - shim"),
         InstanceKind::DropGlue(_, None) => write!(f, " - shim(None)"),
         InstanceKind::DropGlue(_, Some(ty)) => write!(f, " - shim(Some({ty}))"),
         InstanceKind::CloneShim(_, ty) => write!(f, " - shim({ty})"),
@@ -850,7 +838,9 @@ pub fn try_resolve_item_for_coroutine(
                 Some(Instance { def: ty::InstanceKind::Item(coroutine_def_id), args })
             } else {
                 Some(Instance {
-                    def: ty::InstanceKind::CoroutineKindShim { coroutine_def_id },
+                    def: ty::InstanceKind::Item(
+                        tcx.coroutine_by_move_body_def_id(coroutine_def_id),
+                    ),
                     args,
                 })
             }

compiler/rustc_middle/src/ty/mod.rs

@@ -1756,7 +1756,6 @@ pub fn instance_mir(self, instance: ty::InstanceKind<'tcx>) -> &'tcx Body<'tcx>
             | ty::InstanceKind::Virtual(..)
             | ty::InstanceKind::ClosureOnceShim { .. }
             | ty::InstanceKind::ConstructCoroutineInClosureShim { .. }
-            | ty::InstanceKind::CoroutineKindShim { .. }
             | ty::InstanceKind::DropGlue(..)
             | ty::InstanceKind::CloneShim(..)
             | ty::InstanceKind::ThreadLocalShim(..)
@@ -1874,7 +1873,8 @@ pub fn coroutine_layout(
                     identity_kind_ty.to_opt_closure_kind(),
                     Some(ClosureKind::Fn | ClosureKind::FnMut)
                 );
-                mir.coroutine_by_move_body().unwrap().coroutine_layout_raw()
+                self.optimized_mir(self.coroutine_by_move_body_def_id(def_id))
+                    .coroutine_layout_raw()
             }
         }
     }

compiler/rustc_mir_transform/src/coroutine.rs

@@ -53,7 +53,7 @@
 mod by_move_body;
 use std::{iter, ops};
 
-pub use by_move_body::ByMoveBody;
+pub use by_move_body::coroutine_by_move_body_def_id;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::pluralize;
 use rustc_hir as hir;

compiler/rustc_mir_transform/src/coroutine/by_move_body.rs

@@ -69,161 +69,167 @@
 //! in case precise captures (edition 2021 closure capture rules) caused the inner coroutine
 //! to split one field capture into two.
 
+use rustc_data_structures::steal::Steal;
 use rustc_data_structures::unord::UnordMap;
 use rustc_hir as hir;
+use rustc_hir::def::DefKind;
+use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_middle::bug;
 use rustc_middle::hir::place::{Projection, ProjectionKind};
 use rustc_middle::mir::visit::MutVisitor;
-use rustc_middle::mir::{self, dump_mir, MirPass};
+use rustc_middle::mir::{self, dump_mir};
 use rustc_middle::ty::{self, InstanceKind, Ty, TyCtxt, TypeVisitableExt};
+use rustc_span::symbol::kw;
 use rustc_target::abi::{FieldIdx, VariantIdx};
 
-use crate::pass_manager::validate_body;
+pub fn coroutine_by_move_body_def_id<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    coroutine_def_id: LocalDefId,
+) -> DefId {
+    let body = tcx.mir_built(coroutine_def_id).borrow();
 
-pub struct ByMoveBody;
+    let Some(hir::CoroutineKind::Desugared(_, hir::CoroutineSource::Closure)) =
+        tcx.coroutine_kind(coroutine_def_id)
+    else {
+        bug!("should only be invoked on coroutine-closures");
+    };
 
-impl<'tcx> MirPass<'tcx> for ByMoveBody {
+    // Also, let's skip processing any bodies with errors, since there's no guarantee
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut mir::Body<'tcx>) {
+    // the MIR body will be constructed well.
-        // We only need to generate by-move coroutine bodies for coroutines that come
+    let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty;
-        // from coroutine-closures.
-        let Some(coroutine_def_id) = body.source.def_id().as_local() else {
-            return;
-        };
-        let Some(hir::CoroutineKind::Desugared(_, hir::CoroutineSource::Closure)) =
-            tcx.coroutine_kind(coroutine_def_id)
-        else {
-            return;
-        };
 
-        // Also, let's skip processing any bodies with errors, since there's no guarantee
+    let ty::Coroutine(_, args) = *coroutine_ty.kind() else { bug!("{body:#?}") };
-        // the MIR body will be constructed well.
+    let args = args.as_coroutine();
-        let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty;
-        if coroutine_ty.references_error() {
-            return;
-        }
 
-        // We don't need to generate a by-move coroutine if the coroutine body was
+    let coroutine_kind = args.kind_ty().to_opt_closure_kind().unwrap();
-        // produced by the `CoroutineKindShim`, since it's already by-move.
-        if matches!(body.source.instance, ty::InstanceKind::CoroutineKindShim { .. }) {
-            return;
-        }
 
-        let ty::Coroutine(_, args) = *coroutine_ty.kind() else { bug!("{body:#?}") };
+    let parent_def_id = tcx.local_parent(coroutine_def_id);
-        let args = args.as_coroutine();
+    let ty::CoroutineClosure(_, parent_args) =
-
+        *tcx.type_of(parent_def_id).instantiate_identity().kind()
-        let coroutine_kind = args.kind_ty().to_opt_closure_kind().unwrap();
+    else {
-
+        bug!();
-        let parent_def_id = tcx.local_parent(coroutine_def_id);
+    };
-        let ty::CoroutineClosure(_, parent_args) =
+    if parent_args.references_error() {
-            *tcx.type_of(parent_def_id).instantiate_identity().kind()
+        return coroutine_def_id.to_def_id();
-        else {
-            bug!();
-        };
-        let parent_closure_args = parent_args.as_coroutine_closure();
-        let num_args = parent_closure_args
-            .coroutine_closure_sig()
-            .skip_binder()
-            .tupled_inputs_ty
-            .tuple_fields()
-            .len();
-
-        let field_remapping: UnordMap<_, _> = ty::analyze_coroutine_closure_captures(
-            tcx.closure_captures(parent_def_id).iter().copied(),
-            tcx.closure_captures(coroutine_def_id).iter().skip(num_args).copied(),
-            |(parent_field_idx, parent_capture), (child_field_idx, child_capture)| {
-                // Store this set of additional projections (fields and derefs).
-                // We need to re-apply them later.
-                let mut child_precise_captures = child_capture.place.projections
-                    [parent_capture.place.projections.len()..]
-                    .to_vec();
-
-                // If the parent capture is by-ref, then we need to apply an additional
-                // deref before applying any further projections to this place.
-                if parent_capture.is_by_ref() {
-                    child_precise_captures.insert(
-                        0,
-                        Projection { ty: parent_capture.place.ty(), kind: ProjectionKind::Deref },
-                    );
-                }
-                // If the child capture is by-ref, then we need to apply a "ref"
-                // projection (i.e. `&`) at the end. But wait! We don't have that
-                // as a projection kind. So instead, we can apply its dual and
-                // *peel* a deref off of the place when it shows up in the MIR body.
-                // Luckily, by construction this is always possible.
-                let peel_deref = if child_capture.is_by_ref() {
-                    assert!(
-                        parent_capture.is_by_ref() || coroutine_kind != ty::ClosureKind::FnOnce,
-                        "`FnOnce` coroutine-closures return coroutines that capture from \
-                        their body; it will always result in a borrowck error!"
-                    );
-                    true
-                } else {
-                    false
-                };
-
-                // Regarding the behavior above, you may think that it's redundant to both
-                // insert a deref and then peel a deref if the parent and child are both
-                // captured by-ref. This would be correct, except for the case where we have
-                // precise capturing projections, since the inserted deref is to the *beginning*
-                // and the peeled deref is at the *end*. I cannot seem to actually find a
-                // case where this happens, though, but let's keep this code flexible.
-
-                // Finally, store the type of the parent's captured place. We need
-                // this when building the field projection in the MIR body later on.
-                let mut parent_capture_ty = parent_capture.place.ty();
-                parent_capture_ty = match parent_capture.info.capture_kind {
-                    ty::UpvarCapture::ByValue => parent_capture_ty,
-                    ty::UpvarCapture::ByRef(kind) => Ty::new_ref(
-                        tcx,
-                        tcx.lifetimes.re_erased,
-                        parent_capture_ty,
-                        kind.to_mutbl_lossy(),
-                    ),
-                };
-
-                (
-                    FieldIdx::from_usize(child_field_idx + num_args),
-                    (
-                        FieldIdx::from_usize(parent_field_idx + num_args),
-                        parent_capture_ty,
-                        peel_deref,
-                        child_precise_captures,
-                    ),
-                )
-            },
-        )
-        .collect();
-
-        if coroutine_kind == ty::ClosureKind::FnOnce {
-            assert_eq!(field_remapping.len(), tcx.closure_captures(parent_def_id).len());
-            return;
-        }
-
-        let by_move_coroutine_ty = tcx
-            .instantiate_bound_regions_with_erased(parent_closure_args.coroutine_closure_sig())
-            .to_coroutine_given_kind_and_upvars(
-                tcx,
-                parent_closure_args.parent_args(),
-                coroutine_def_id.to_def_id(),
-                ty::ClosureKind::FnOnce,
-                tcx.lifetimes.re_erased,
-                parent_closure_args.tupled_upvars_ty(),
-                parent_closure_args.coroutine_captures_by_ref_ty(),
-            );
-
-        let mut by_move_body = body.clone();
-        MakeByMoveBody { tcx, field_remapping, by_move_coroutine_ty }.visit_body(&mut by_move_body);
-        dump_mir(tcx, false, "coroutine_by_move", &0, &by_move_body, |_, _| Ok(()));
-
-        // Let's just always validate this body.
-        validate_body(tcx, &mut by_move_body, "Initial coroutine_by_move body".to_string());
-
-        // FIXME: use query feeding to generate the body right here and then only store the `DefId` of the new body.
-        by_move_body.source = mir::MirSource::from_instance(InstanceKind::CoroutineKindShim {
-            coroutine_def_id: coroutine_def_id.to_def_id(),
-        });
-        body.coroutine.as_mut().unwrap().by_move_body = Some(by_move_body);
     }
+
+    let parent_closure_args = parent_args.as_coroutine_closure();
+    let num_args = parent_closure_args
+        .coroutine_closure_sig()
+        .skip_binder()
+        .tupled_inputs_ty
+        .tuple_fields()
+        .len();
+
+    let field_remapping: UnordMap<_, _> = ty::analyze_coroutine_closure_captures(
+        tcx.closure_captures(parent_def_id).iter().copied(),
+        tcx.closure_captures(coroutine_def_id).iter().skip(num_args).copied(),
+        |(parent_field_idx, parent_capture), (child_field_idx, child_capture)| {
+            // Store this set of additional projections (fields and derefs).
+            // We need to re-apply them later.
+            let mut child_precise_captures =
+                child_capture.place.projections[parent_capture.place.projections.len()..].to_vec();
+
+            // If the parent capture is by-ref, then we need to apply an additional
+            // deref before applying any further projections to this place.
+            if parent_capture.is_by_ref() {
+                child_precise_captures.insert(
+                    0,
+                    Projection { ty: parent_capture.place.ty(), kind: ProjectionKind::Deref },
+                );
+            }
+            // If the child capture is by-ref, then we need to apply a "ref"
+            // projection (i.e. `&`) at the end. But wait! We don't have that
+            // as a projection kind. So instead, we can apply its dual and
+            // *peel* a deref off of the place when it shows up in the MIR body.
+            // Luckily, by construction this is always possible.
+            let peel_deref = if child_capture.is_by_ref() {
+                assert!(
+                    parent_capture.is_by_ref() || coroutine_kind != ty::ClosureKind::FnOnce,
+                    "`FnOnce` coroutine-closures return coroutines that capture from \
+                        their body; it will always result in a borrowck error!"
+                );
+                true
+            } else {
+                false
+            };
+
+            // Regarding the behavior above, you may think that it's redundant to both
+            // insert a deref and then peel a deref if the parent and child are both
+            // captured by-ref. This would be correct, except for the case where we have
+            // precise capturing projections, since the inserted deref is to the *beginning*
+            // and the peeled deref is at the *end*. I cannot seem to actually find a
+            // case where this happens, though, but let's keep this code flexible.
+
+            // Finally, store the type of the parent's captured place. We need
+            // this when building the field projection in the MIR body later on.
+            let mut parent_capture_ty = parent_capture.place.ty();
+            parent_capture_ty = match parent_capture.info.capture_kind {
+                ty::UpvarCapture::ByValue => parent_capture_ty,
+                ty::UpvarCapture::ByRef(kind) => Ty::new_ref(
+                    tcx,
+                    tcx.lifetimes.re_erased,
+                    parent_capture_ty,
+                    kind.to_mutbl_lossy(),
+                ),
+            };
+
+            (
+                FieldIdx::from_usize(child_field_idx + num_args),
+                (
+                    FieldIdx::from_usize(parent_field_idx + num_args),
+                    parent_capture_ty,
+                    peel_deref,
+                    child_precise_captures,
+                ),
+            )
+        },
+    )
+    .collect();
+
+    if coroutine_kind == ty::ClosureKind::FnOnce {
+        assert_eq!(field_remapping.len(), tcx.closure_captures(parent_def_id).len());
+        // The by-move body is just the body :)
+        return coroutine_def_id.to_def_id();
+    }
+
+    let by_move_coroutine_ty = tcx
+        .instantiate_bound_regions_with_erased(parent_closure_args.coroutine_closure_sig())
+        .to_coroutine_given_kind_and_upvars(
+            tcx,
+            parent_closure_args.parent_args(),
+            coroutine_def_id.to_def_id(),
+            ty::ClosureKind::FnOnce,
+            tcx.lifetimes.re_erased,
+            parent_closure_args.tupled_upvars_ty(),
+            parent_closure_args.coroutine_captures_by_ref_ty(),
+        );
+
+    let mut by_move_body = body.clone();
+    MakeByMoveBody { tcx, field_remapping, by_move_coroutine_ty }.visit_body(&mut by_move_body);
+    dump_mir(tcx, false, "coroutine_by_move", &0, &by_move_body, |_, _| Ok(()));
+
+    let body_def = tcx.create_def(coroutine_def_id, kw::Empty, DefKind::SyntheticCoroutineBody);
+    by_move_body.source =
+        mir::MirSource::from_instance(InstanceKind::Item(body_def.def_id().to_def_id()));
+
+    // Inherited from the by-ref coroutine.
+    body_def.codegen_fn_attrs(tcx.codegen_fn_attrs(coroutine_def_id).clone());
+    body_def.constness(tcx.constness(coroutine_def_id).clone());
+    body_def.coroutine_kind(tcx.coroutine_kind(coroutine_def_id).clone());
+    body_def.def_ident_span(tcx.def_ident_span(coroutine_def_id));
+    body_def.def_span(tcx.def_span(coroutine_def_id));
+    body_def.explicit_predicates_of(tcx.explicit_predicates_of(coroutine_def_id).clone());
+    body_def.generics_of(tcx.generics_of(coroutine_def_id).clone());
+    body_def.param_env(tcx.param_env(coroutine_def_id).clone());
+    body_def.predicates_of(tcx.predicates_of(coroutine_def_id).clone());
+
+    // The type of the coroutine is the `by_move_coroutine_ty`.
+    body_def.type_of(ty::EarlyBinder::bind(by_move_coroutine_ty));
+
+    body_def.mir_built(tcx.arena.alloc(Steal::new(by_move_body)));
+
+    body_def.def_id().to_def_id()
 }
 
 struct MakeByMoveBody<'tcx> {

compiler/rustc_mir_transform/src/inline.rs

@@ -341,7 +341,6 @@ fn check_mir_is_available(
             | InstanceKind::FnPtrShim(..)
             | InstanceKind::ClosureOnceShim { .. }
             | InstanceKind::ConstructCoroutineInClosureShim { .. }
-            | InstanceKind::CoroutineKindShim { .. }
             | InstanceKind::DropGlue(..)
             | InstanceKind::CloneShim(..)
             | InstanceKind::ThreadLocalShim(..)

compiler/rustc_mir_transform/src/inline/cycle.rs

@@ -88,7 +88,6 @@ fn process<'tcx>(
                 | InstanceKind::FnPtrShim(..)
                 | InstanceKind::ClosureOnceShim { .. }
                 | InstanceKind::ConstructCoroutineInClosureShim { .. }
-                | InstanceKind::CoroutineKindShim { .. }
                 | InstanceKind::ThreadLocalShim { .. }
                 | InstanceKind::CloneShim(..) => {}
 

compiler/rustc_mir_transform/src/lib.rs

@@ -135,6 +135,7 @@ pub fn provide(providers: &mut Providers) {
         mir_inliner_callees: inline::cycle::mir_inliner_callees,
         promoted_mir,
         deduced_param_attrs: deduce_param_attrs::deduced_param_attrs,
+        coroutine_by_move_body_def_id: coroutine::coroutine_by_move_body_def_id,
         ..providers.queries
     };
 }
@@ -293,10 +294,6 @@ fn mir_built(tcx: TyCtxt<'_>, def: LocalDefId) -> &Steal<Body<'_>> {
             &Lint(check_packed_ref::CheckPackedRef),
             &Lint(check_const_item_mutation::CheckConstItemMutation),
             &Lint(function_item_references::FunctionItemReferences),
-            // If this is an async closure's output coroutine, generate
-            // by-move and by-mut bodies if needed. We do this first so
-            // they can be optimized in lockstep with their parent bodies.
-            &coroutine::ByMoveBody,
             // What we need to do constant evaluation.
             &simplify::SimplifyCfg::Initial,
             &rustc_peek::SanityCheck, // Just a lint
@@ -329,8 +326,15 @@ fn mir_promoted(
         | DefKind::AnonConst => tcx.mir_const_qualif(def),
         _ => ConstQualifs::default(),
     };
-    // has_ffi_unwind_calls query uses the raw mir, so make sure it is run.
+
+    // the `has_ffi_unwind_calls` query uses the raw mir, so make sure it is run.
     tcx.ensure_with_value().has_ffi_unwind_calls(def);
+
+    // the `by_move_body` query uses the raw mir, so make sure it is run.
+    if tcx.needs_coroutine_by_move_body_def_id(def) {
+        tcx.ensure_with_value().coroutine_by_move_body_def_id(def);
+    }
+
     let mut body = tcx.mir_built(def).steal();
     if let Some(error_reported) = const_qualifs.tainted_by_errors {
         body.tainted_by_errors = Some(error_reported);
@@ -339,14 +343,6 @@ fn mir_promoted(
     // Collect `required_consts` *before* promotion, so if there are any consts being promoted
     // we still add them to the list in the outer MIR body.
     RequiredConstsVisitor::compute_required_consts(&mut body);
-    // If this has an associated by-move async closure body, that doesn't get run through these
-    // passes itself, it gets "tagged along" by the pass manager. `RequiredConstsVisitor` is not
-    // a regular pass so we have to also apply it manually to the other body.
-    if let Some(coroutine) = body.coroutine.as_mut() {
-        if let Some(by_move_body) = coroutine.by_move_body.as_mut() {
-            RequiredConstsVisitor::compute_required_consts(by_move_body);
-        }
-    }
 
     // What we need to run borrowck etc.
     let promote_pass = promote_consts::PromoteTemps::default();
@@ -398,7 +394,10 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> &
     if tcx.is_coroutine(def.to_def_id()) {
         tcx.ensure_with_value().mir_coroutine_witnesses(def);
     }
-    let mir_borrowck = tcx.mir_borrowck(def);
+
+    // We only need to borrowck non-synthetic MIR.
+    let tainted_by_errors =
+        if !tcx.is_synthetic_mir(def) { tcx.mir_borrowck(def).tainted_by_errors } else { None };
 
     let is_fn_like = tcx.def_kind(def).is_fn_like();
     if is_fn_like {
@@ -410,7 +409,8 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> &
 
     let (body, _) = tcx.mir_promoted(def);
     let mut body = body.steal();
-    if let Some(error_reported) = mir_borrowck.tainted_by_errors {
+
+    if let Some(error_reported) = tainted_by_errors {
         body.tainted_by_errors = Some(error_reported);
     }
 
@@ -660,14 +660,6 @@ fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> {
     // visited does not depend on the optimization level.
     // We do not use `run_passes` for this as that might skip the pass if `injection_phase` is set.
     mentioned_items::MentionedItems.run_pass(tcx, &mut body);
-    // If this has an associated by-move async closure body, that doesn't get run through these
-    // passes itself, it gets "tagged along" by the pass manager. Since we're not using the pass
-    // manager we have to do this by hand.
-    if let Some(coroutine) = body.coroutine.as_mut() {
-        if let Some(by_move_body) = coroutine.by_move_body.as_mut() {
-            mentioned_items::MentionedItems.run_pass(tcx, by_move_body);
-        }
-    }
 
     // If `mir_drops_elaborated_and_const_checked` found that the current body has unsatisfiable
     // predicates, it will shrink the MIR to a single `unreachable` terminator.
@@ -690,7 +682,9 @@ fn promoted_mir(tcx: TyCtxt<'_>, def: LocalDefId) -> &IndexVec<Promoted, Body<'_
         return tcx.arena.alloc(IndexVec::new());
     }
 
-    tcx.ensure_with_value().mir_borrowck(def);
+    if !tcx.is_synthetic_mir(def) {
+        tcx.ensure_with_value().mir_borrowck(def);
+    }
     let mut promoted = tcx.mir_promoted(def).1.steal();
 
     for body in &mut promoted {

compiler/rustc_mir_transform/src/pass_manager.rs

@@ -182,12 +182,6 @@ fn run_passes_inner<'tcx>(
 
         body.pass_count = 1;
     }
-
-    if let Some(coroutine) = body.coroutine.as_mut() {
-        if let Some(by_move_body) = coroutine.by_move_body.as_mut() {
-            run_passes_inner(tcx, by_move_body, passes, phase_change, validate_each);
-        }
-    }
 }
 
 pub fn validate_body<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, when: String) {

compiler/rustc_mir_transform/src/shim.rs

@@ -78,15 +78,11 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body<
             receiver_by_ref,
         } => build_construct_coroutine_by_move_shim(tcx, coroutine_closure_def_id, receiver_by_ref),
 
-        ty::InstanceKind::CoroutineKindShim { coroutine_def_id } => {
-            return tcx.optimized_mir(coroutine_def_id).coroutine_by_move_body().unwrap().clone();
-        }
-
         ty::InstanceKind::DropGlue(def_id, ty) => {
             // FIXME(#91576): Drop shims for coroutines aren't subject to the MIR passes at the end
             // of this function. Is this intentional?
-            if let Some(ty::Coroutine(coroutine_def_id, args)) = ty.map(Ty::kind) {
+            if let Some(&ty::Coroutine(coroutine_def_id, args)) = ty.map(Ty::kind) {
-                let coroutine_body = tcx.optimized_mir(*coroutine_def_id);
+                let coroutine_body = tcx.optimized_mir(coroutine_def_id);
 
                 let ty::Coroutine(_, id_args) = *tcx.type_of(coroutine_def_id).skip_binder().kind()
                 else {
@@ -105,7 +101,9 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body<
                         args.as_coroutine().kind_ty().to_opt_closure_kind().unwrap(),
                         ty::ClosureKind::FnOnce
                     );
-                    coroutine_body.coroutine_by_move_body().unwrap().coroutine_drop().unwrap()
+                    tcx.optimized_mir(tcx.coroutine_by_move_body_def_id(coroutine_def_id))
+                        .coroutine_drop()
+                        .unwrap()
                 };
 
                 let mut body = EarlyBinder::bind(body.clone()).instantiate(tcx, args);

compiler/rustc_mir_transform/src/validate.rs

@@ -102,25 +102,6 @@ fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
                 }
             }
         }
-
-        // Enforce that coroutine-closure layouts are identical.
-        if let Some(layout) = body.coroutine_layout_raw()
-            && let Some(by_move_body) = body.coroutine_by_move_body()
-            && let Some(by_move_layout) = by_move_body.coroutine_layout_raw()
-        {
-            // FIXME(async_closures): We could do other validation here?
-            if layout.variant_fields.len() != by_move_layout.variant_fields.len() {
-                cfg_checker.fail(
-                    Location::START,
-                    format!(
-                        "Coroutine layout has different number of variant fields from \
-                        by-move coroutine layout:\n\
-                        layout: {layout:#?}\n\
-                        by_move_layout: {by_move_layout:#?}",
-                    ),
-                );
-            }
-        }
     }
 }
 

compiler/rustc_monomorphize/src/collector.rs

@@ -920,7 +920,6 @@ fn visit_instance_use<'tcx>(
         | ty::InstanceKind::ReifyShim(..)
         | ty::InstanceKind::ClosureOnceShim { .. }
         | ty::InstanceKind::ConstructCoroutineInClosureShim { .. }
-        | ty::InstanceKind::CoroutineKindShim { .. }
         | ty::InstanceKind::Item(..)
         | ty::InstanceKind::FnPtrShim(..)
         | ty::InstanceKind::CloneShim(..)

compiler/rustc_monomorphize/src/partitioning.rs

@@ -626,7 +626,6 @@ fn characteristic_def_id_of_mono_item<'tcx>(
                 | ty::InstanceKind::FnPtrShim(..)
                 | ty::InstanceKind::ClosureOnceShim { .. }
                 | ty::InstanceKind::ConstructCoroutineInClosureShim { .. }
-                | ty::InstanceKind::CoroutineKindShim { .. }
                 | ty::InstanceKind::Intrinsic(..)
                 | ty::InstanceKind::DropGlue(..)
                 | ty::InstanceKind::Virtual(..)
@@ -796,7 +795,6 @@ fn mono_item_visibility<'tcx>(
         | InstanceKind::Intrinsic(..)
         | InstanceKind::ClosureOnceShim { .. }
         | InstanceKind::ConstructCoroutineInClosureShim { .. }
-        | InstanceKind::CoroutineKindShim { .. }
         | InstanceKind::DropGlue(..)
         | InstanceKind::AsyncDropGlueCtorShim(..)
         | InstanceKind::CloneShim(..)

compiler/rustc_monomorphize/src/polymorphize.rs

@@ -127,7 +127,7 @@ fn mark_used_by_default_parameters<'tcx>(
     unused_parameters: &mut UnusedGenericParams,
 ) {
     match tcx.def_kind(def_id) {
-        DefKind::Closure => {
+        DefKind::Closure | DefKind::SyntheticCoroutineBody => {
             for param in &generics.own_params {
                 debug!(?param, "(closure/gen)");
                 unused_parameters.mark_used(param.index);

compiler/rustc_privacy/src/lib.rs

@@ -626,7 +626,8 @@ fn update_macro_reachable_def(
             | DefKind::Field
             | DefKind::GlobalAsm
             | DefKind::Impl { .. }
-            | DefKind::Closure => (),
+            | DefKind::Closure
+            | DefKind::SyntheticCoroutineBody => (),
         }
     }
 }

compiler/rustc_resolve/src/build_reduced_graph.rs

@@ -260,6 +260,7 @@ fn build_reduced_graph_for_external_crate_res(
                 | DefKind::LifetimeParam
                 | DefKind::GlobalAsm
                 | DefKind::Closure
+                | DefKind::SyntheticCoroutineBody
                 | DefKind::Impl { .. },
                 _,
             )

compiler/rustc_smir/src/rustc_smir/convert/ty.rs

@@ -851,7 +851,6 @@ fn stable(&self, tables: &mut Tables<'_>) -> Self::T {
             | ty::InstanceKind::FnPtrAddrShim(..)
             | ty::InstanceKind::ClosureOnceShim { .. }
             | ty::InstanceKind::ConstructCoroutineInClosureShim { .. }
-            | ty::InstanceKind::CoroutineKindShim { .. }
             | ty::InstanceKind::ThreadLocalShim(..)
             | ty::InstanceKind::DropGlue(..)
             | ty::InstanceKind::CloneShim(..)

compiler/rustc_smir/src/rustc_smir/mod.rs

@@ -114,7 +114,9 @@ pub(crate) fn new_item_kind(kind: DefKind) -> ItemKind {
         | DefKind::GlobalAsm => {
             unreachable!("Not a valid item kind: {kind:?}");
         }
-        DefKind::Closure | DefKind::AssocFn | DefKind::Fn => ItemKind::Fn,
+        DefKind::Closure | DefKind::AssocFn | DefKind::Fn | DefKind::SyntheticCoroutineBody => {
+            ItemKind::Fn
+        }
         DefKind::Const | DefKind::InlineConst | DefKind::AssocConst | DefKind::AnonConst => {
             ItemKind::Const
         }

compiler/rustc_symbol_mangling/src/legacy.rs

@@ -92,9 +92,6 @@ pub(super) fn mangle<'tcx>(
                 .write_str(if receiver_by_ref { "{{by-move-shim}}" } else { "{{by-ref-shim}}" })
                 .unwrap();
         }
-        ty::InstanceKind::CoroutineKindShim { .. } => {
-            printer.write_str("{{by-move-body-shim}}").unwrap();
-        }
         _ => {}
     }
 

compiler/rustc_symbol_mangling/src/v0.rs

@@ -56,7 +56,6 @@ pub(super) fn mangle<'tcx>(
         ty::InstanceKind::ConstructCoroutineInClosureShim { receiver_by_ref: false, .. } => {
             Some("by_ref")
         }
-        ty::InstanceKind::CoroutineKindShim { .. } => Some("by_move_body"),
 
         _ => None,
     };

compiler/rustc_ty_utils/src/abi.rs

@@ -169,39 +169,6 @@ fn fn_sig_for_fn_abi<'tcx>(
                 kind: ty::BoundRegionKind::BrEnv,
             };
 
-            let mut ty = ty;
-            // When this `Closure` comes from a `CoroutineKindShim`,
-            // make sure we respect the `target_kind` in that shim.
-            // FIXME(async_closures): This shouldn't be needed, and we should be populating
-            // a separate def-id for these bodies.
-            if let InstanceKind::CoroutineKindShim { .. } = instance.def {
-                // Grab the parent coroutine-closure. It has the same args for the purposes
-                // of instantiation, so this will be okay to do.
-                let ty::CoroutineClosure(_, coroutine_closure_args) = *tcx
-                    .instantiate_and_normalize_erasing_regions(
-                        args,
-                        param_env,
-                        tcx.type_of(tcx.parent(did)),
-                    )
-                    .kind()
-                else {
-                    bug!("CoroutineKindShim comes from calling a coroutine-closure");
-                };
-                let coroutine_closure_args = coroutine_closure_args.as_coroutine_closure();
-                ty = tcx.instantiate_bound_regions_with_erased(
-                    coroutine_closure_args.coroutine_closure_sig().map_bound(|sig| {
-                        sig.to_coroutine_given_kind_and_upvars(
-                            tcx,
-                            coroutine_closure_args.parent_args(),
-                            did,
-                            ty::ClosureKind::FnOnce,
-                            tcx.lifetimes.re_erased,
-                            coroutine_closure_args.tupled_upvars_ty(),
-                            coroutine_closure_args.coroutine_captures_by_ref_ty(),
-                        )
-                    }),
-                );
-            }
             let env_ty = Ty::new_mut_ref(tcx, ty::Region::new_bound(tcx, ty::INNERMOST, br), ty);
 
             let pin_did = tcx.require_lang_item(LangItem::Pin, None);

compiler/rustc_ty_utils/src/implied_bounds.rs

@@ -144,7 +144,8 @@ fn assumed_wf_types<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> &'tcx [(Ty<'
         | DefKind::Field
         | DefKind::LifetimeParam
         | DefKind::GlobalAsm
-        | DefKind::Closure => ty::List::empty(),
+        | DefKind::Closure
+        | DefKind::SyntheticCoroutineBody => ty::List::empty(),
     }
 }
 

compiler/rustc_ty_utils/src/opaque_types.rs

@@ -348,7 +348,8 @@ fn opaque_types_defined_by<'tcx>(
         | DefKind::Field
         | DefKind::LifetimeParam
         | DefKind::GlobalAsm
-        | DefKind::Impl { .. } => {}
+        | DefKind::Impl { .. }
+        | DefKind::SyntheticCoroutineBody => {}
         // Closures and coroutines are type checked with their parent, so we need to allow all
         // opaques from the closure signature *and* from the parent body.
         DefKind::Closure | DefKind::InlineConst => {

compiler/rustc_ty_utils/src/sig_types.rs

@@ -87,7 +87,7 @@ pub fn walk_types<'tcx, V: SpannedTypeVisitor<'tcx>>(
         // These are not part of a public API, they can only appear as hidden types, and there
         // the interesting parts are solely in the signature of the containing item's opaque type
         // or dyn type.
-        DefKind::InlineConst | DefKind::Closure => {}
+        DefKind::InlineConst | DefKind::Closure | DefKind::SyntheticCoroutineBody => {}
         DefKind::Impl { of_trait } => {
             if of_trait {
                 let span = tcx.hir_node_by_def_id(item).expect_item().expect_impl().of_trait.unwrap().path.span;

src/librustdoc/formats/item_type.rs

@@ -162,7 +162,8 @@ pub(crate) fn from_def_kind(kind: DefKind, parent_kind: Option<DefKind>) -> Self
             | DefKind::LifetimeParam
             | DefKind::GlobalAsm
             | DefKind::Impl { .. }
-            | DefKind::Closure => Self::ForeignType,
+            | DefKind::Closure
+            | DefKind::SyntheticCoroutineBody => Self::ForeignType,
         }
     }
 

src/librustdoc/passes/collect_intra_doc_links.rs

@@ -1950,7 +1950,9 @@ fn resolution_failure(
                             | TraitAlias
                             | TyParam
                             | Static { .. } => "associated item",
-                            Impl { .. } | GlobalAsm => unreachable!("not a path"),
+                            Impl { .. } | GlobalAsm | SyntheticCoroutineBody => {
+                                unreachable!("not a path")
+                            }
                         }
                     } else {
                         "associated item"

tests/ui/lint/unused/lint-unused-variables.stderr

@@ -16,6 +16,12 @@ error: unused variable: `a`
 LL |         a: i32,
    |         ^ help: if this is intentional, prefix it with an underscore: `_a`
 
+error: unused variable: `a`
+  --> $DIR/lint-unused-variables.rs:68:9
+   |
+LL |         a: i32,
+   |         ^ help: if this is intentional, prefix it with an underscore: `_a`
+
 error: unused variable: `b`
   --> $DIR/lint-unused-variables.rs:14:5
    |
@@ -58,12 +64,6 @@ error: unused variable: `b`
 LL |         b: i32,
    |         ^ help: if this is intentional, prefix it with an underscore: `_b`
 
-error: unused variable: `a`
-  --> $DIR/lint-unused-variables.rs:68:9
-   |
-LL |         a: i32,
-   |         ^ help: if this is intentional, prefix it with an underscore: `_a`
-
 error: unused variable: `b`
   --> $DIR/lint-unused-variables.rs:74:9
    |

tests/ui/rfcs/rfc-2565-param-attrs/param-attrs-cfg.stderr

@@ -16,6 +16,12 @@ error: unused variable: `a`
 LL |         #[cfg(something)] a: i32,
    |                           ^ help: if this is intentional, prefix it with an underscore: `_a`
 
+error: unused variable: `a`
+  --> $DIR/param-attrs-cfg.rs:107:27
+   |
+LL |         #[cfg(something)] a: i32,
+   |                           ^ help: if this is intentional, prefix it with an underscore: `_a`
+
 error: unused variable: `b`
   --> $DIR/param-attrs-cfg.rs:30:23
    |
@@ -100,12 +106,6 @@ error: unused variable: `c`
 LL |         #[cfg_attr(nothing, cfg(nothing))] c: i32,
    |                                            ^ help: if this is intentional, prefix it with an underscore: `_c`
 
-error: unused variable: `a`
-  --> $DIR/param-attrs-cfg.rs:107:27
-   |
-LL |         #[cfg(something)] a: i32,
-   |                           ^ help: if this is intentional, prefix it with an underscore: `_a`
-
 error: unused variable: `b`
   --> $DIR/param-attrs-cfg.rs:113:27
    |