Expand async drops during drop elaboration

compiler/rustc_borrowck/src/lib.rs

@@ -881,14 +881,7 @@ impl<'a, 'tcx> ResultsVisitor<'tcx, Borrowck<'a, 'tcx>> for MirBorrowckCtxt<'a,
             TerminatorKind::SwitchInt { discr, targets: _ } => {
                 self.consume_operand(loc, (discr, span), state);
             }
-            TerminatorKind::Drop {
-                place,
-                target: _,
-                unwind: _,
-                replace,
-                drop: _,
-                async_fut: _,
-            } => {
+            TerminatorKind::Drop { place, target: _, unwind: _, replace, drop: _ } => {
                 debug!(
                     "visit_terminator_drop \
                      loc: {:?} term: {:?} place: {:?} span: {:?}",

compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs

@@ -97,14 +97,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'a, 'tcx> {
             TerminatorKind::SwitchInt { discr, targets: _ } => {
                 self.consume_operand(location, discr);
             }
-            TerminatorKind::Drop {
-                place: drop_place,
-                target: _,
-                unwind: _,
-                replace,
-                drop: _,
-                async_fut: _,
-            } => {
+            TerminatorKind::Drop { place: drop_place, target: _, unwind: _, replace, drop: _ } => {
                 let write_kind =
                     if *replace { WriteKind::Replace } else { WriteKind::StorageDeadOrDrop };
                 self.access_place(

compiler/rustc_codegen_cranelift/src/base.rs

@@ -582,9 +582,9 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
             | TerminatorKind::CoroutineDrop => {
                 bug!("shouldn't exist at codegen {:?}", bb_data.terminator());
             }
-            TerminatorKind::Drop { place, target, unwind, replace: _, drop, async_fut } => {
+            TerminatorKind::Drop { place, target, unwind, replace: _, drop } => {
                 assert!(
-                    async_fut.is_none() && drop.is_none(),
+                    drop.is_none(),
                     "Async Drop must be expanded or reset to sync before codegen"
                 );
                 let drop_place = codegen_place(fx, *place);

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -1548,9 +1548,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 MergingSucc::False
             }
 
-            mir::TerminatorKind::Drop { place, target, unwind, replace: _, drop, async_fut } => {
+            mir::TerminatorKind::Drop { place, target, unwind, replace: _, drop } => {
                 assert!(
-                    async_fut.is_none() && drop.is_none(),
+                    drop.is_none(),
                     "Async Drop must be expanded or reset to sync before codegen"
                 );
                 self.codegen_drop_terminator(

compiler/rustc_const_eval/src/interpret/step.rs

@@ -590,9 +590,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 }
             }
 
-            Drop { place, target, unwind, replace: _, drop, async_fut } => {
+            Drop { place, target, unwind, replace: _, drop } => {
                 assert!(
-                    async_fut.is_none() && drop.is_none(),
+                    drop.is_none(),
                     "Async Drop must be expanded or reset to sync in runtime MIR"
                 );
                 let place = self.eval_place(place)?;

compiler/rustc_hir/src/hir.rs

@@ -2327,6 +2327,13 @@ impl CoroutineKind {
         matches!(self, CoroutineKind::Desugared(_, CoroutineSource::Fn))
     }
 
+    pub fn is_async_desugaring(self) -> bool {
+        matches!(
+            self,
+            CoroutineKind::Desugared(CoroutineDesugaring::Async | CoroutineDesugaring::AsyncGen, _)
+        )
+    }
+
     pub fn to_plural_string(&self) -> String {
         match self {
             CoroutineKind::Desugared(d, CoroutineSource::Fn) => format!("{d:#}fn bodies"),

compiler/rustc_middle/src/mir/pretty.rs

@@ -15,6 +15,7 @@ use crate::mir::interpret::{
 };
 use crate::mir::visit::Visitor;
 use crate::mir::*;
+use crate::ty::CoroutineArgsExt;
 
 const INDENT: &str = "    ";
 /// Alignment for lining up comments following MIR statements
@@ -185,9 +186,6 @@ impl<'a, 'tcx> MirDumper<'a, 'tcx> {
             Some(promoted) => write!(w, "::{promoted:?}`")?,
         }
         writeln!(w, " {} {}", self.disambiguator, self.pass_name)?;
-        if let Some(ref layout) = body.coroutine_layout_raw() {
-            writeln!(w, "/* coroutine_layout = {layout:#?} */")?;
-        }
         writeln!(w)?;
         (self.writer.extra_data)(PassWhere::BeforeCFG, w)?;
         write_user_type_annotations(self.tcx(), body, w)?;
@@ -429,6 +427,31 @@ fn write_scope_tree(
         }
     }
 
+    // Coroutine debuginfo.
+    if let Some(layout) = body.coroutine_layout_raw() {
+        for (field, name) in layout.field_names.iter_enumerated() {
+            if let Some(name) = name
+                && let source_info = layout.field_tys[field].source_info
+                && source_info.scope == parent
+            {
+                let indented_debug_info =
+                    format!("{0:1$}coroutine debug {2} => {3:?};", INDENT, indent, name, field);
+
+                if options.include_extra_comments {
+                    writeln!(
+                        w,
+                        "{0:1$} // in {2}",
+                        indented_debug_info,
+                        ALIGN,
+                        comment(tcx, source_info),
+                    )?;
+                } else {
+                    writeln!(w, "{indented_debug_info}")?;
+                }
+            }
+        }
+    }
+
     // Local variable types.
     for (local, local_decl) in body.local_decls.iter_enumerated() {
         if (1..body.arg_count + 1).contains(&local.index()) {
@@ -530,6 +553,50 @@ impl Debug for VarDebugInfo<'_> {
     }
 }
 
+fn write_coroutine_layout<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    layout: &CoroutineLayout<'_>,
+    w: &mut dyn io::Write,
+    options: PrettyPrintMirOptions,
+) -> io::Result<()> {
+    let CoroutineLayout {
+        field_tys,
+        field_names: _, // Dumped in scope tree with debug info.
+        variant_fields,
+        variant_source_info,
+        storage_conflicts,
+    } = layout;
+
+    writeln!(w, "{INDENT}coroutine layout {{")?;
+
+    for (field, CoroutineSavedTy { ty, source_info, ignore_for_traits }) in
+        field_tys.iter_enumerated()
+    {
+        let ignore_for_traits = if *ignore_for_traits { " (ignored for traits)" } else { "" };
+        let indented_body = format!("{INDENT}{INDENT}field {field:?}: {ty}{ignore_for_traits};",);
+        if options.include_extra_comments {
+            writeln!(w, "{0:ALIGN$} // in {1}", indented_body, comment(tcx, *source_info))?;
+        } else {
+            writeln!(w, "{}", indented_body)?;
+        }
+    }
+
+    writeln!(w, "{INDENT}{INDENT}variant_fields = {{")?;
+    for (variant, fields) in variant_fields.iter_enumerated() {
+        let variant_name = ty::CoroutineArgs::variant_name(variant);
+        let header = format!("{INDENT}{INDENT}{INDENT}{variant_name:9}({variant:?}): {fields:?},");
+        if options.include_extra_comments {
+            let source_info = variant_source_info[variant];
+            writeln!(w, "{0:ALIGN$} // in {1}", header, comment(tcx, source_info))?;
+        } else {
+            writeln!(w, "{}", header)?;
+        }
+    }
+    writeln!(w, "{INDENT}{INDENT}}}")?;
+    writeln!(w, "{INDENT}{INDENT}storage_conflicts = {storage_conflicts:?}")?;
+    writeln!(w, "{INDENT}}}")
+}
+
 /// Write out a human-readable textual representation of the MIR's `fn` type and the types of its
 /// local variables (both user-defined bindings and compiler temporaries).
 fn write_mir_intro<'tcx>(
@@ -541,6 +608,10 @@ fn write_mir_intro<'tcx>(
     write_mir_sig(tcx, body, w)?;
     writeln!(w, "{{")?;
 
+    if let Some(ref layout) = body.coroutine_layout_raw() {
+        write_coroutine_layout(tcx, layout, w, options)?;
+    }
+
     // construct a scope tree and write it out
     let mut scope_tree: FxHashMap<SourceScope, Vec<SourceScope>> = Default::default();
     for (index, scope_data) in body.source_scopes.iter_enumerated() {
@@ -920,10 +991,7 @@ impl<'tcx> TerminatorKind<'tcx> {
             }
             Yield { value, resume_arg, .. } => write!(fmt, "{resume_arg:?} = yield({value:?})"),
             Unreachable => write!(fmt, "unreachable"),
-            Drop { place, async_fut: None, .. } => write!(fmt, "drop({place:?})"),
-            Drop { place, async_fut: Some(async_fut), .. } => {
-                write!(fmt, "async drop({place:?}; poll={async_fut:?})")
-            }
+            Drop { place, .. } => write!(fmt, "drop({place:?})"),
             Call { func, args, destination, .. } => {
                 write!(fmt, "{destination:?} = ")?;
                 write!(fmt, "{func:?}(")?;

compiler/rustc_middle/src/mir/statement.rs

@@ -624,11 +624,7 @@ impl<'tcx> Operand<'tcx> {
         span: Span,
     ) -> Self {
         let ty = Ty::new_fn_def(tcx, def_id, args);
-        Operand::Constant(Box::new(ConstOperand {
-            span,
-            user_ty: None,
-            const_: Const::Val(ConstValue::ZeroSized, ty),
-        }))
+        Operand::zero_sized_constant(ty, span)
     }
 
     /// Convenience helper to make a constant that refers to the given `DefId` and args. Since this
@@ -643,6 +639,12 @@ impl<'tcx> Operand<'tcx> {
         Operand::Constant(Box::new(ConstOperand { span, user_ty: None, const_ }))
     }
 
+    /// Convenience helper to make a constant that refers to a zero-sized type.
+    pub fn zero_sized_constant(ty: Ty<'tcx>, span: Span) -> Self {
+        let const_ = Const::Val(ConstValue::ZeroSized, ty);
+        Operand::Constant(Box::new(ConstOperand { span, user_ty: None, const_ }))
+    }
+
     pub fn is_move(&self) -> bool {
         matches!(self, Operand::Move(..))
     }

compiler/rustc_middle/src/mir/syntax.rs

@@ -760,21 +760,16 @@ pub enum TerminatorKind<'tcx> {
     /// meaning.
     ///
     /// Async drop processing:
-    /// In compiler/rustc_mir_build/src/build/scope.rs we detect possible async drop:
-    ///   drop of object with `needs_async_drop`.
-    /// Async drop later, in StateTransform pass, may be expanded into additional yield-point
-    ///   for poll-loop of async drop future.
-    /// So we need prepared 'drop' target block in the similar way as for `Yield` terminator
-    ///   (see `drops.build_mir::<CoroutineDrop>` in scopes.rs).
-    /// In compiler/rustc_mir_transform/src/elaborate_drops.rs for object implementing `AsyncDrop` trait
-    ///   we need to prepare async drop feature - resolve `AsyncDrop::drop` and codegen call.
-    /// `async_fut` is set to the corresponding local.
-    /// For coroutine drop we don't need this logic because coroutine drop works with the same
-    ///   layout object as coroutine itself. So `async_fut` will be `None` for coroutine drop.
-    /// Both `drop` and `async_fut` fields are only used in compiler/rustc_mir_transform/src/coroutine.rs,
-    ///   StateTransform pass. In `expand_async_drops` async drops are expanded
-    ///   into one or two yield points with poll ready/pending switch.
-    /// When a coroutine has any internal async drop, the coroutine drop function will be async
+    ///   MIR building detects possible async drops, and constructs a complete CFG. To correctly
+    ///   handle the coroutine being dropped while itself drops, we need a 'drop' target
+    ///   similar to `Yield` terminator (see `drops.build_mir::<CoroutineDrop>`).
+    ///
+    ///   Drop elaboration later refines the set of useful async drops. If there is no need for an
+    ///   async drop, it is downgraded to a sync drop by setting `drop` to `None` If this is an
+    ///   actual async drop, it is expanded to an `await` loop over the `async_drop_in_place` or
+    ///   `AsyncDrop::drop` coroutine.
+    ///
+    ///   When a coroutine has any internal async drop, the coroutine drop function will be async
     ///   (generated by `create_coroutine_drop_shim_async`, not `create_coroutine_drop_shim`).
     Drop {
         place: Place<'tcx>,
@@ -783,8 +778,6 @@ pub enum TerminatorKind<'tcx> {
         replace: bool,
         /// Cleanup to be done if the coroutine is dropped at this suspend point (for async drop).
         drop: Option<BasicBlock>,
-        /// Prepared async future local (for async drop)
-        async_fut: Option<Local>,
     },
 
     /// Roughly speaking, evaluates the `func` operand and the arguments, and starts execution of

compiler/rustc_middle/src/mir/terminator.rs

@@ -743,7 +743,7 @@ impl<'tcx> TerminatorKind<'tcx> {
             // FIXME: Maybe we need also TerminatorEdges::Trio for async drop
             // (target + unwind + dropline)
             Assert { target, unwind, expected: _, msg: _, cond: _ }
-            | Drop { target, unwind, place: _, replace: _, drop: _, async_fut: _ }
+            | Drop { target, unwind, place: _, replace: _, drop: _ }
             | FalseUnwind { real_target: target, unwind } => match unwind {
                 UnwindAction::Cleanup(unwind) => TerminatorEdges::Double(target, unwind),
                 UnwindAction::Continue | UnwindAction::Terminate(_) | UnwindAction::Unreachable => {

compiler/rustc_middle/src/mir/visit.rs

@@ -555,20 +555,12 @@ macro_rules! make_mir_visitor {
                         unwind: _,
                         replace: _,
                         drop: _,
-                        async_fut,
                     } => {
                         self.visit_place(
                             place,
                             PlaceContext::MutatingUse(MutatingUseContext::Drop),
                             location
                         );
-                        if let Some(async_fut) = async_fut {
-                            self.visit_local(
-                                $(&$mutability)? *async_fut,
-                                PlaceContext::MutatingUse(MutatingUseContext::Borrow),
-                                location
-                            );
-                        }
                     }
 
                     TerminatorKind::Call {

compiler/rustc_mir_build/src/builder/custom/parse/instruction.rs

@@ -63,7 +63,6 @@ impl<'a, 'tcx> ParseCtxt<'a, 'tcx> {
                     unwind: self.parse_unwind_action(args[2])?,
                     replace: false,
                     drop: None,
-                    async_fut: None,
                 })
             },
             @call(mir_call, args) => {

compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs

@@ -642,7 +642,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         unwind: UnwindAction::Continue,
                         replace: false,
                         drop: None,
-                        async_fut: None,
                     },
                 );
                 this.diverge_from(block);

compiler/rustc_mir_build/src/builder/scope.rs

@@ -427,7 +427,6 @@ impl DropTree {
                         place: drop_node.data.local.into(),
                         replace: false,
                         drop: None,
-                        async_fut: None,
                     };
                     cfg.terminate(block, drop_node.data.source_info, terminator);
                 }
@@ -1170,7 +1169,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                                 unwind: UnwindAction::Continue,
                                 replace: false,
                                 drop: None,
-                                async_fut: None,
                             },
                         );
                         block = next;
@@ -1745,7 +1743,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 unwind: UnwindAction::Cleanup(assign_unwind),
                 replace: true,
                 drop: None,
-                async_fut: None,
             },
         );
         self.diverge_from(block);
@@ -1916,7 +1913,6 @@ where
                         unwind: UnwindAction::Continue,
                         replace: false,
                         drop: None,
-                        async_fut: None,
                     },
                 );
                 block = next;

compiler/rustc_mir_dataflow/src/impls/initialized.rs

@@ -412,14 +412,8 @@ impl<'tcx> Analysis<'tcx> for MaybeInitializedPlaces<'_, 'tcx> {
         // the result of `is_unwind_dead`.
         let mut edges = terminator.edges();
         if self.skip_unreachable_unwind
-            && let mir::TerminatorKind::Drop {
-                target,
-                unwind,
-                place,
-                replace: _,
-                drop: _,
-                async_fut: _,
-            } = terminator.kind
+            && let mir::TerminatorKind::Drop { target, unwind, place, replace: _, drop: _ } =
+                terminator.kind
             && matches!(unwind, mir::UnwindAction::Cleanup(_))
             && self.is_unwind_dead(place, state)
         {

compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs

@@ -83,9 +83,7 @@ fn add_move_for_packed_drop<'tcx>(
     is_cleanup: bool,
 ) {
     debug!("add_move_for_packed_drop({:?} @ {:?})", terminator, loc);
-    let TerminatorKind::Drop { ref place, target, unwind, replace, drop, async_fut } =
-        terminator.kind
-    else {
+    let TerminatorKind::Drop { ref place, target, unwind, replace, drop } = terminator.kind else {
         unreachable!();
     };
 
@@ -109,7 +107,6 @@ fn add_move_for_packed_drop<'tcx>(
             unwind,
             replace,
             drop,
-            async_fut,
         },
     );
 }