Add post-mono MIR optimizations

compiler/rustc_attr_parsing/src/attributes/prototype.rs

@@ -93,6 +93,7 @@ fn parse_dialect(
         sym::analysis => MirDialect::Analysis,
         sym::built => MirDialect::Built,
         sym::runtime => MirDialect::Runtime,
+        sym::mono => MirDialect::Mono,
 
         _ => {
             cx.adcx().expected_specific_argument(span, &[sym::analysis, sym::built, sym::runtime]);
@@ -136,43 +137,28 @@ fn check_custom_mir(
     failed: &mut bool,
 ) {
     let attr_span = cx.attr_span;
+    let Some((phase, phase_span)) = phase else { return };
     let Some((dialect, dialect_span)) = dialect else {
-        if let Some((_, phase_span)) = phase {
+        *failed = true;
+        cx.emit_err(session_diagnostics::CustomMirPhaseRequiresDialect { attr_span, phase_span });
+        return;
+    };
+
+    match (dialect, phase) {
+        (MirDialect::Built, _)
+        | (MirDialect::Analysis, MirPhase::Optimized)
+        | (MirDialect::Mono, MirPhase::PostCleanup) => {
             *failed = true;
-            cx.emit_err(session_diagnostics::CustomMirPhaseRequiresDialect {
+            cx.emit_err(session_diagnostics::CustomMirIncompatibleDialectAndPhase {
+                dialect,
+                phase,
                 attr_span,
+                dialect_span,
                 phase_span,
             });
         }
-        return;
-    };
-
-    match dialect {
-        MirDialect::Analysis => {
-            if let Some((MirPhase::Optimized, phase_span)) = phase {
-                *failed = true;
-                cx.emit_err(session_diagnostics::CustomMirIncompatibleDialectAndPhase {
-                    dialect,
-                    phase: MirPhase::Optimized,
-                    attr_span,
-                    dialect_span,
-                    phase_span,
-                });
-            }
-        }
-
-        MirDialect::Built => {
-            if let Some((phase, phase_span)) = phase {
-                *failed = true;
-                cx.emit_err(session_diagnostics::CustomMirIncompatibleDialectAndPhase {
-                    dialect,
-                    phase,
-                    attr_span,
-                    dialect_span,
-                    phase_span,
-                });
-            }
-        }
-        MirDialect::Runtime => {}
+        (MirDialect::Analysis, MirPhase::Initial | MirPhase::PostCleanup)
+        | (MirDialect::Runtime, MirPhase::Initial | MirPhase::PostCleanup | MirPhase::Optimized)
+        | (MirDialect::Mono, MirPhase::Initial | MirPhase::Optimized) => {}
     }
 }

compiler/rustc_codegen_cranelift/src/abi/mod.rs

@@ -263,7 +263,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
         .mir
         .args_iter()
         .map(|local| {
-            let arg_ty = fx.monomorphize(fx.mir.local_decls[local].ty);
+            let arg_ty = fx.mir.local_decls[local].ty;
 
             // Adapted from https://github.com/rust-lang/rust/blob/145155dc96757002c7b2e9de8489416e2fdbbd57/src/librustc_codegen_llvm/mir/mod.rs#L442-L482
             if Some(local) == fx.mir.spread_arg {
@@ -352,7 +352,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
     }
 
     for local in fx.mir.vars_and_temps_iter() {
-        let ty = fx.monomorphize(fx.mir.local_decls[local].ty);
+        let ty = fx.mir.local_decls[local].ty;
         let layout = fx.layout_of(ty);
 
         let is_ssa = ssa_analyzed[local].is_ssa(fx, ty);
@@ -461,9 +461,9 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     };
 
     let extra_args = &args[fn_sig.inputs().skip_binder().len()..];
-    let extra_args = fx.tcx.mk_type_list_from_iter(
-        extra_args.iter().map(|op_arg| fx.monomorphize(op_arg.node.ty(fx.mir, fx.tcx))),
-    );
+    let extra_args = fx
+        .tcx
+        .mk_type_list_from_iter(extra_args.iter().map(|op_arg| op_arg.node.ty(fx.mir, fx.tcx)));
     let fn_abi = if let Some(instance) = instance {
         FullyMonomorphizedLayoutCx(fx.tcx).fn_abi_of_instance(instance, extra_args)
     } else {

compiler/rustc_codegen_cranelift/src/base.rs

@@ -8,6 +8,7 @@ use rustc_ast::InlineAsmOptions;
 use rustc_codegen_ssa::base::is_call_from_compiler_builtins_to_upstream_monomorphization;
 use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_index::IndexVec;
+use rustc_middle::mono::{InstantiationMode, MonoItem};
 use rustc_middle::ty::TypeVisitableExt;
 use rustc_middle::ty::adjustment::PointerCoercion;
 use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv as _};
@@ -44,7 +45,11 @@ pub(crate) fn codegen_fn<'tcx>(
     let symbol_name = tcx.symbol_name(instance).name.to_string();
     let _timer = tcx.prof.generic_activity_with_arg("codegen fn", &*symbol_name);
 
-    let mir = tcx.instance_mir(instance.def);
+    let mir = tcx.build_codegen_mir(instance);
+    let mir = match MonoItem::Fn(instance).instantiation_mode(tcx) {
+        InstantiationMode::LocalCopy => &*mir.borrow(),
+        InstantiationMode::GloballyShared { .. } => &*mir.steal(),
+    };
     let _mir_guard = crate::PrintOnPanic(|| {
         let mut buf = Vec::new();
         with_no_trimmed_paths!({
@@ -282,10 +287,8 @@ fn verify_func(tcx: TyCtxt<'_>, writer: &crate::pretty_clif::CommentWriter, func
 }
 
 fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
-    let arg_uninhabited = fx
-        .mir
-        .args_iter()
-        .any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).is_uninhabited());
+    let arg_uninhabited =
+        fx.mir.args_iter().any(|arg| fx.layout_of(fx.mir.local_decls[arg].ty).is_uninhabited());
     if arg_uninhabited {
         fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
         fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
@@ -297,19 +300,10 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
         .generic_activity("codegen prelude")
         .run(|| crate::abi::codegen_fn_prelude(fx, start_block));
 
-    let reachable_blocks = traversal::mono_reachable_as_bitset(fx.mir, fx.tcx, fx.instance);
-
     for (bb, bb_data) in fx.mir.basic_blocks.iter_enumerated() {
         let block = fx.get_block(bb);
         fx.bcx.switch_to_block(block);
 
-        if !reachable_blocks.contains(bb) {
-            // We want to skip this block, because it's not reachable. But we still create
-            // the block so terminators in other blocks can reference it.
-            fx.bcx.ins().trap(TrapCode::user(1 /* unreachable */).unwrap());
-            continue;
-        }
-
         if bb_data.is_cleanup {
             if cfg!(not(feature = "unwinding")) {
                 continue;
@@ -698,8 +692,8 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
                     ref operand,
                     to_ty,
                 ) => {
-                    let from_ty = fx.monomorphize(operand.ty(&fx.mir.local_decls, fx.tcx));
-                    let to_layout = fx.layout_of(fx.monomorphize(to_ty));
+                    let from_ty = operand.ty(&fx.mir.local_decls, fx.tcx);
+                    let to_layout = fx.layout_of(to_ty);
                     match *from_ty.kind() {
                         ty::FnDef(def_id, args) => {
                             let func_ref = fx.get_function_ref(
@@ -722,7 +716,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
                     ref operand,
                     to_ty,
                 ) => {
-                    let to_layout = fx.layout_of(fx.monomorphize(to_ty));
+                    let to_layout = fx.layout_of(to_ty);
                     let operand = codegen_operand(fx, operand);
                     lval.write_cvalue(fx, operand.cast_pointer_to(to_layout));
                 }
@@ -752,7 +746,6 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
                 ) => {
                     let operand = codegen_operand(fx, operand);
                     let from_ty = operand.layout().ty;
-                    let to_ty = fx.monomorphize(to_ty);
 
                     fn is_wide_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool {
                         ty.builtin_deref(true).is_some_and(|pointee_ty| {
@@ -824,8 +817,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
                 }
                 Rvalue::Repeat(ref operand, times) => {
                     let operand = codegen_operand(fx, operand);
-                    let times = fx
-                        .monomorphize(times)
+                    let times = times
                         .try_to_target_usize(fx.tcx)
                         .expect("expected monomorphic const in codegen");
                     if operand.layout().size.bytes() == 0 {
@@ -862,7 +854,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
                     if matches!(**kind, AggregateKind::RawPtr(..)) =>
                 {
                     let ty = to_place_and_rval.1.ty(&fx.mir.local_decls, fx.tcx);
-                    let layout = fx.layout_of(fx.monomorphize(ty));
+                    let layout = fx.layout_of(ty);
                     let [data, meta] = &*operands.raw else {
                         bug!("RawPtr fields: {operands:?}");
                     };
@@ -957,8 +949,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
 fn codegen_array_len<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, place: CPlace<'tcx>) -> Value {
     match *place.layout().ty.kind() {
         ty::Array(_elem_ty, len) => {
-            let len = fx
-                .monomorphize(len)
+            let len = len
                 .try_to_target_usize(fx.tcx)
                 .expect("expected monomorphic const in codegen") as i64;
             fx.bcx.ins().iconst(fx.pointer_type, len)
@@ -981,7 +972,7 @@ pub(crate) fn codegen_place<'tcx>(
             }
             PlaceElem::OpaqueCast(ty) => bug!("encountered OpaqueCast({ty}) in codegen"),
             PlaceElem::UnwrapUnsafeBinder(ty) => {
-                cplace = cplace.place_transmute_type(fx, fx.monomorphize(ty));
+                cplace = cplace.place_transmute_type(fx, ty);
             }
             PlaceElem::Field(field, _ty) => {
                 cplace = cplace.place_field(fx, field);

compiler/rustc_codegen_cranelift/src/common.rs

@@ -2,7 +2,6 @@ use cranelift_codegen::isa::TargetFrontendConfig;
 use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable};
 use rustc_abi::{Float, Integer, Primitive};
 use rustc_index::IndexVec;
-use rustc_middle::ty::TypeFoldable;
 use rustc_middle::ty::layout::{
     self, FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers,
 };
@@ -267,7 +266,7 @@ pub(crate) fn create_wrapper_function(
     module.define_function(wrapper_func_id, &mut ctx).unwrap();
 }
 
-pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
+pub(crate) struct FunctionCx<'m, 'clif, 'tcx> {
     pub(crate) module: &'m mut dyn Module,
     pub(crate) debug_context: Option<&'clif mut DebugContext>,
     pub(crate) tcx: TyCtxt<'tcx>,
@@ -279,7 +278,7 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
     pub(crate) cgu_name: Symbol,
     pub(crate) instance: Instance<'tcx>,
     pub(crate) symbol_name: String,
-    pub(crate) mir: &'tcx Body<'tcx>,
+    pub(crate) mir: &'m Body<'tcx>,
     pub(crate) fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>,
 
     pub(crate) bcx: FunctionBuilder<'clif>,
@@ -342,17 +341,6 @@ impl<'tcx> HasTargetSpec for FunctionCx<'_, '_, 'tcx> {
 }
 
 impl<'tcx> FunctionCx<'_, '_, 'tcx> {
-    pub(crate) fn monomorphize<T>(&self, value: T) -> T
-    where
-        T: TypeFoldable<TyCtxt<'tcx>> + Copy,
-    {
-        self.instance.instantiate_mir_and_normalize_erasing_regions(
-            self.tcx,
-            ty::TypingEnv::fully_monomorphized(),
-            ty::EarlyBinder::bind(value),
-        )
-    }
-
     pub(crate) fn clif_type(&self, ty: Ty<'tcx>) -> Option<Type> {
         clif_type_from_ty(self.tcx, ty)
     }

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -78,7 +78,7 @@ pub(crate) fn eval_mir_constant<'tcx>(
     fx: &FunctionCx<'_, '_, 'tcx>,
     constant: &ConstOperand<'tcx>,
 ) -> (ConstValue, Ty<'tcx>) {
-    let cv = fx.monomorphize(constant.const_);
+    let cv = constant.const_;
     // This cannot fail because we checked all required_consts in advance.
     let val = cv
         .eval(fx.tcx, ty::TypingEnv::fully_monomorphized(), constant.span)

compiler/rustc_codegen_cranelift/src/inline_asm.rs

@@ -111,7 +111,7 @@ pub(crate) fn codegen_inline_asm_terminator<'tcx>(
                         .span_err(span, "asm! and global_asm! sym operands are not yet supported");
                 }
 
-                let const_ = fx.monomorphize(value.const_);
+                let const_ = value.const_;
                 if let ty::FnDef(def_id, args) = *const_.ty().kind() {
                     let instance = ty::Instance::resolve_for_fn_ptr(
                         fx.tcx,

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -180,7 +180,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             }
 
             // Make sure this is actually a SIMD vector.
-            let idx_ty = fx.monomorphize(idx.node.ty(fx.mir, fx.tcx));
+            let idx_ty = idx.node.ty(fx.mir, fx.tcx);
             if !idx_ty.is_simd()
                 || !matches!(idx_ty.simd_size_and_type(fx.tcx).1.kind(), ty::Uint(ty::UintTy::U32))
             {

compiler/rustc_codegen_ssa/src/base.rs

@@ -401,7 +401,7 @@ pub(crate) fn codegen_instance<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>>(
     // release builds.
     info!("codegen_instance({})", instance);
 
-    mir::codegen_mir::<Bx>(cx, instance);
+    mir::lower_mir::<Bx>(cx, instance);
 }
 
 pub fn codegen_global_asm<'tcx, Cx>(cx: &mut Cx, item_id: ItemId)

compiler/rustc_codegen_ssa/src/mir/analyze.rs

@@ -15,7 +15,7 @@ use super::FunctionCx;
 use crate::traits::*;
 
 pub(crate) fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
-    fx: &FunctionCx<'a, 'tcx, Bx>,
+    fx: &FunctionCx<'_, 'a, 'tcx, Bx>,
     traversal_order: &[mir::BasicBlock],
 ) -> DenseBitSet<mir::Local> {
     let mir = fx.mir;
@@ -24,8 +24,7 @@ pub(crate) fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         .local_decls
         .iter()
         .map(|decl| {
-            let ty = fx.monomorphize(decl.ty);
-            let layout = fx.cx.spanned_layout_of(ty, decl.source_info.span);
+            let layout = fx.cx.spanned_layout_of(decl.ty, decl.source_info.span);
             if layout.is_zst() { LocalKind::ZST } else { LocalKind::Unused }
         })
         .collect();
@@ -66,13 +65,13 @@ enum LocalKind {
     SSA(DefLocation),
 }
 
-struct LocalAnalyzer<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> {
-    fx: &'a FunctionCx<'b, 'tcx, Bx>,
+struct LocalAnalyzer<'a, 'mir, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> {
+    fx: &'a FunctionCx<'mir, 'b, 'tcx, Bx>,
     dominators: &'a Dominators<mir::BasicBlock>,
     locals: IndexVec<mir::Local, LocalKind>,
 }
 
-impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'a, 'b, 'tcx, Bx> {
+impl<'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'_, '_, 'b, 'tcx, Bx> {
     fn define(&mut self, local: mir::Local, location: DefLocation) {
         let fx = self.fx;
         let kind = &mut self.locals[local];
@@ -81,8 +80,7 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'a, 'b, 'tcx, Bx>
             LocalKind::ZST => {}
             LocalKind::Memory => {}
             LocalKind::Unused => {
-                let ty = fx.monomorphize(decl.ty);
-                let layout = fx.cx.spanned_layout_of(ty, decl.source_info.span);
+                let layout = fx.cx.spanned_layout_of(decl.ty, decl.source_info.span);
                 *kind =
                     if fx.cx.is_backend_immediate(layout) || fx.cx.is_backend_scalar_pair(layout) {
                         LocalKind::SSA(location)
@@ -138,7 +136,7 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'a, 'b, 'tcx, Bx>
 
             // Scan through to ensure the only projections are those which
             // `FunctionCx::maybe_codegen_consume_direct` can handle.
-            let base_ty = self.fx.monomorphized_place_ty(mir::PlaceRef::from(place_ref.local));
+            let base_ty = self.fx.mir.local_decls[place_ref.local].ty;
             let mut layout = self.fx.cx.layout_of(base_ty);
             for elem in place_ref.projection {
                 layout = match *elem {
@@ -168,7 +166,7 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'a, 'b, 'tcx, Bx>
     }
 }
 
-impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> Visitor<'tcx> for LocalAnalyzer<'a, 'b, 'tcx, Bx> {
+impl<'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> Visitor<'tcx> for LocalAnalyzer<'_, '_, 'b, 'tcx, Bx> {
     fn visit_assign(
         &mut self,
         place: &mir::Place<'tcx>,
@@ -256,7 +254,6 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> Visitor<'tcx> for LocalAnalyzer
                 let kind = &mut self.locals[local];
                 if *kind != LocalKind::Memory {
                     let ty = self.fx.mir.local_decls[local].ty;
-                    let ty = self.fx.monomorphize(ty);
                     if self.fx.cx.type_needs_drop(ty) {
                         // Only need the place if we're actually dropping it.
                         *kind = LocalKind::Memory;