diff --git a/compiler/rustc_attr_parsing/src/attributes/prototype.rs b/compiler/rustc_attr_parsing/src/attributes/prototype.rs index 8b8eccab029dd..12d8a27fb52d6 100644 --- a/compiler/rustc_attr_parsing/src/attributes/prototype.rs +++ b/compiler/rustc_attr_parsing/src/attributes/prototype.rs @@ -93,6 +93,7 @@ fn parse_dialect( sym::analysis => MirDialect::Analysis, sym::built => MirDialect::Built, sym::runtime => MirDialect::Runtime, + sym::mono => MirDialect::Mono, _ => { cx.adcx().expected_specific_argument(span, &[sym::analysis, sym::built, sym::runtime]); @@ -136,43 +137,28 @@ fn check_custom_mir( failed: &mut bool, ) { let attr_span = cx.attr_span; + let Some((phase, phase_span)) = phase else { return }; let Some((dialect, dialect_span)) = dialect else { - if let Some((_, phase_span)) = phase { + *failed = true; + cx.emit_err(session_diagnostics::CustomMirPhaseRequiresDialect { attr_span, phase_span }); + return; + }; + + match (dialect, phase) { + (MirDialect::Built, _) + | (MirDialect::Analysis, MirPhase::Optimized) + | (MirDialect::Mono, MirPhase::PostCleanup) => { *failed = true; - cx.emit_err(session_diagnostics::CustomMirPhaseRequiresDialect { + cx.emit_err(session_diagnostics::CustomMirIncompatibleDialectAndPhase { + dialect, + phase, attr_span, + dialect_span, phase_span, }); } - return; - }; - - match dialect { - MirDialect::Analysis => { - if let Some((MirPhase::Optimized, phase_span)) = phase { - *failed = true; - cx.emit_err(session_diagnostics::CustomMirIncompatibleDialectAndPhase { - dialect, - phase: MirPhase::Optimized, - attr_span, - dialect_span, - phase_span, - }); - } - } - - MirDialect::Built => { - if let Some((phase, phase_span)) = phase { - *failed = true; - cx.emit_err(session_diagnostics::CustomMirIncompatibleDialectAndPhase { - dialect, - phase, - attr_span, - dialect_span, - phase_span, - }); - } - } - MirDialect::Runtime => {} + (MirDialect::Analysis, MirPhase::Initial | MirPhase::PostCleanup) + | (MirDialect::Runtime, MirPhase::Initial | MirPhase::PostCleanup | MirPhase::Optimized) + | (MirDialect::Mono, MirPhase::Initial | MirPhase::Optimized) => {} } } diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs index a8b179f169bf6..83e2a050990a2 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs @@ -263,7 +263,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ .mir .args_iter() .map(|local| { - let arg_ty = fx.monomorphize(fx.mir.local_decls[local].ty); + let arg_ty = fx.mir.local_decls[local].ty; // Adapted from https://github.com/rust-lang/rust/blob/145155dc96757002c7b2e9de8489416e2fdbbd57/src/librustc_codegen_llvm/mir/mod.rs#L442-L482 if Some(local) == fx.mir.spread_arg { @@ -352,7 +352,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ } for local in fx.mir.vars_and_temps_iter() { - let ty = fx.monomorphize(fx.mir.local_decls[local].ty); + let ty = fx.mir.local_decls[local].ty; let layout = fx.layout_of(ty); let is_ssa = ssa_analyzed[local].is_ssa(fx, ty); @@ -461,9 +461,9 @@ pub(crate) fn codegen_terminator_call<'tcx>( }; let extra_args = &args[fn_sig.inputs().skip_binder().len()..]; - let extra_args = fx.tcx.mk_type_list_from_iter( - extra_args.iter().map(|op_arg| fx.monomorphize(op_arg.node.ty(fx.mir, fx.tcx))), - ); + let extra_args = fx + .tcx + .mk_type_list_from_iter(extra_args.iter().map(|op_arg| op_arg.node.ty(fx.mir, fx.tcx))); let fn_abi = if let Some(instance) = instance { FullyMonomorphizedLayoutCx(fx.tcx).fn_abi_of_instance(instance, extra_args) } else { diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index 1d90c8e0dadcf..36626343eac79 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -8,6 +8,7 @@ use rustc_ast::InlineAsmOptions; use rustc_codegen_ssa::base::is_call_from_compiler_builtins_to_upstream_monomorphization; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_index::IndexVec; +use rustc_middle::mono::{InstantiationMode, MonoItem}; use rustc_middle::ty::TypeVisitableExt; use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv as _}; @@ -44,7 +45,11 @@ pub(crate) fn codegen_fn<'tcx>( let symbol_name = tcx.symbol_name(instance).name.to_string(); let _timer = tcx.prof.generic_activity_with_arg("codegen fn", &*symbol_name); - let mir = tcx.instance_mir(instance.def); + let mir = tcx.build_codegen_mir(instance); + let mir = match MonoItem::Fn(instance).instantiation_mode(tcx) { + InstantiationMode::LocalCopy => &*mir.borrow(), + InstantiationMode::GloballyShared { .. } => &*mir.steal(), + }; let _mir_guard = crate::PrintOnPanic(|| { let mut buf = Vec::new(); with_no_trimmed_paths!({ @@ -282,10 +287,8 @@ fn verify_func(tcx: TyCtxt<'_>, writer: &crate::pretty_clif::CommentWriter, func } fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { - let arg_uninhabited = fx - .mir - .args_iter() - .any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).is_uninhabited()); + let arg_uninhabited = + fx.mir.args_iter().any(|arg| fx.layout_of(fx.mir.local_decls[arg].ty).is_uninhabited()); if arg_uninhabited { fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); @@ -297,19 +300,10 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { .generic_activity("codegen prelude") .run(|| crate::abi::codegen_fn_prelude(fx, start_block)); - let reachable_blocks = traversal::mono_reachable_as_bitset(fx.mir, fx.tcx, fx.instance); - for (bb, bb_data) in fx.mir.basic_blocks.iter_enumerated() { let block = fx.get_block(bb); fx.bcx.switch_to_block(block); - if !reachable_blocks.contains(bb) { - // We want to skip this block, because it's not reachable. But we still create - // the block so terminators in other blocks can reference it. - fx.bcx.ins().trap(TrapCode::user(1 /* unreachable */).unwrap()); - continue; - } - if bb_data.is_cleanup { if cfg!(not(feature = "unwinding")) { continue; @@ -698,8 +692,8 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: ref operand, to_ty, ) => { - let from_ty = fx.monomorphize(operand.ty(&fx.mir.local_decls, fx.tcx)); - let to_layout = fx.layout_of(fx.monomorphize(to_ty)); + let from_ty = operand.ty(&fx.mir.local_decls, fx.tcx); + let to_layout = fx.layout_of(to_ty); match *from_ty.kind() { ty::FnDef(def_id, args) => { let func_ref = fx.get_function_ref( @@ -722,7 +716,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: ref operand, to_ty, ) => { - let to_layout = fx.layout_of(fx.monomorphize(to_ty)); + let to_layout = fx.layout_of(to_ty); let operand = codegen_operand(fx, operand); lval.write_cvalue(fx, operand.cast_pointer_to(to_layout)); } @@ -752,7 +746,6 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: ) => { let operand = codegen_operand(fx, operand); let from_ty = operand.layout().ty; - let to_ty = fx.monomorphize(to_ty); fn is_wide_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool { ty.builtin_deref(true).is_some_and(|pointee_ty| { @@ -824,8 +817,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: } Rvalue::Repeat(ref operand, times) => { let operand = codegen_operand(fx, operand); - let times = fx - .monomorphize(times) + let times = times .try_to_target_usize(fx.tcx) .expect("expected monomorphic const in codegen"); if operand.layout().size.bytes() == 0 { @@ -862,7 +854,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: if matches!(**kind, AggregateKind::RawPtr(..)) => { let ty = to_place_and_rval.1.ty(&fx.mir.local_decls, fx.tcx); - let layout = fx.layout_of(fx.monomorphize(ty)); + let layout = fx.layout_of(ty); let [data, meta] = &*operands.raw else { bug!("RawPtr fields: {operands:?}"); }; @@ -957,8 +949,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: fn codegen_array_len<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, place: CPlace<'tcx>) -> Value { match *place.layout().ty.kind() { ty::Array(_elem_ty, len) => { - let len = fx - .monomorphize(len) + let len = len .try_to_target_usize(fx.tcx) .expect("expected monomorphic const in codegen") as i64; fx.bcx.ins().iconst(fx.pointer_type, len) @@ -981,7 +972,7 @@ pub(crate) fn codegen_place<'tcx>( } PlaceElem::OpaqueCast(ty) => bug!("encountered OpaqueCast({ty}) in codegen"), PlaceElem::UnwrapUnsafeBinder(ty) => { - cplace = cplace.place_transmute_type(fx, fx.monomorphize(ty)); + cplace = cplace.place_transmute_type(fx, ty); } PlaceElem::Field(field, _ty) => { cplace = cplace.place_field(fx, field); diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs index b22afca847aa9..78d298af514d8 100644 --- a/compiler/rustc_codegen_cranelift/src/common.rs +++ b/compiler/rustc_codegen_cranelift/src/common.rs @@ -2,7 +2,6 @@ use cranelift_codegen::isa::TargetFrontendConfig; use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable}; use rustc_abi::{Float, Integer, Primitive}; use rustc_index::IndexVec; -use rustc_middle::ty::TypeFoldable; use rustc_middle::ty::layout::{ self, FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, }; @@ -267,7 +266,7 @@ pub(crate) fn create_wrapper_function( module.define_function(wrapper_func_id, &mut ctx).unwrap(); } -pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> { +pub(crate) struct FunctionCx<'m, 'clif, 'tcx> { pub(crate) module: &'m mut dyn Module, pub(crate) debug_context: Option<&'clif mut DebugContext>, pub(crate) tcx: TyCtxt<'tcx>, @@ -279,7 +278,7 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> { pub(crate) cgu_name: Symbol, pub(crate) instance: Instance<'tcx>, pub(crate) symbol_name: String, - pub(crate) mir: &'tcx Body<'tcx>, + pub(crate) mir: &'m Body<'tcx>, pub(crate) fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, pub(crate) bcx: FunctionBuilder<'clif>, @@ -342,17 +341,6 @@ impl<'tcx> HasTargetSpec for FunctionCx<'_, '_, 'tcx> { } impl<'tcx> FunctionCx<'_, '_, 'tcx> { - pub(crate) fn monomorphize(&self, value: T) -> T - where - T: TypeFoldable> + Copy, - { - self.instance.instantiate_mir_and_normalize_erasing_regions( - self.tcx, - ty::TypingEnv::fully_monomorphized(), - ty::EarlyBinder::bind(value), - ) - } - pub(crate) fn clif_type(&self, ty: Ty<'tcx>) -> Option { clif_type_from_ty(self.tcx, ty) } diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index f85d21db11fb2..c9e0949d84584 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -78,7 +78,7 @@ pub(crate) fn eval_mir_constant<'tcx>( fx: &FunctionCx<'_, '_, 'tcx>, constant: &ConstOperand<'tcx>, ) -> (ConstValue, Ty<'tcx>) { - let cv = fx.monomorphize(constant.const_); + let cv = constant.const_; // This cannot fail because we checked all required_consts in advance. let val = cv .eval(fx.tcx, ty::TypingEnv::fully_monomorphized(), constant.span) diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs index 8100d565b3974..b19c832c49349 100644 --- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs +++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs @@ -111,7 +111,7 @@ pub(crate) fn codegen_inline_asm_terminator<'tcx>( .span_err(span, "asm! and global_asm! sym operands are not yet supported"); } - let const_ = fx.monomorphize(value.const_); + let const_ = value.const_; if let ty::FnDef(def_id, args) = *const_.ty().kind() { let instance = ty::Instance::resolve_for_fn_ptr( fx.tcx, diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs index b8e1886b2d3c1..3696118b4051f 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs @@ -180,7 +180,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( } // Make sure this is actually a SIMD vector. - let idx_ty = fx.monomorphize(idx.node.ty(fx.mir, fx.tcx)); + let idx_ty = idx.node.ty(fx.mir, fx.tcx); if !idx_ty.is_simd() || !matches!(idx_ty.simd_size_and_type(fx.tcx).1.kind(), ty::Uint(ty::UintTy::U32)) { diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index b0ce3833446d3..3243446c24470 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -401,7 +401,7 @@ pub(crate) fn codegen_instance<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>>( // release builds. info!("codegen_instance({})", instance); - mir::codegen_mir::(cx, instance); + mir::lower_mir::(cx, instance); } pub fn codegen_global_asm<'tcx, Cx>(cx: &mut Cx, item_id: ItemId) diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs index fb5734ba087c6..8b40a59dab5b7 100644 --- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs +++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs @@ -15,7 +15,7 @@ use super::FunctionCx; use crate::traits::*; pub(crate) fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - fx: &FunctionCx<'a, 'tcx, Bx>, + fx: &FunctionCx<'_, 'a, 'tcx, Bx>, traversal_order: &[mir::BasicBlock], ) -> DenseBitSet { let mir = fx.mir; @@ -24,8 +24,7 @@ pub(crate) fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( .local_decls .iter() .map(|decl| { - let ty = fx.monomorphize(decl.ty); - let layout = fx.cx.spanned_layout_of(ty, decl.source_info.span); + let layout = fx.cx.spanned_layout_of(decl.ty, decl.source_info.span); if layout.is_zst() { LocalKind::ZST } else { LocalKind::Unused } }) .collect(); @@ -66,13 +65,13 @@ enum LocalKind { SSA(DefLocation), } -struct LocalAnalyzer<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> { - fx: &'a FunctionCx<'b, 'tcx, Bx>, +struct LocalAnalyzer<'a, 'mir, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> { + fx: &'a FunctionCx<'mir, 'b, 'tcx, Bx>, dominators: &'a Dominators, locals: IndexVec, } -impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'a, 'b, 'tcx, Bx> { +impl<'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'_, '_, 'b, 'tcx, Bx> { fn define(&mut self, local: mir::Local, location: DefLocation) { let fx = self.fx; let kind = &mut self.locals[local]; @@ -81,8 +80,7 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'a, 'b, 'tcx, Bx> LocalKind::ZST => {} LocalKind::Memory => {} LocalKind::Unused => { - let ty = fx.monomorphize(decl.ty); - let layout = fx.cx.spanned_layout_of(ty, decl.source_info.span); + let layout = fx.cx.spanned_layout_of(decl.ty, decl.source_info.span); *kind = if fx.cx.is_backend_immediate(layout) || fx.cx.is_backend_scalar_pair(layout) { LocalKind::SSA(location) @@ -138,7 +136,7 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'a, 'b, 'tcx, Bx> // Scan through to ensure the only projections are those which // `FunctionCx::maybe_codegen_consume_direct` can handle. - let base_ty = self.fx.monomorphized_place_ty(mir::PlaceRef::from(place_ref.local)); + let base_ty = self.fx.mir.local_decls[place_ref.local].ty; let mut layout = self.fx.cx.layout_of(base_ty); for elem in place_ref.projection { layout = match *elem { @@ -168,7 +166,7 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'a, 'b, 'tcx, Bx> } } -impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> Visitor<'tcx> for LocalAnalyzer<'a, 'b, 'tcx, Bx> { +impl<'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> Visitor<'tcx> for LocalAnalyzer<'_, '_, 'b, 'tcx, Bx> { fn visit_assign( &mut self, place: &mir::Place<'tcx>, @@ -256,7 +254,6 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> Visitor<'tcx> for LocalAnalyzer let kind = &mut self.locals[local]; if *kind != LocalKind::Memory { let ty = self.fx.mir.local_decls[local].ty; - let ty = self.fx.monomorphize(ty); if self.fx.cx.type_needs_drop(ty) { // Only need the place if we're actually dropping it. *kind = LocalKind::Memory; diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index b6b95c5f12aae..36ef610531e52 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -46,17 +46,17 @@ enum CallKind { /// Used by `FunctionCx::codegen_terminator` for emitting common patterns /// e.g., creating a basic block, calling a function, etc. -struct TerminatorCodegenHelper<'tcx> { +struct TerminatorCodegenHelper<'mir, 'tcx> { bb: mir::BasicBlock, - terminator: &'tcx mir::Terminator<'tcx>, + terminator: &'mir mir::Terminator<'tcx>, } -impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { +impl<'a, 'tcx> TerminatorCodegenHelper<'_, 'tcx> { /// Returns the appropriate `Funclet` for the current funclet, if on MSVC, /// either already previously cached, or newly created, by `landing_pad_for`. fn funclet<'b, Bx: BuilderMethods<'a, 'tcx>>( &self, - fx: &'b mut FunctionCx<'a, 'tcx, Bx>, + fx: &'b mut FunctionCx<'_, 'a, 'tcx, Bx>, ) -> Option<&'b Bx::Funclet> { let cleanup_kinds = fx.cleanup_kinds.as_ref()?; let funclet_bb = cleanup_kinds[self.bb].funclet_bb(self.bb)?; @@ -82,7 +82,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { /// stuff in it or next to it. fn llbb_with_cleanup>( &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'_, 'a, 'tcx, Bx>, target: mir::BasicBlock, ) -> Bx::BasicBlock { let (needs_landing_pad, is_cleanupret) = self.llbb_characteristics(fx, target); @@ -106,7 +106,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { fn llbb_characteristics>( &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'_, 'a, 'tcx, Bx>, target: mir::BasicBlock, ) -> (bool, bool) { if let Some(ref cleanup_kinds) = fx.cleanup_kinds { @@ -131,7 +131,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { fn funclet_br>( &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'_, 'a, 'tcx, Bx>, bx: &mut Bx, target: mir::BasicBlock, mergeable_succ: bool, @@ -160,7 +160,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { /// return destination `destination` and the unwind action `unwind`. fn do_call>( &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'_, 'a, 'tcx, Bx>, bx: &mut Bx, fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, fn_ptr: Bx::Value, @@ -296,7 +296,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { /// Generates inline assembly with optional `destination` and `unwind`. fn do_inlineasm>( &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'_, 'a, 'tcx, Bx>, bx: &mut Bx, template: &[InlineAsmTemplatePiece], operands: &[InlineAsmOperandRef<'tcx, Bx>], @@ -363,9 +363,13 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { } /// Codegen implementations for some terminator variants. -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'_, 'a, 'tcx, Bx> { /// Generates code for a `Resume` terminator. - fn codegen_resume_terminator(&mut self, helper: TerminatorCodegenHelper<'tcx>, bx: &mut Bx) { + fn codegen_resume_terminator( + &mut self, + helper: TerminatorCodegenHelper<'_, 'tcx>, + bx: &mut Bx, + ) { if let Some(funclet) = helper.funclet(self) { bx.cleanup_ret(funclet, None); } else { @@ -382,7 +386,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn codegen_switchint_terminator( &mut self, - helper: TerminatorCodegenHelper<'tcx>, + helper: TerminatorCodegenHelper<'_, 'tcx>, bx: &mut Bx, discr: &mir::Operand<'tcx>, targets: &SwitchTargets, @@ -594,7 +598,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { #[tracing::instrument(level = "trace", skip(self, helper, bx))] fn codegen_drop_terminator( &mut self, - helper: TerminatorCodegenHelper<'tcx>, + helper: TerminatorCodegenHelper<'_, 'tcx>, bx: &mut Bx, source_info: &mir::SourceInfo, location: mir::Place<'tcx>, @@ -603,7 +607,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mergeable_succ: bool, ) -> MergingSucc { let ty = location.ty(self.mir, bx.tcx()).ty; - let ty = self.monomorphize(ty); let drop_fn = Instance::resolve_drop_glue(bx.tcx(), ty); if let ty::InstanceKind::DropGlue(_, None) = drop_fn.def { @@ -693,7 +696,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn codegen_assert_terminator( &mut self, - helper: TerminatorCodegenHelper<'tcx>, + helper: TerminatorCodegenHelper<'_, 'tcx>, bx: &mut Bx, terminator: &mir::Terminator<'tcx>, cond: &mir::Operand<'tcx>, @@ -794,7 +797,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn codegen_terminate_terminator( &mut self, - helper: TerminatorCodegenHelper<'tcx>, + helper: TerminatorCodegenHelper<'_, 'tcx>, bx: &mut Bx, terminator: &mir::Terminator<'tcx>, reason: UnwindTerminateReason, @@ -825,7 +828,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { /// Returns `Some` if this is indeed a panic intrinsic and codegen is done. fn codegen_panic_intrinsic( &mut self, - helper: &TerminatorCodegenHelper<'tcx>, + helper: &TerminatorCodegenHelper<'_, 'tcx>, bx: &mut Bx, intrinsic: ty::IntrinsicDef, instance: Instance<'tcx>, @@ -892,7 +895,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn codegen_call_terminator( &mut self, - helper: TerminatorCodegenHelper<'tcx>, + helper: TerminatorCodegenHelper<'_, 'tcx>, bx: &mut Bx, terminator: &mir::Terminator<'tcx>, func: &mir::Operand<'tcx>, @@ -943,7 +946,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } let result_layout = - self.cx.layout_of(self.monomorphized_place_ty(destination.as_ref())); + self.cx.layout_of(destination.ty(self.mir, self.cx.tcx()).ty); let (result_place, store_in_local) = if let Some(local) = destination.as_local() { @@ -1088,8 +1091,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { assert!(!instance.args.has_infer()); assert!(!instance.args.has_escaping_bound_vars()); - let result_layout = - self.cx.layout_of(self.monomorphized_place_ty(destination.as_ref())); + let result_layout = self.cx.layout_of(destination.ty(self.mir, self.cx.tcx()).ty); let return_dest = if result_layout.is_zst() { ReturnDest::Nothing @@ -1136,10 +1138,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let sig = callee.layout.ty.fn_sig(bx.tcx()); let extra_args = &args[sig.inputs().skip_binder().len()..]; - let extra_args = bx.tcx().mk_type_list_from_iter(extra_args.iter().map(|op_arg| { - let op_ty = op_arg.node.ty(self.mir, bx.tcx()); - self.monomorphize(op_ty) - })); + let extra_args = bx.tcx().mk_type_list_from_iter( + extra_args.iter().map(|op_arg| op_arg.node.ty(self.mir, bx.tcx())), + ); let fn_abi = match instance { Some(instance) => bx.fn_abi_of_instance(instance, extra_args), @@ -1396,7 +1397,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn codegen_asm_terminator( &mut self, - helper: TerminatorCodegenHelper<'tcx>, + helper: TerminatorCodegenHelper<'_, 'tcx>, bx: &mut Bx, asm_macro: InlineAsmMacro, terminator: &mir::Terminator<'tcx>, @@ -1439,7 +1440,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { InlineAsmOperandRef::Const { string } } mir::InlineAsmOperand::SymFn { ref value } => { - let const_ = self.monomorphize(value.const_); + let const_ = value.const_; if let ty::FnDef(def_id, args) = *const_.ty().kind() { let instance = ty::Instance::resolve_for_fn_ptr( bx.tcx(), @@ -1517,21 +1518,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } - pub(crate) fn codegen_block_as_unreachable(&mut self, bb: mir::BasicBlock) { - let llbb = match self.try_llbb(bb) { - Some(llbb) => llbb, - None => return, - }; - let bx = &mut Bx::build(self.cx, llbb); - debug!("codegen_block_as_unreachable({:?})", bb); - bx.unreachable(); - } - fn codegen_terminator( &mut self, bx: &mut Bx, bb: mir::BasicBlock, - terminator: &'tcx mir::Terminator<'tcx>, + terminator: &mir::Terminator<'tcx>, ) -> MergingSucc { debug!("codegen_terminator: {:?}", terminator); diff --git a/compiler/rustc_codegen_ssa/src/mir/constant.rs b/compiler/rustc_codegen_ssa/src/mir/constant.rs index abdac4c7c3721..693707fba4c1a 100644 --- a/compiler/rustc_codegen_ssa/src/mir/constant.rs +++ b/compiler/rustc_codegen_ssa/src/mir/constant.rs @@ -9,21 +9,22 @@ use crate::errors; use crate::mir::operand::OperandRef; use crate::traits::*; -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'_, 'a, 'tcx, Bx> { pub(crate) fn eval_mir_constant_to_operand( &self, bx: &mut Bx, constant: &mir::ConstOperand<'tcx>, ) -> OperandRef<'tcx, Bx::Value> { let val = self.eval_mir_constant(constant); - let ty = self.monomorphize(constant.ty()); + let ty = constant.ty(); OperandRef::from_const(bx, val, ty) } pub fn eval_mir_constant(&self, constant: &mir::ConstOperand<'tcx>) -> mir::ConstValue { // `MirUsedCollector` visited all required_consts before codegen began, so if we got here // there can be no more constants that fail to evaluate. - self.monomorphize(constant.const_) + constant + .const_ .eval(self.cx.tcx(), self.cx.typing_env(), constant.span) .expect("erroneous constant missed by mono item collection") } @@ -38,7 +39,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { &self, constant: &mir::ConstOperand<'tcx>, ) -> Result, Ty<'tcx>>, ErrorHandled> { - let uv = match self.monomorphize(constant.const_) { + let uv = match constant.const_ { mir::Const::Unevaluated(uv, _) => uv.shrink(), mir::Const::Ty(_, c) => match c.kind() { // A constant that came from a const generic but was then used as an argument to @@ -56,7 +57,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // real const generic, and get rid of this entire function. other => span_bug!(constant.span, "{other:#?}"), }; - let uv = self.monomorphize(uv); self.cx.tcx().const_eval_resolve_for_typeck(self.cx.typing_env(), uv, constant.span) } @@ -66,7 +66,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx: &Bx, constant: &mir::ConstOperand<'tcx>, ) -> (Bx::Value, Ty<'tcx>) { - let ty = self.monomorphize(constant.ty()); + let ty = constant.ty(); assert!(ty.is_simd()); let field_ty = ty.simd_size_and_type(bx.tcx()).1; diff --git a/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs b/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs index 33a050ef2804f..b2102fd54a666 100644 --- a/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs +++ b/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs @@ -4,11 +4,11 @@ use rustc_middle::mir::coverage::CoverageKind; use super::FunctionCx; use crate::traits::*; -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'_, 'a, 'tcx, Bx> { pub(crate) fn codegen_coverage(&self, bx: &mut Bx, kind: &CoverageKind, scope: SourceScope) { // Determine the instance that coverage data was originally generated for. let instance = if let Some(inlined) = scope.inlined_instance(&self.mir.source_scopes) { - self.monomorphize(inlined) + inlined } else { self.instance }; diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs index 60ab13dbc6f76..9d92c0dc425c4 100644 --- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs +++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs @@ -6,7 +6,7 @@ use rustc_abi::{BackendRepr, FieldIdx, FieldsShape, Size, VariantIdx}; use rustc_data_structures::fx::FxHashMap; use rustc_index::IndexVec; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; +use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout}; use rustc_middle::ty::{Instance, Ty}; use rustc_middle::{bug, mir, ty}; use rustc_session::config::DebugInfo; @@ -213,7 +213,7 @@ fn calculate_debuginfo_offset< DebugInfoOffset { direct_offset, indirect_offsets, result: place } } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'_, 'a, 'tcx, Bx> { pub fn set_debug_loc(&self, bx: &mut Bx, source_info: mir::SourceInfo) { bx.set_span(source_info.span); if let Some(dbg_loc) = self.dbg_loc(source_info) { @@ -293,7 +293,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } pub(crate) fn debug_poison_to_local(&self, bx: &mut Bx, local: mir::Local) { - let ty = self.monomorphize(self.mir.local_decls[local].ty); + let ty = self.mir.local_decls[local].ty; let layout = bx.cx().layout_of(ty); let to_backend_ty = bx.cx().immediate_backend_type(layout); let place_ref = PlaceRef::new_sized(bx.cx().const_poison(to_backend_ty), layout); @@ -337,10 +337,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { |(dbg_scope, _, span)| { // FIXME(eddyb) is this `+ 1` needed at all? let kind = VariableKind::ArgumentVariable(arg_index + 1); - - let arg_ty = self.monomorphize(decl.ty); - - self.cx.create_dbg_var(name, arg_ty, dbg_scope, kind, span) + self.cx.create_dbg_var(name, decl.ty, dbg_scope, kind, span) }, ) } else { @@ -572,13 +569,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }; let var_ty = if let Some(ref fragment) = var.composite { - self.monomorphize(fragment.ty) + fragment.ty } else { match var.value { - mir::VarDebugInfoContents::Place(place) => { - self.monomorphized_place_ty(place.as_ref()) - } - mir::VarDebugInfoContents::Const(c) => self.monomorphize(c.ty()), + mir::VarDebugInfoContents::Place(place) => place.ty(self.mir, self.cx.tcx()).ty, + mir::VarDebugInfoContents::Const(c) => c.ty(), } }; diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index ac6c6a0a52efa..c80875075dd46 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -52,7 +52,7 @@ fn memset_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( bx.memset(dst, val, size, align, flags); } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'_, 'a, 'tcx, Bx> { /// In the `Fallback` case, returns the instance that should be called instead. pub fn codegen_intrinsic_call( &mut self, diff --git a/compiler/rustc_codegen_ssa/src/mir/locals.rs b/compiler/rustc_codegen_ssa/src/mir/locals.rs index 5d586d97641d2..cca5cf265fe88 100644 --- a/compiler/rustc_codegen_ssa/src/mir/locals.rs +++ b/compiler/rustc_codegen_ssa/src/mir/locals.rs @@ -37,7 +37,7 @@ impl<'tcx, V> Locals<'tcx, V> { } } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'_, 'a, 'tcx, Bx> { pub(super) fn initialize_locals(&mut self, values: Vec>) { assert!(self.locals.values.is_empty()); self.locals.values = IndexVec::from_raw(values); @@ -46,7 +46,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { match value { LocalRef::Place(_) | LocalRef::UnsizedPlace(_) | LocalRef::PendingOperand => (), LocalRef::Operand(op) => { - let expected_ty = self.monomorphize(self.mir.local_decls[local].ty); + let expected_ty = self.mir.local_decls[local].ty; if expected_ty != op.layout.ty { warn!( "Unexpected initial operand type:\nexpected {expected_ty:?},\nfound {:?}.\n\ @@ -67,7 +67,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { match value { LocalRef::Place(_) | LocalRef::UnsizedPlace(_) | LocalRef::PendingOperand => (), LocalRef::Operand(ref mut op) => { - let local_ty = self.monomorphize(self.mir.local_decls[local].ty); + let local_ty = self.mir.local_decls[local].ty; if local_ty != op.layout.ty { // FIXME(#112651): This can be changed to an ICE afterwards. debug!("updating type of operand due to subtyping"); diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index 4bcf037ecce07..dbda6599f5efd 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -3,9 +3,10 @@ use std::iter; use rustc_index::IndexVec; use rustc_index::bit_set::DenseBitSet; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::mir::{Body, Local, UnwindTerminateReason, traversal}; -use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, TyAndLayout}; -use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitableExt}; +use rustc_middle::mir::{Local, UnwindTerminateReason, traversal}; +use rustc_middle::mono::{InstantiationMode, MonoItem}; +use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, TyAndLayout}; +use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeVisitableExt}; use rustc_middle::{bug, mir, span_bug}; use rustc_span::ErrorGuaranteed; use rustc_target::callconv::{FnAbi, PassMode}; @@ -48,10 +49,10 @@ type PerLocalVarDebugInfoIndexVec<'tcx, V> = IndexVec>>; /// Master context for codegenning from MIR. -pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { +pub struct FunctionCx<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { instance: Instance<'tcx>, - mir: &'tcx mir::Body<'tcx>, + mir: &'mir mir::Body<'tcx>, debug_context: Option>, @@ -126,20 +127,6 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { caller_location: Option>, } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { - pub fn monomorphize(&self, value: T) -> T - where - T: Copy + TypeFoldable>, - { - debug!("monomorphize: self.instance={:?}", self.instance); - self.instance.instantiate_mir_and_normalize_erasing_regions( - self.cx.tcx(), - self.cx.typing_env(), - ty::EarlyBinder::bind(value), - ) - } -} - enum LocalRef<'tcx, V> { Place(PlaceRef<'tcx, V>), /// `UnsizedPlace(p)`: `p` itself is a thin pointer (indirect place). @@ -197,7 +184,7 @@ impl<'tcx, V: CodegenObject> LocalRef<'tcx, V> { /////////////////////////////////////////////////////////////////////////// #[instrument(level = "debug", skip(cx))] -pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( +pub fn lower_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( cx: &'a Bx::CodegenCx, instance: Instance<'tcx>, ) { @@ -206,7 +193,13 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let tcx = cx.tcx(); let llfn = cx.get_fn(instance); - let mut mir = tcx.instance_mir(instance.def); + tcx.ensure_done().check_mono_item(instance); + let mir = tcx.build_codegen_mir(instance); + let mir = match MonoItem::Fn(instance).instantiation_mode(tcx) { + InstantiationMode::LocalCopy => &*mir.borrow(), + InstantiationMode::GloballyShared { .. } => &*mir.steal(), + }; + // Note that the ABI logic has deduced facts about the functions' parameters based on the MIR we // got here (`deduce_param_attrs`). That means we can *not* apply arbitrary further MIR // transforms as that may invalidate those deduced facts! @@ -214,15 +207,6 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty()); debug!("fn_abi: {:?}", fn_abi); - if tcx.features().ergonomic_clones() { - let monomorphized_mir = instance.instantiate_mir_and_normalize_erasing_regions( - tcx, - ty::TypingEnv::fully_monomorphized(), - ty::EarlyBinder::bind(mir.clone()), - ); - mir = tcx.arena.alloc(optimize_use_clone::(cx, monomorphized_mir)); - } - let debug_context = cx.create_function_debug_context(instance, fn_abi, llfn, &mir); let start_llbb = Bx::append_block(cx, llfn, "start"); @@ -274,7 +258,8 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( fx.compute_per_local_var_debug_info(&mut start_bx).unzip(); fx.per_local_var_debug_info = per_local_var_debug_info; - let traversal_order = traversal::mono_reachable_reverse_postorder(mir, tcx, instance); + let traversal_order: Vec<_> = + traversal::reverse_postorder(mir).map(|(block, _data)| block).collect(); let memory_locals = analyze::non_ssa_locals(&fx, &traversal_order); // Allocate variable and temp allocas @@ -283,7 +268,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let mut allocate_local = |local: Local| { let decl = &mir.local_decls[local]; - let layout = start_bx.layout_of(fx.monomorphize(decl.ty)); + let layout = start_bx.layout_of(decl.ty); assert!(!layout.ty.has_erasable_regions()); if local == mir::RETURN_PLACE { @@ -330,80 +315,10 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( // So drop the builder of `start_llbb` to avoid having two at the same time. drop(start_bx); - let mut unreached_blocks = DenseBitSet::new_filled(mir.basic_blocks.len()); // Codegen the body of each reachable block using our reverse postorder list. for bb in traversal_order { fx.codegen_block(bb); - unreached_blocks.remove(bb); } - - // FIXME: These empty unreachable blocks are *mostly* a waste. They are occasionally - // targets for a SwitchInt terminator, but the reimplementation of the mono-reachable - // simplification in SwitchInt lowering sometimes misses cases that - // mono_reachable_reverse_postorder manages to figure out. - // The solution is to do something like post-mono GVN. But for now we have this hack. - for bb in unreached_blocks.iter() { - fx.codegen_block_as_unreachable(bb); - } -} - -/// Replace `clone` calls that come from `use` statements with direct copies if possible. -// FIXME: Move this function to mir::transform when post-mono MIR passes land. -fn optimize_use_clone<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - cx: &'a Bx::CodegenCx, - mut mir: Body<'tcx>, -) -> Body<'tcx> { - let tcx = cx.tcx(); - - if tcx.features().ergonomic_clones() { - for bb in mir.basic_blocks.as_mut() { - let mir::TerminatorKind::Call { - args, - destination, - target, - call_source: mir::CallSource::Use, - .. - } = &bb.terminator().kind - else { - continue; - }; - - // CallSource::Use calls always use 1 argument. - assert_eq!(args.len(), 1); - let arg = &args[0]; - - // These types are easily available from locals, so check that before - // doing DefId lookups to figure out what we're actually calling. - let arg_ty = arg.node.ty(&mir.local_decls, tcx); - - let ty::Ref(_region, inner_ty, mir::Mutability::Not) = *arg_ty.kind() else { continue }; - - if !tcx.type_is_copy_modulo_regions(cx.typing_env(), inner_ty) { - continue; - } - - let Some(arg_place) = arg.node.place() else { continue }; - - let destination_block = target.unwrap(); - - bb.statements.push(mir::Statement::new( - bb.terminator().source_info, - mir::StatementKind::Assign(Box::new(( - *destination, - mir::Rvalue::Use( - mir::Operand::Copy( - arg_place.project_deeper(&[mir::ProjectionElem::Deref], tcx), - ), - mir::WithRetag::Yes, - ), - ))), - )); - - bb.terminator_mut().kind = mir::TerminatorKind::Goto { target: destination_block }; - } - } - - mir } /// Produces, for each argument, a `Value` pointing at the @@ -411,7 +326,7 @@ fn optimize_use_clone<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( /// indirect. fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( bx: &mut Bx, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'_, 'a, 'tcx, Bx>, memory_locals: &DenseBitSet, ) -> Vec> { let mir = fx.mir; @@ -430,7 +345,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( .enumerate() .map(|(arg_index, local)| { let arg_decl = &mir.local_decls[local]; - let arg_ty = fx.monomorphize(arg_decl.ty); + let arg_ty = arg_decl.ty; if Some(local) == mir.spread_arg { // This argument (e.g., the last argument in the "rust-call" ABI) diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 83fce5a5c8deb..24a349b8adb87 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -8,7 +8,7 @@ use rustc_abi::{ use rustc_hir::LangItem; use rustc_middle::mir::interpret::{Pointer, Scalar, alloc_range}; use rustc_middle::mir::{self, ConstValue}; -use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; +use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, Ty}; use rustc_middle::{bug, span_bug}; use rustc_session::config::{AnnotateMoves, DebugInfo, OptLevel}; @@ -344,7 +344,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { pub(crate) fn extract_field>( &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'_, 'a, 'tcx, Bx>, bx: &mut Bx, i: usize, ) -> Self { @@ -416,7 +416,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { #[instrument(level = "trace", skip(fx, bx))] pub fn codegen_get_discr>( self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'_, 'a, 'tcx, Bx>, bx: &mut Bx, cast_to: Ty<'tcx>, ) -> V { @@ -961,7 +961,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue { } } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'_, 'a, 'tcx, Bx> { fn maybe_codegen_consume_direct( &mut self, bx: &mut Bx, @@ -1015,7 +1015,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { ) -> OperandRef<'tcx, Bx::Value> { debug!("codegen_consume(place_ref={:?})", place_ref); - let ty = self.monomorphized_place_ty(place_ref); + let ty = place_ref.ty(self.mir, self.cx.tcx()).ty; let layout = bx.cx().layout_of(ty); // ZSTs don't require any actual memory access. @@ -1066,7 +1066,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } mir::Operand::Constant(ref constant) => { - let constant_ty = self.monomorphize(constant.ty()); + let constant_ty = constant.ty(); // Most SIMD vector constants should be passed as immediates. // (In particular, some intrinsics really rely on this.) if constant_ty.is_simd() { @@ -1124,7 +1124,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { AnnotateMoves::Enabled(Some(limit)) => limit, }; - let ty = self.monomorphized_place_ty(place); + let ty = place.ty(self.mir, self.cx.tcx()).ty; let layout = bx.cx().layout_of(ty); let ty_size = layout.size.bytes(); diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs index 53518fd816f31..eb1a97c6dc7b8 100644 --- a/compiler/rustc_codegen_ssa/src/mir/place.rs +++ b/compiler/rustc_codegen_ssa/src/mir/place.rs @@ -316,7 +316,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { } } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'_, 'a, 'tcx, Bx> { #[instrument(level = "trace", skip(self, bx))] pub fn codegen_place( &mut self, @@ -360,9 +360,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::ProjectionElem::OpaqueCast(ty) => { bug!("encountered OpaqueCast({ty}) in codegen") } - mir::ProjectionElem::UnwrapUnsafeBinder(ty) => { - cg_base.project_type(bx, self.monomorphize(ty)) - } + mir::ProjectionElem::UnwrapUnsafeBinder(ty) => cg_base.project_type(bx, ty), mir::ProjectionElem::Index(index) => { let index = &mir::Operand::Copy(mir::Place::from(index)); let index = self.codegen_operand(bx, index); @@ -383,7 +381,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let mut subslice = cg_base.project_index(bx, bx.cx().const_usize(from)); let projected_ty = PlaceTy::from_ty(cg_base.layout.ty).projection_ty(tcx, *elem).ty; - subslice.layout = bx.cx().layout_of(self.monomorphize(projected_ty)); + subslice.layout = bx.cx().layout_of(projected_ty); if subslice.layout.is_unsized() { assert!(from_end, "slice subslices should be `from_end`"); @@ -400,12 +398,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { debug!("codegen_place(place={:?}) => {:?}", place_ref, cg_base); cg_base } - - pub fn monomorphized_place_ty(&self, place_ref: mir::PlaceRef<'tcx>) -> Ty<'tcx> { - let tcx = self.cx.tcx(); - let place_ty = place_ref.ty(self.mir, tcx); - self.monomorphize(place_ty.ty) - } } fn round_up_const_value_to_alignment<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 90ac8c89ba9ad..da9aae1b42029 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -14,7 +14,7 @@ use crate::common::{IntPredicate, TypeKind}; use crate::traits::*; use crate::{MemFlags, base}; -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'_, 'a, 'tcx, Bx> { #[instrument(level = "trace", skip(self, bx))] pub(crate) fn codegen_rvalue( &mut self, @@ -155,8 +155,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { return; } - let count = self - .monomorphize(count) + let count = count .try_to_target_usize(bx.tcx()) .expect("expected monomorphic const in codegen"); @@ -403,7 +402,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::Rvalue::Cast(ref kind, ref source, mir_cast_ty) => { let operand = self.codegen_operand(bx, source); debug!("cast operand is {:?}", operand); - let cast = bx.cx().layout_of(self.monomorphize(mir_cast_ty)); + let cast = bx.cx().layout_of(mir_cast_ty); let val = match *kind { mir::CastKind::PointerExposeProvenance => { @@ -633,7 +632,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::Rvalue::Discriminant(ref place) => { let discr_ty = rvalue.ty(self.mir, bx.tcx()); - let discr_ty = self.monomorphize(discr_ty); let operand = self.codegen_consume(bx, place.as_ref()); let discr = operand.codegen_get_discr(self, bx, discr_ty); OperandRef { @@ -683,7 +681,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // `Memory`, and thus ends up handled in `codegen_rvalue` instead. let operand = self.codegen_operand(bx, elem); let array_ty = Ty::new_array_with_const_len(bx.tcx(), operand.layout.ty, len_const); - let array_ty = self.monomorphize(array_ty); let array_layout = bx.layout_of(array_ty); assert!(array_layout.is_zst()); OperandRef { @@ -702,7 +699,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }; let ty = rvalue.ty(self.mir, self.cx.tcx()); - let ty = self.monomorphize(ty); let layout = self.cx.layout_of(ty); let mut builder = OperandRefBuilder::new(layout); @@ -733,7 +729,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::Rvalue::WrapUnsafeBinder(ref operand, binder_ty) => { let operand = self.codegen_operand(bx, operand); - let binder_ty = self.monomorphize(binder_ty); let layout = bx.cx().layout_of(binder_ty); OperandRef { val: operand.val, layout, move_annotation: None } } diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs index afbef86bd089e..528f2265d8d83 100644 --- a/compiler/rustc_codegen_ssa/src/mir/statement.rs +++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs @@ -5,7 +5,7 @@ use tracing::instrument; use super::{FunctionCx, LocalRef}; use crate::traits::*; -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'_, 'a, 'tcx, Bx> { #[instrument(level = "debug", skip(self, bx))] pub(crate) fn codegen_statement(&mut self, bx: &mut Bx, statement: &mir::Statement<'tcx>) { self.codegen_stmt_debuginfos(bx, &statement.debuginfos); diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs index 4ff56a640c19e..b27ad0af36d36 100644 --- a/compiler/rustc_hir/src/attrs/data_structures.rs +++ b/compiler/rustc_hir/src/attrs/data_structures.rs @@ -303,6 +303,7 @@ pub enum MirDialect { Analysis, Built, Runtime, + Mono, } impl IntoDiagArg for MirDialect { @@ -311,6 +312,7 @@ impl IntoDiagArg for MirDialect { MirDialect::Analysis => "analysis", MirDialect::Built => "built", MirDialect::Runtime => "runtime", + MirDialect::Mono => "mono", }; DiagArgValue::Str(Cow::Borrowed(arg)) } diff --git a/compiler/rustc_middle/src/mir/basic_blocks.rs b/compiler/rustc_middle/src/mir/basic_blocks.rs index 41e21749f3920..209742998f909 100644 --- a/compiler/rustc_middle/src/mir/basic_blocks.rs +++ b/compiler/rustc_middle/src/mir/basic_blocks.rs @@ -63,7 +63,7 @@ impl<'tcx> BasicBlocks<'tcx> { #[inline] pub fn reverse_postorder(&self) -> &[BasicBlock] { self.cache.reverse_postorder.get_or_init(|| { - let mut rpo: Vec<_> = Postorder::new(&self.basic_blocks, START_BLOCK, None).collect(); + let mut rpo: Vec<_> = Postorder::new(&self.basic_blocks, START_BLOCK).collect(); rpo.reverse(); rpo }) diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index 57c2883ef42e4..48a790a16d6cc 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -32,8 +32,8 @@ use crate::mir::interpret::{AllocRange, Scalar}; use crate::ty::codec::{TyDecoder, TyEncoder}; use crate::ty::print::{FmtPrinter, Printer, pretty_print_const, with_no_trimmed_paths}; use crate::ty::{ - self, GenericArg, GenericArgsRef, Instance, InstanceKind, List, Ty, TyCtxt, TypeVisitableExt, - TypingEnv, UserTypeAnnotationIndex, + self, GenericArg, GenericArgsRef, InstanceKind, List, Ty, TyCtxt, TypeVisitableExt, TypingEnv, + UserTypeAnnotationIndex, }; mod basic_blocks; @@ -98,6 +98,8 @@ impl MirPhase { MirPhase::Runtime(RuntimePhase::Initial) => "runtime", MirPhase::Runtime(RuntimePhase::PostCleanup) => "runtime-post-cleanup", MirPhase::Runtime(RuntimePhase::Optimized) => "runtime-optimized", + MirPhase::Monomorphic(MonomorphicPhase::Initial) => "mono", + MirPhase::Monomorphic(MonomorphicPhase::Codegen) => "mono-codegen", } } @@ -108,6 +110,7 @@ impl MirPhase { MirPhase::Built => (1, 1), MirPhase::Analysis(analysis_phase) => (2, 1 + analysis_phase as usize), MirPhase::Runtime(runtime_phase) => (3, 1 + runtime_phase as usize), + MirPhase::Monomorphic(mono_phase) => (4, 1 + mono_phase as usize), } } } @@ -434,6 +437,7 @@ impl<'tcx> Body<'tcx> { TypingEnv::post_analysis(tcx, self.source.def_id()) } MirPhase::Runtime(_) => TypingEnv::post_analysis(tcx, self.source.def_id()), + MirPhase::Monomorphic(_) => TypingEnv::fully_monomorphized(), } } else { match self.phase { @@ -442,6 +446,7 @@ impl<'tcx> Body<'tcx> { ty::TypingMode::non_body_analysis(), ), MirPhase::Runtime(_) => TypingEnv::post_analysis(tcx, self.source.def_id()), + MirPhase::Monomorphic(_) => TypingEnv::fully_monomorphized(), } } } @@ -609,77 +614,6 @@ impl<'tcx> Body<'tcx> { self.injection_phase.is_some() } - /// If this basic block ends with a [`TerminatorKind::SwitchInt`] for which we can evaluate the - /// discriminant in monomorphization, we return the discriminant bits and the - /// [`SwitchTargets`], just so the caller doesn't also have to match on the terminator. - fn try_const_mono_switchint<'a>( - tcx: TyCtxt<'tcx>, - instance: Instance<'tcx>, - block: &'a BasicBlockData<'tcx>, - ) -> Option<(u128, &'a SwitchTargets)> { - // There are two places here we need to evaluate a constant. - let eval_mono_const = |constant: &ConstOperand<'tcx>| { - // FIXME(#132279): what is this, why are we using an empty environment here. - let typing_env = ty::TypingEnv::fully_monomorphized(); - let mono_literal = instance.instantiate_mir_and_normalize_erasing_regions( - tcx, - typing_env, - crate::ty::EarlyBinder::bind(constant.const_), - ); - mono_literal.try_eval_bits(tcx, typing_env) - }; - - let TerminatorKind::SwitchInt { discr, targets } = &block.terminator().kind else { - return None; - }; - - // If this is a SwitchInt(const _), then we can just evaluate the constant and return. - let discr = match discr { - Operand::Constant(constant) => { - let bits = eval_mono_const(constant)?; - return Some((bits, targets)); - } - Operand::RuntimeChecks(check) => { - let bits = check.value(tcx.sess) as u128; - return Some((bits, targets)); - } - Operand::Move(place) | Operand::Copy(place) => place, - }; - - // MIR for `if false` actually looks like this: - // _1 = const _ - // SwitchInt(_1) - // - // And MIR for if intrinsics::ub_checks() looks like this: - // _1 = UbChecks() - // SwitchInt(_1) - // - // So we're going to try to recognize this pattern. - // - // If we have a SwitchInt on a non-const place, we find the most recent statement that - // isn't a storage marker. If that statement is an assignment of a const to our - // discriminant place, we evaluate and return the const, as if we've const-propagated it - // into the SwitchInt. - - let last_stmt = block.statements.iter().rev().find(|stmt| { - !matches!(stmt.kind, StatementKind::StorageDead(_) | StatementKind::StorageLive(_)) - })?; - - let (place, rvalue) = last_stmt.kind.as_assign()?; - - if discr != place { - return None; - } - - match rvalue { - Rvalue::Use(Operand::Constant(constant), _) => { - let bits = eval_mono_const(constant)?; - Some((bits, targets)) - } - _ => None, - } - } - /// For a `Location` in this scope, determine what the "caller location" at that point is. This /// is interesting because of inlining: the `#[track_caller]` attribute of inlined functions /// must be honored. Falls back to the `tracked_caller` value for `#[track_caller]` functions, @@ -1379,19 +1313,6 @@ impl<'tcx> BasicBlockData<'tcx> { self.statements.is_empty() && matches!(self.terminator().kind, TerminatorKind::Unreachable) } - /// Like [`Terminator::successors`] but tries to use information available from the [`Instance`] - /// to skip successors like the `false` side of an `if const {`. - /// - /// This is used to implement [`traversal::mono_reachable`] and - /// [`traversal::mono_reachable_reverse_postorder`]. - pub fn mono_successors(&self, tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Successors<'_> { - if let Some((bits, targets)) = Body::try_const_mono_switchint(tcx, instance, self) { - targets.successors_for_value(bits) - } else { - self.terminator().successors() - } - } - pub fn retain_statements(&mut self, mut f: F) where F: FnMut(&Statement<'tcx>) -> bool, diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs index 8b015e6cecaae..26523b9ddd206 100644 --- a/compiler/rustc_middle/src/mir/syntax.rs +++ b/compiler/rustc_middle/src/mir/syntax.rs @@ -92,6 +92,12 @@ pub enum MirPhase { /// /// The phases of this dialect are described in `RuntimePhase`. Runtime(RuntimePhase), + + /// The "monomorphic MIR" dialect, used for CTFE, optimizations, and codegen. + /// + /// This MIR has the same semantics as runtime MIR, but it requires all types to be + /// monomorphic. + Monomorphic(MonomorphicPhase), } /// See [`MirPhase::Analysis`]. @@ -141,6 +147,16 @@ pub enum RuntimePhase { Optimized = 2, } +/// See [`MirPhase::Monomorphic`]. +#[derive(Copy, Clone, TyEncodable, TyDecodable, Debug, PartialEq, Eq, PartialOrd, Ord)] +#[derive(StableHash)] +pub enum MonomorphicPhase { + /// This MIR has just been monomorphized from runtime MIR. + Initial = 0, + /// MIR is ready for codegen. + Codegen = 1, +} + /////////////////////////////////////////////////////////////////////////// // Borrow kinds diff --git a/compiler/rustc_middle/src/mir/traversal.rs b/compiler/rustc_middle/src/mir/traversal.rs index a3c0a5b83ffad..f29e3847134f3 100644 --- a/compiler/rustc_middle/src/mir/traversal.rs +++ b/compiler/rustc_middle/src/mir/traversal.rs @@ -98,21 +98,17 @@ pub struct Postorder<'a, 'tcx> { basic_blocks: &'a IndexSlice>, visited: DenseBitSet, visit_stack: Vec<(BasicBlock, Successors<'a>)>, - /// A non-empty `extra` allows for a precise calculation of the successors. - extra: Option<(TyCtxt<'tcx>, Instance<'tcx>)>, } impl<'a, 'tcx> Postorder<'a, 'tcx> { pub fn new( basic_blocks: &'a IndexSlice>, root: BasicBlock, - extra: Option<(TyCtxt<'tcx>, Instance<'tcx>)>, ) -> Postorder<'a, 'tcx> { let mut po = Postorder { basic_blocks, visited: DenseBitSet::new_empty(basic_blocks.len()), visit_stack: Vec::new(), - extra, }; po.visit(root); @@ -126,11 +122,7 @@ impl<'a, 'tcx> Postorder<'a, 'tcx> { return; } let data = &self.basic_blocks[bb]; - let successors = if let Some(extra) = self.extra { - data.mono_successors(extra.0, extra.1) - } else { - data.terminator().successors() - }; + let successors = data.terminator().successors(); self.visit_stack.push((bb, successors)); } @@ -225,20 +217,6 @@ pub fn postorder<'a, 'tcx>( reverse_postorder(body).rev() } -pub fn mono_reachable_reverse_postorder<'a, 'tcx>( - body: &'a Body<'tcx>, - tcx: TyCtxt<'tcx>, - instance: Instance<'tcx>, -) -> Vec { - let mut iter = Postorder::new(&body.basic_blocks, START_BLOCK, Some((tcx, instance))); - let mut items = Vec::with_capacity(body.basic_blocks.len()); - while let Some(block) = iter.next() { - items.push(block); - } - items.reverse(); - items -} - /// Returns an iterator over all basic blocks reachable from the `START_BLOCK` in no particular /// order. /// @@ -286,91 +264,3 @@ pub fn reverse_postorder<'a, 'tcx>( { body.basic_blocks.reverse_postorder().iter().map(|&bb| (bb, &body.basic_blocks[bb])) } - -/// Traversal of a [`Body`] that tries to avoid unreachable blocks in a monomorphized [`Instance`]. -/// -/// This is allowed to have false positives; blocks may be visited even if they are not actually -/// reachable. -/// -/// Such a traversal is mostly useful because it lets us skip lowering the `false` side -/// of `if ::CONST`, as well as [`Operand::RuntimeChecks`]. -/// -/// [`Operand::RuntimeChecks`]: rustc_middle::mir::Operand::RuntimeChecks -pub fn mono_reachable<'a, 'tcx>( - body: &'a Body<'tcx>, - tcx: TyCtxt<'tcx>, - instance: Instance<'tcx>, -) -> MonoReachable<'a, 'tcx> { - MonoReachable::new(body, tcx, instance) -} - -/// [`MonoReachable`] internally accumulates a [`DenseBitSet`] of visited blocks. This is just a -/// convenience function to run that traversal then extract its set of reached blocks. -pub fn mono_reachable_as_bitset<'a, 'tcx>( - body: &'a Body<'tcx>, - tcx: TyCtxt<'tcx>, - instance: Instance<'tcx>, -) -> DenseBitSet { - let mut iter = mono_reachable(body, tcx, instance); - while let Some(_) = iter.next() {} - iter.visited -} - -pub struct MonoReachable<'a, 'tcx> { - body: &'a Body<'tcx>, - tcx: TyCtxt<'tcx>, - instance: Instance<'tcx>, - visited: DenseBitSet, - // Other traversers track their worklist in a Vec. But we don't care about order, so we can - // store ours in a DenseBitSet and thus save allocations because DenseBitSet has a small size - // optimization. - worklist: DenseBitSet, -} - -impl<'a, 'tcx> MonoReachable<'a, 'tcx> { - pub fn new( - body: &'a Body<'tcx>, - tcx: TyCtxt<'tcx>, - instance: Instance<'tcx>, - ) -> MonoReachable<'a, 'tcx> { - let mut worklist = DenseBitSet::new_empty(body.basic_blocks.len()); - worklist.insert(START_BLOCK); - MonoReachable { - body, - tcx, - instance, - visited: DenseBitSet::new_empty(body.basic_blocks.len()), - worklist, - } - } - - fn add_work(&mut self, blocks: impl IntoIterator) { - for block in blocks.into_iter() { - if !self.visited.contains(block) { - self.worklist.insert(block); - } - } - } -} - -impl<'a, 'tcx> Iterator for MonoReachable<'a, 'tcx> { - type Item = (BasicBlock, &'a BasicBlockData<'tcx>); - - fn next(&mut self) -> Option<(BasicBlock, &'a BasicBlockData<'tcx>)> { - while let Some(idx) = self.worklist.iter().next() { - self.worklist.remove(idx); - if !self.visited.insert(idx) { - continue; - } - - let data = &self.body[idx]; - - let targets = data.mono_successors(self.tcx, self.instance); - self.add_work(targets); - - return Some((idx, data)); - } - - None - } -} diff --git a/compiler/rustc_middle/src/queries.rs b/compiler/rustc_middle/src/queries.rs index d62e4ec941d21..0788e989f1daa 100644 --- a/compiler/rustc_middle/src/queries.rs +++ b/compiler/rustc_middle/src/queries.rs @@ -45,6 +45,7 @@ #![allow(unused_parens)] +use std::borrow::Cow; use std::ffi::OsStr; use std::path::PathBuf; use std::sync::Arc; @@ -725,14 +726,22 @@ rustc_queries! { } } - /// MIR after our optimization passes have run. This is MIR that is ready - /// for codegen. This is also the only query that can fetch non-local MIR, at present. + /// Polymorphic MIR after our pre-mono optimization passes have run. + /// This is the MIR that crates export. query optimized_mir(key: DefId) -> &'tcx mir::Body<'tcx> { desc { "optimizing MIR for `{}`", tcx.def_path_str(key) } cache_on_disk separate_provide_extern } + /// MIR for a specific Instance ready for codegen. This is `optimized_mir` but monomorphized + /// and with extra transforms applied. + query build_codegen_mir(key: ty::Instance<'tcx>) -> &'tcx Steal>> { + desc { "finalizing codegen MIR for `{}`", tcx.def_path_str_with_args(key.def_id(), key.args) } + arena_cache + no_hash + } + /// Checks for the nearest `#[coverage(off)]` or `#[coverage(on)]` on /// this def and any enclosing defs, up to the crate root. /// diff --git a/compiler/rustc_mir_build/src/builder/custom/mod.rs b/compiler/rustc_mir_build/src/builder/custom/mod.rs index 1005dd30d73f4..4f58a8a230069 100644 --- a/compiler/rustc_mir_build/src/builder/custom/mod.rs +++ b/compiler/rustc_mir_build/src/builder/custom/mod.rs @@ -116,20 +116,27 @@ fn parse_attribute(dialect: Option, phase: Option match phase { None | Some(attrs::MirPhase::Initial) => MirPhase::Analysis(AnalysisPhase::Initial), - Some(attrs::MirPhase::PostCleanup) => MirPhase::Analysis(AnalysisPhase::PostCleanup), - + // Caught during attribute checking. Some(attrs::MirPhase::Optimized) => { - // Caught during attribute checking. bug!("`optimized` dialect is not compatible with the `analysis` dialect") } }, - attrs::MirDialect::Runtime => match phase { None | Some(attrs::MirPhase::Initial) => MirPhase::Runtime(RuntimePhase::Initial), Some(attrs::MirPhase::PostCleanup) => MirPhase::Runtime(RuntimePhase::PostCleanup), Some(attrs::MirPhase::Optimized) => MirPhase::Runtime(RuntimePhase::Optimized), }, + attrs::MirDialect::Mono => match phase { + None | Some(attrs::MirPhase::Initial) => { + MirPhase::Monomorphic(MonomorphicPhase::Initial) + } + // Caught during attribute checking. + Some(attrs::MirPhase::PostCleanup) => { + bug!("`post-cleanup` dialect is not compatible with the `mono` dialect") + } + Some(attrs::MirPhase::Optimized) => MirPhase::Monomorphic(MonomorphicPhase::Codegen), + }, } } diff --git a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs index cd6395eb2e8e8..9e48cd16055d6 100644 --- a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs +++ b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs @@ -9,14 +9,19 @@ //! after `optimized_mir`! We check for things that are *not* guaranteed to be preserved by MIR //! transforms, such as which local variables happen to be mutated. +use rustc_hir::ConstContext; use rustc_hir::def_id::LocalDefId; use rustc_index::IndexVec; use rustc_middle::middle::deduced_param_attrs::{DeducedParamAttrs, UsageSummary}; -use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor}; +use rustc_middle::mir::visit::{ + MutVisitor, MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor, +}; use rustc_middle::mir::*; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_session::config::OptLevel; +use crate::MirPass; + /// A visitor that determines how a return place and arguments are used inside MIR body. /// To determine whether a local is mutated we can't use the mutability field on LocalDecl /// because it has no meaning post-optimization. @@ -164,6 +169,12 @@ fn type_will_always_be_passed_directly(ty: Ty<'_>) -> bool { ) } +fn is_enabled(sess: &rustc_session::Session) -> bool { + // This computation is unfortunately rather expensive, so don't do it unless we're optimizing. + // Also skip it in incremental mode. + sess.opts.optimize != OptLevel::No && sess.opts.incremental.is_none() +} + /// Returns the deduced parameter attributes for a function. /// /// Deduced parameter attributes are those that can only be soundly determined by examining the @@ -175,9 +186,7 @@ pub(super) fn deduced_param_attrs<'tcx>( tcx: TyCtxt<'tcx>, def_id: LocalDefId, ) -> &'tcx [DeducedParamAttrs] { - // This computation is unfortunately rather expensive, so don't do it unless we're optimizing. - // Also skip it in incremental mode. - if tcx.sess.opts.optimize == OptLevel::No || tcx.sess.opts.incremental.is_some() { + if !is_enabled(tcx.sess) { return &[]; } @@ -231,3 +240,133 @@ pub(super) fn deduced_param_attrs<'tcx>( deduced_param_attrs } + +/// `deduced_param_attrs` works on polymorphic optimized MIR. However, codegen works on +/// monomorphic MIR that may have modified in between. This pass makes sure that deduced and actual +/// param attrs match. +pub(crate) struct RecoverDeducedParamAttrs; + +impl<'tcx> MirPass<'tcx> for RecoverDeducedParamAttrs { + fn is_required(&self) -> bool { + true + } + + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + is_enabled(sess) + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + // Only run this for MIR that has `optimized_mir`. + let ty::InstanceKind::Item(..) = body.source.instance else { return }; + let Some(def_id) = body.source.def_id().as_local() else { return }; + match tcx.hir_body_const_context(def_id) { + Some(ConstContext::ConstFn) | None => {} + Some(_) => return, + }; + + // If we deduced nothing, there is no fixup to perform. + let deduced_param_attrs = tcx.deduced_param_attrs(def_id); + if deduced_param_attrs.is_empty() { + return; + } + + // Reuse the computation from `deduced_param_attrs`. + let mut actual_read_only = DeduceParamAttrs::new(body); + actual_read_only.visit_body(body); + + let typing_env = body.typing_env(tcx); + + let wrong_mutated_args = body + .args_iter() + .filter(|&local| { + let Some(deduced) = deduced_param_attrs.get(local.as_usize()) else { return false }; + let actual = DeducedParamAttrs { usage: actual_read_only.usage[local] }; + let ty = body.local_decls[local].ty; + + // If we deduced `read_only` and the actual MIR is mutable, we must do something. + (deduced.read_only(tcx, typing_env, ty) && !actual.read_only(tcx, typing_env, ty)) + || (deduced.captures_none(tcx, typing_env, ty) + && !actual.captures_none(tcx, typing_env, ty)) + }) + .collect::>(); + + let wrong_captures_return = { + let deduced = deduced_param_attrs[0]; + let actual = DeducedParamAttrs { usage: actual_read_only.usage[RETURN_PLACE] }; + let ty = body.local_decls[RETURN_PLACE].ty; + + deduced.captures_none(tcx, typing_env, ty) && !actual.captures_none(tcx, typing_env, ty) + }; + + if wrong_mutated_args.is_empty() && !wrong_captures_return { + // All arguments match, we have nothing to do. + return; + } + + // For each flagged local, insert a move at the beginning of MIR. This ensures that the + // original argument's value is not mutated, and that all mutation happens on the newly + // introduced local. + let mut renamed_args = IndexVec::from_fn_n(|l| l, body.arg_count + 1); + if wrong_captures_return { + let decl = &body.local_decls[RETURN_PLACE]; + let new_local = body.local_decls.push(decl.clone()); + renamed_args[RETURN_PLACE] = new_local; + } + + let mut new_statements = Vec::with_capacity(wrong_mutated_args.len()); + for local in wrong_mutated_args { + let decl = &body.local_decls[local]; + let source_info = decl.source_info; + let new_local = body.local_decls.push(decl.clone()); + renamed_args[local] = new_local; + + // `new_local = move local` + let stmt = StatementKind::Assign(Box::new(( + new_local.into(), + Rvalue::Use(Operand::Move(local.into()), WithRetag::Yes), + ))); + new_statements.push(Statement::new(source_info, stmt)); + } + + RenameLocals { tcx, renamed_args: &renamed_args }.visit_body_preserves_cfg(body); + + body.basic_blocks.as_mut_preserves_cfg()[START_BLOCK] + .statements + .splice(0..0, new_statements); + if wrong_captures_return { + let new_local = renamed_args[RETURN_PLACE]; + for bbdata in body.basic_blocks.as_mut_preserves_cfg().iter_mut() { + let term = bbdata.terminator(); + if let TerminatorKind::Return = term.kind { + let stmt = StatementKind::Assign(Box::new(( + RETURN_PLACE.into(), + Rvalue::Use(Operand::Move(new_local.into()), WithRetag::Yes), + ))); + bbdata.statements.push(Statement::new(term.source_info, stmt)); + } + } + } + + struct RenameLocals<'tcx, 'a> { + tcx: TyCtxt<'tcx>, + renamed_args: &'a IndexVec, + } + impl<'tcx> MutVisitor<'tcx> for RenameLocals<'tcx, '_> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + fn visit_local(&mut self, local: &mut Local, _: PlaceContext, _: Location) { + if let Some(&new_local) = self.renamed_args.get(*local) { + *local = new_local; + } + } + fn visit_terminator(&mut self, term: &mut Terminator<'tcx>, loc: Location) { + // Do not visit the implicit `_0` in `return`, as we create an assignment for it. + if let TerminatorKind::Return = term.kind { + return; + } + self.super_terminator(term, loc) + } + } + } +} diff --git a/compiler/rustc_mir_transform/src/instsimplify.rs b/compiler/rustc_mir_transform/src/instsimplify.rs index 87104364da6af..047619b2c67d1 100644 --- a/compiler/rustc_mir_transform/src/instsimplify.rs +++ b/compiler/rustc_mir_transform/src/instsimplify.rs @@ -30,10 +30,6 @@ impl<'tcx> crate::MirPass<'tcx> for InstSimplify { } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let preserve_ub_checks = find_attr!(tcx.hir_krate_attrs(), RustcPreserveUbChecks); - if !preserve_ub_checks { - SimplifyUbCheck { tcx }.visit_body(body); - } let mut ctx = InstSimplifyContext { tcx, typing_env: body.typing_env(tcx), @@ -449,11 +445,45 @@ fn resolve_rust_intrinsic<'tcx>( Some((intrinsic.name, args)) } -struct SimplifyUbCheck<'tcx> { +pub(super) enum SimplifyUbChecks { + AfterSimplifyCfg, + PostMono, +} + +impl<'tcx> crate::MirPass<'tcx> for SimplifyUbChecks { + fn name(&self) -> &'static str { + match self { + SimplifyUbChecks::AfterSimplifyCfg => "SimplifyUbChecks-after-simplifycfg", + SimplifyUbChecks::PostMono => "SimplifyUbChecks-post-mono", + } + } + + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() > 0 + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let preserve_ub_checks = match self { + SimplifyUbChecks::PostMono => false, + SimplifyUbChecks::AfterSimplifyCfg => { + find_attr!(tcx.hir_krate_attrs(), RustcPreserveUbChecks) + } + }; + if !preserve_ub_checks { + SimplifyUbCheckVisitor { tcx }.visit_body(body); + } + } + + fn is_required(&self) -> bool { + false + } +} + +struct SimplifyUbCheckVisitor<'tcx> { tcx: TyCtxt<'tcx>, } -impl<'tcx> MutVisitor<'tcx> for SimplifyUbCheck<'tcx> { +impl<'tcx> MutVisitor<'tcx> for SimplifyUbCheckVisitor<'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 310604207f8de..36c5cb6b96db3 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -8,6 +8,9 @@ #![feature(yeet_expr)] // tidy-alphabetical-end +use std::borrow::Cow; +use std::ops::ControlFlow; + use hir::ConstContext; use required_consts::RequiredConstsVisitor; use rustc_const_eval::check_consts::{self, ConstCx}; @@ -18,12 +21,15 @@ use rustc_hir as hir; use rustc_hir::def::{CtorKind, DefKind}; use rustc_hir::def_id::LocalDefId; use rustc_index::IndexVec; +use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::{ AnalysisPhase, Body, CallSource, ClearCrossCrate, ConstOperand, ConstQualifs, LocalDecl, - MirPhase, Operand, Place, ProjectionElem, Promoted, RuntimePhase, Rvalue, START_BLOCK, - SourceInfo, Statement, StatementKind, TerminatorKind, WithRetag, + Location, MirPhase, MonomorphicPhase, Operand, Place, ProjectionElem, Promoted, RuntimePhase, + Rvalue, START_BLOCK, SourceInfo, Statement, StatementKind, TerminatorKind, WithRetag, +}; +use rustc_middle::ty::{ + self, GenericParamDefKind, Instance, Ty, TyCtxt, TypeVisitable, TypeVisitableExt, Unnormalized, }; -use rustc_middle::ty::{self, TyCtxt, TypeVisitableExt}; use rustc_middle::util::Providers; use rustc_middle::{bug, query, span_bug}; use rustc_span::{DUMMY_SP, Spanned, sym}; @@ -39,7 +45,6 @@ use pass_manager::{self as pm, Lint, MirLint, MirPass, WithMinOptLevel}; mod check_pointers; mod cost_checker; mod cross_crate_inline; -mod deduce_param_attrs; mod elaborate_drop; mod errors; mod ffi_unwind_calls; @@ -139,6 +144,7 @@ declare_passes! { Initial, Final }; + mod deduce_param_attrs : RecoverDeducedParamAttrs; mod deref_separator : Derefer; mod dest_prop : DestinationPropagation; mod early_otherwise_branch : EarlyOtherwiseBranch; @@ -151,12 +157,15 @@ declare_passes! { // by custom rustc drivers, running all the steps by themselves. See #114628. pub mod inline : Inline, ForceInline; mod impossible_predicates : ImpossiblePredicates; - mod instsimplify : InstSimplify { BeforeInline, AfterSimplifyCfg }; + mod instsimplify : + InstSimplify { BeforeInline, AfterSimplifyCfg }, + SimplifyUbChecks { AfterSimplifyCfg, PostMono }; mod jump_threading : JumpThreading; mod known_panics_lint : KnownPanicsLint; mod large_enums : EnumSizeOpt; mod lower_intrinsics : LowerIntrinsics; mod lower_slice_len : LowerSliceLenCalls; + mod optimize_use_clone : OptimizeUseClone; mod match_branches : MatchBranchSimplification; mod mentioned_items : MentionedItems; mod multiple_return_terminators : MultipleReturnTerminators; @@ -183,7 +192,8 @@ declare_passes! { PreOptimizations, Final, MakeShim, - AfterUnreachableEnumBranching + AfterUnreachableEnumBranching, + PostMono }, SimplifyLocals { BeforeConstProp, @@ -193,7 +203,8 @@ declare_passes! { mod simplify_branches : SimplifyConstCondition { AfterInstSimplify, AfterConstProp, - Final + Final, + PostMono }; mod simplify_comparison_integral : SimplifyComparisonIntegral; mod single_use_consts : SingleUseConsts; @@ -227,6 +238,7 @@ pub fn provide(providers: &mut Providers) { deduced_param_attrs: deduce_param_attrs::deduced_param_attrs, coroutine_by_move_body_def_id: coroutine::coroutine_by_move_body_def_id, trivial_const: trivial_const::trivial_const_provider, + build_codegen_mir, ..providers.queries }; } @@ -669,11 +681,11 @@ fn run_runtime_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { } } -pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - fn o1(x: T) -> WithMinOptLevel { - WithMinOptLevel(1, x) - } +fn o1(x: T) -> WithMinOptLevel { + WithMinOptLevel(1, x) +} +pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let def_id = body.source.def_id(); let optimizations = if tcx.def_kind(def_id).has_codegen_attrs() && tcx.codegen_fn_attrs(def_id).optimize.do_not_optimize() @@ -722,6 +734,7 @@ pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<' // optimizations. This invalidates CFG caches, so avoid putting between // `ReferencePropagation` and `GVN` which both use the dominator tree. &instsimplify::InstSimplify::AfterSimplifyCfg, + &instsimplify::SimplifyUbChecks::AfterSimplifyCfg, // After `InstSimplify-after-simplifycfg` with `-Zub_checks=false`, simplify // ``` // _13 = const false; @@ -758,8 +771,6 @@ pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<' &simplify::SimplifyLocals::Final, &multiple_return_terminators::MultipleReturnTerminators, &large_enums::EnumSizeOpt { discrepancy: 128 }, - // Some cleanup necessary at least for LLVM and potentially other codegen backends. - &add_call_guards::CriticalCallEdges, // Cleanup for human readability, off by default. &prettify::ReorderBasicBlocks, &prettify::ReorderLocals, @@ -815,6 +826,129 @@ fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> { run_optimization_passes(tcx, &mut body); + // If the body is polymorphic, we're done. Otherwise, we're going to turn the body into codegen + // MIR ahead of time, as an optimization. + if tcx.generics_of(did).requires_monomorphization(tcx) { + return body; + } + + // If the body has impossible predicates, the normalization we need to do to turn this into + // post-mono MIR will fail. So we detect that case and return just optimized MIR. + let args = ty::GenericArgs::for_item(tcx, did.into(), |param, _| match param.kind { + GenericParamDefKind::Lifetime => tcx.lifetimes.re_erased.into(), + GenericParamDefKind::Type { .. } | GenericParamDefKind::Const { .. } => { + unreachable!( + "`requires_monomorphization` check means that we should have no type/const params" + ) + } + }); + if tcx.instantiate_and_check_impossible_predicates((did.to_def_id(), args)) { + return body; + } + + // If the body contains the UB check intrinsic that is only to be monomorphized at codegen + // time, we cannot generate codegen MIR early. We want post-mono optimizations to optimize on + // whether UB checks are enabled or disabled at codegen time. + let mut vis = MonoCompatVisitor { contains_runtime_check: false }; + vis.visit_body(&body); + if vis.contains_runtime_check { + return body; + } + + if body.visit_with(&mut Normalizable { tcx }).is_break() { + return body; + } + + // Monomorphizing this body won't reveal any new information that is useful for + // optimizations, so we just run passes that make the MIR ready for codegen backends. + pm::run_passes( + tcx, + &mut body, + &[&add_call_guards::CriticalCallEdges], + Some(MirPhase::Monomorphic(MonomorphicPhase::Codegen)), + pm::Optimizations::Allowed, + ); + return body; + + struct MonoCompatVisitor { + contains_runtime_check: bool, + } + impl<'tcx> Visitor<'tcx> for MonoCompatVisitor { + fn visit_operand(&mut self, operand: &Operand<'tcx>, location: Location) { + if let Operand::RuntimeChecks(_) = operand { + self.contains_runtime_check = true; + } + self.super_operand(operand, location); + } + } + + struct Normalizable<'tcx> { + tcx: TyCtxt<'tcx>, + } + impl<'tcx> ty::TypeVisitor> for Normalizable<'tcx> { + type Result = ControlFlow<(), ()>; + + fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<()> { + match self.tcx.try_normalize_erasing_regions( + ty::TypingEnv::fully_monomorphized(), + Unnormalized::new_wip(t), + ) { + Ok(_) => ControlFlow::Continue(()), + Err(_) => ControlFlow::Break(()), + } + } + } +} + +pub fn build_codegen_mir<'tcx>( + tcx: TyCtxt<'tcx>, + instance: Instance<'tcx>, +) -> Steal>> { + let body = tcx.instance_mir(instance.def); + + // If we have generic params, assert that we didn't get codegen MIR out of instance_mir + if instance.args.non_erasable_generics().next().is_some() { + assert!(!matches!(body.phase, MirPhase::Monomorphic(MonomorphicPhase::Codegen))); + } + + let body = if !matches!(body.phase, MirPhase::Monomorphic(MonomorphicPhase::Codegen)) { + let body = transform_to_codegen_mir(tcx, instance, body.clone()); + Cow::Owned(body) + } else { + Cow::Borrowed(body) + }; + + Steal::new(body) +} + +fn transform_to_codegen_mir<'tcx>( + tcx: TyCtxt<'tcx>, + instance: Instance<'tcx>, + body: Body<'tcx>, +) -> Body<'tcx> { + let mut body = instance.instantiate_mir_and_normalize_erasing_regions( + tcx, + ty::TypingEnv::fully_monomorphized(), + ty::EarlyBinder::bind(body), + ); + body.phase = MirPhase::Monomorphic(MonomorphicPhase::Initial); + body.pass_count = 0; + pm::dump_mir_for_phase_change(tcx, &body); + + pm::run_passes( + tcx, + &mut body, + &[ + &instsimplify::SimplifyUbChecks::PostMono, + &o1(simplify_branches::SimplifyConstCondition::PostMono), + &o1(simplify::SimplifyCfg::PostMono), + &optimize_use_clone::OptimizeUseClone, + &add_call_guards::CriticalCallEdges, + &deduce_param_attrs::RecoverDeducedParamAttrs, + ], + Some(MirPhase::Monomorphic(MonomorphicPhase::Codegen)), + pm::Optimizations::Allowed, + ); body } diff --git a/compiler/rustc_mir_transform/src/optimize_use_clone.rs b/compiler/rustc_mir_transform/src/optimize_use_clone.rs new file mode 100644 index 0000000000000..b411844f2aa99 --- /dev/null +++ b/compiler/rustc_mir_transform/src/optimize_use_clone.rs @@ -0,0 +1,65 @@ +//! Optimize `Clone` calls created from `use` statements into direct copies for codegen MIR. + +use rustc_middle::mir::*; +use rustc_middle::ty::{self, TyCtxt}; +use rustc_session::Session; + +use crate::deref_separator::deref_finder; + +pub(super) struct OptimizeUseClone; + +impl<'tcx> crate::MirPass<'tcx> for OptimizeUseClone { + fn is_enabled(&self, sess: &Session) -> bool { + sess.mir_opt_level() >= 4 + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + if !tcx.features().ergonomic_clones() { + return; + } + + let typing_env = body.typing_env(tcx); + let basic_blocks = body.basic_blocks.as_mut(); + for block in basic_blocks { + let TerminatorKind::Call { + args: [arg], + destination, + target: Some(target), + call_source: CallSource::Use, + .. + } = &block.terminator().kind + else { + continue; + }; + + let arg_ty = arg.node.ty(&body.local_decls, tcx); + let ty::Ref(_, inner_ty, Mutability::Not) = *arg_ty.kind() else { + continue; + }; + + if !tcx.type_is_copy_modulo_regions(typing_env, inner_ty) { + continue; + } + + let Some(arg_place) = arg.node.place() else { + continue; + }; + + let target = *target; + block.statements.push(Statement::new( + block.terminator().source_info, + StatementKind::Assign(Box::new(( + *destination, + Rvalue::Use(Operand::Copy(tcx.mk_place_deref(arg_place)), WithRetag::Yes), + ))), + )); + block.terminator_mut().kind = TerminatorKind::Goto { target }; + } + + deref_finder(tcx, body, false); + } + + fn is_required(&self) -> bool { + false + } +} diff --git a/compiler/rustc_mir_transform/src/post_analysis_normalize.rs b/compiler/rustc_mir_transform/src/post_analysis_normalize.rs index 776840a6fe8b2..26c7f6c655f22 100644 --- a/compiler/rustc_mir_transform/src/post_analysis_normalize.rs +++ b/compiler/rustc_mir_transform/src/post_analysis_normalize.rs @@ -83,4 +83,16 @@ impl<'tcx> MutVisitor<'tcx> for PostAnalysisNormalizeVisitor<'tcx> { *ty = t; } } + + #[inline] + fn visit_args(&mut self, args: &mut ty::GenericArgsRef<'tcx>, _: Location) { + // We have to use `try_normalize_erasing_regions` here, since it's + // possible that we visit impossible-to-satisfy where clauses here, + // see #91745 + if let Ok(a) = + self.tcx.try_normalize_erasing_regions(self.typing_env, Unnormalized::new_wip(*args)) + { + *args = a; + } + } } diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index 14ab4fb0e74eb..92cc0ffd8dbcd 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -57,6 +57,7 @@ pub(super) enum SimplifyCfg { Final, MakeShim, AfterUnreachableEnumBranching, + PostMono, } impl SimplifyCfg { @@ -72,6 +73,7 @@ impl SimplifyCfg { SimplifyCfg::AfterUnreachableEnumBranching => { "SimplifyCfg-after-unreachable-enum-branching" } + SimplifyCfg::PostMono => "SimplifyCfg-post-mono", } } } diff --git a/compiler/rustc_mir_transform/src/simplify_branches.rs b/compiler/rustc_mir_transform/src/simplify_branches.rs index 6d854dfcc6445..b35c0b3293ca5 100644 --- a/compiler/rustc_mir_transform/src/simplify_branches.rs +++ b/compiler/rustc_mir_transform/src/simplify_branches.rs @@ -8,6 +8,7 @@ pub(super) enum SimplifyConstCondition { AfterInstSimplify, AfterConstProp, Final, + PostMono, } /// A pass that replaces a branch with a goto when its condition is known. @@ -19,6 +20,7 @@ impl<'tcx> crate::MirPass<'tcx> for SimplifyConstCondition { } SimplifyConstCondition::AfterConstProp => "SimplifyConstCondition-after-const-prop", SimplifyConstCondition::Final => "SimplifyConstCondition-final", + SimplifyConstCondition::PostMono => "SimplifyConstCondition-post-mono", } } diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index 815b94dc510e8..3713c95ee9a80 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -45,25 +45,27 @@ impl<'tcx> crate::MirPass<'tcx> for Validator { } let def_id = body.source.def_id(); let typing_env = body.typing_env(tcx); - let can_unwind = if body.phase <= MirPhase::Runtime(RuntimePhase::Initial) { - // In this case `AbortUnwindingCalls` haven't yet been executed. - true - } else if !tcx.def_kind(def_id).is_fn_like() { - true - } else { - let body_ty = tcx.type_of(def_id).skip_binder(); - let body_abi = match body_ty.kind() { - ty::FnDef(..) => body_ty.fn_sig(tcx).abi(), - ty::Closure(..) => ExternAbi::RustCall, - ty::CoroutineClosure(..) => ExternAbi::RustCall, - ty::Coroutine(..) => ExternAbi::Rust, - // No need to do MIR validation on error bodies - ty::Error(_) => return, - _ => span_bug!(body.span, "unexpected body ty: {body_ty}"), - }; + let can_unwind = + if body.phase <= MirPhase::Runtime(RuntimePhase::Initial) || !def_id.is_local() { + // In this case `AbortUnwindingCalls` haven't yet been executed. + // Or we are using MIR from another crate with different compilation flags. + true + } else if !tcx.def_kind(def_id).is_fn_like() { + true + } else { + let body_ty = tcx.type_of(def_id).skip_binder(); + let body_abi = match body_ty.kind() { + ty::FnDef(..) => body_ty.fn_sig(tcx).abi(), + ty::Closure(..) => ExternAbi::RustCall, + ty::CoroutineClosure(..) => ExternAbi::RustCall, + ty::Coroutine(..) => ExternAbi::Rust, + // No need to do MIR validation on error bodies + ty::Error(_) => return, + _ => span_bug!(body.span, "unexpected body ty: {body_ty}"), + }; - ty::layout::fn_can_unwind(tcx, Some(def_id), body_abi) - }; + ty::layout::fn_can_unwind(tcx, Some(def_id), body_abi) + }; let mut cfg_checker = CfgChecker { when: &self.when, @@ -383,7 +385,7 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { // the return edge from the call. FIXME(tmiasko): Since this is a strictly code // generation concern, the code generation should be responsible for handling // it. - if self.body.phase >= MirPhase::Runtime(RuntimePhase::Optimized) + if self.body.phase >= MirPhase::Monomorphic(MonomorphicPhase::Codegen) && self.is_critical_call_edge(target, unwind) { self.fail( @@ -1095,7 +1097,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } AggregateKind::RawPtr(pointee_ty, mutability) => { - if !matches!(self.body.phase, MirPhase::Runtime(_)) { + if self.body.phase < MirPhase::Runtime(RuntimePhase::Initial) { // It would probably be fine to support this in earlier phases, but at the // time of writing it's only ever introduced from intrinsic lowering, so // earlier things just `bug!` on it. diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index 9ddd33678c341..1ddda984ae246 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -1339,7 +1339,7 @@ fn collect_items_of_instance<'tcx>( collector.visit_var_debug_info(var_debug_info); } } - for (bb, data) in traversal::mono_reachable(body, tcx, instance) { + for (bb, data) in traversal::reverse_postorder(body) { collector.visit_basic_block_data(bb, data) } } diff --git a/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs b/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs index 10218523ca232..f882ffbec47ab 100644 --- a/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs +++ b/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs @@ -2,7 +2,7 @@ //! that target feature is enabled both on the callee and all callers. use rustc_abi::{BackendRepr, CanonAbi, ExternAbi, RegKind, X86Call}; use rustc_hir::{CRATE_HIR_ID, HirId}; -use rustc_middle::mir::{self, Location, traversal}; +use rustc_middle::mir::{self, Location}; use rustc_middle::ty::{self, Instance, InstanceKind, Ty, TyCtxt}; use rustc_span::def_id::DefId; use rustc_span::{DUMMY_SP, Span, Symbol, sym}; @@ -236,7 +236,7 @@ fn check_call_site_abi<'tcx>( fn check_callees_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, body: &mir::Body<'tcx>) { // Check all function call terminators. - for (bb, _data) in traversal::mono_reachable(body, tcx, instance) { + for (bb, _data) in body.basic_blocks.iter_enumerated() { let terminator = body.basic_blocks[bb].terminator(); match terminator.kind { mir::TerminatorKind::Call { ref func, ref fn_span, .. } @@ -269,7 +269,7 @@ fn check_callees_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, body: &m pub(crate) fn check_feature_dependent_abi<'tcx>( tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, - body: &'tcx mir::Body<'tcx>, + body: &mir::Body<'tcx>, ) { check_instance_abi(tcx, instance); check_callees_abi(tcx, instance, body); diff --git a/compiler/rustc_monomorphize/src/mono_checks/mod.rs b/compiler/rustc_monomorphize/src/mono_checks/mod.rs index 1ecda824fb8c2..40fec8e321adb 100644 --- a/compiler/rustc_monomorphize/src/mono_checks/mod.rs +++ b/compiler/rustc_monomorphize/src/mono_checks/mod.rs @@ -9,9 +9,9 @@ mod abi_check; mod move_check; fn check_mono_item<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) { - let body = tcx.instance_mir(instance.def); + let body = &*tcx.build_codegen_mir(instance).borrow(); abi_check::check_feature_dependent_abi(tcx, instance, body); - move_check::check_moves(tcx, instance, body); + move_check::check_moves(tcx, body); } pub(super) fn provide(providers: &mut Providers) { diff --git a/compiler/rustc_monomorphize/src/mono_checks/move_check.rs b/compiler/rustc_monomorphize/src/mono_checks/move_check.rs index a24b0443d39c9..212854ac62b01 100644 --- a/compiler/rustc_monomorphize/src/mono_checks/move_check.rs +++ b/compiler/rustc_monomorphize/src/mono_checks/move_check.rs @@ -3,40 +3,34 @@ use rustc_data_structures::fx::FxIndexSet; use rustc_hir::def_id::DefId; use rustc_hir::limit::Limit; use rustc_middle::mir::visit::Visitor as MirVisitor; -use rustc_middle::mir::{self, Location, traversal}; -use rustc_middle::ty::{self, AssocTag, Instance, Ty, TyCtxt, TypeFoldable}; +use rustc_middle::mir::{self, Location}; +use rustc_middle::ty::{self, AssocTag, Ty, TyCtxt}; use rustc_session::lint::builtin::LARGE_ASSIGNMENTS; use rustc_span::{Ident, Span, Spanned, sym}; -use tracing::{debug, trace}; +use tracing::debug; use crate::errors::LargeAssignmentsLint; -struct MoveCheckVisitor<'tcx> { +struct MoveCheckVisitor<'a, 'tcx> { tcx: TyCtxt<'tcx>, - instance: Instance<'tcx>, - body: &'tcx mir::Body<'tcx>, + body: &'a mir::Body<'tcx>, /// Spans for move size lints already emitted. Helps avoid duplicate lints. move_size_spans: Vec, } -pub(crate) fn check_moves<'tcx>( - tcx: TyCtxt<'tcx>, - instance: Instance<'tcx>, - body: &'tcx mir::Body<'tcx>, -) { - let mut visitor = MoveCheckVisitor { tcx, instance, body, move_size_spans: vec![] }; - for (bb, data) in traversal::mono_reachable(body, tcx, instance) { +pub(crate) fn check_moves<'tcx>(tcx: TyCtxt<'tcx>, body: &mir::Body<'tcx>) { + let mut visitor = MoveCheckVisitor { tcx, body, move_size_spans: vec![] }; + for (bb, data) in body.basic_blocks.iter_enumerated() { visitor.visit_basic_block_data(bb, data) } } -impl<'tcx> MirVisitor<'tcx> for MoveCheckVisitor<'tcx> { +impl<'tcx> MirVisitor<'tcx> for MoveCheckVisitor<'_, 'tcx> { fn visit_terminator(&mut self, terminator: &mir::Terminator<'tcx>, location: Location) { match terminator.kind { mir::TerminatorKind::Call { ref func, ref args, ref fn_span, .. } | mir::TerminatorKind::TailCall { ref func, ref args, ref fn_span } => { let callee_ty = func.ty(self.body, self.tcx); - let callee_ty = self.monomorphize(callee_ty); self.check_fn_args_move_size(callee_ty, args, *fn_span, location); } _ => {} @@ -51,19 +45,7 @@ impl<'tcx> MirVisitor<'tcx> for MoveCheckVisitor<'tcx> { } } -impl<'tcx> MoveCheckVisitor<'tcx> { - fn monomorphize(&self, value: T) -> T - where - T: TypeFoldable>, - { - trace!("monomorphize: self.instance={:?}", self.instance); - self.instance.instantiate_mir_and_normalize_erasing_regions( - self.tcx, - ty::TypingEnv::fully_monomorphized(), - ty::EarlyBinder::bind(value), - ) - } - +impl<'tcx> MoveCheckVisitor<'_, 'tcx> { fn check_operand_move_size(&mut self, operand: &mir::Operand<'tcx>, location: Location) { let limit = self.tcx.move_size_limit(); if limit.0 == 0 { @@ -125,7 +107,6 @@ impl<'tcx> MoveCheckVisitor<'tcx> { operand: &mir::Operand<'tcx>, ) -> Option { let ty = operand.ty(self.body, self.tcx); - let ty = self.monomorphize(ty); let Ok(layout) = self.tcx.layout_of(ty::TypingEnv::fully_monomorphized().as_query_input(ty)) else { diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 72339efd0a132..7a4607382a58a 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1321,6 +1321,7 @@ symbols! { modifiers, module, module_path, + mono, more_maybe_bounds, more_qualified_paths, more_struct_aliases, diff --git a/src/tools/clippy/clippy_utils/src/mir/mod.rs b/src/tools/clippy/clippy_utils/src/mir/mod.rs index 40c553d1c9e69..6a141a52d447e 100644 --- a/src/tools/clippy/clippy_utils/src/mir/mod.rs +++ b/src/tools/clippy/clippy_utils/src/mir/mod.rs @@ -36,7 +36,7 @@ pub fn visit_local_usage( } }; N]; - traversal::Postorder::new(&mir.basic_blocks, location.block, None) + traversal::Postorder::new(&mir.basic_blocks, location.block) .collect::>() .into_iter() .rev() diff --git a/tests/codegen-llvm/skip-mono-inside-if-false.rs b/tests/codegen-llvm/skip-mono-inside-if-false.rs index 8b95de99dd3bd..fb3a7dbb03a17 100644 --- a/tests/codegen-llvm/skip-mono-inside-if-false.rs +++ b/tests/codegen-llvm/skip-mono-inside-if-false.rs @@ -13,13 +13,9 @@ pub fn demo_for_i32() { // CHECK-LABEL: ; skip_mono_inside_if_false::generic_impl // CHECK: start: -// CHECK-NEXT: br label %[[ELSE_BRANCH:bb[0-9]+]] -// CHECK: [[ELSE_BRANCH]]: -// CHECK-NEXT: call skip_mono_inside_if_false::small_impl -// CHECK: bb{{[0-9]+}}: +// CHECK-NEXT: ; call skip_mono_inside_if_false::small_impl +// CHECK-NEXT: call void // CHECK-NEXT: ret void -// CHECK: bb{{[0-9+]}}: -// CHECK-NEXT: unreachable fn generic_impl() { trait MagicTrait { diff --git a/tests/codegen-llvm/vec-in-place.rs b/tests/codegen-llvm/vec-in-place.rs index a5ef8653b997e..97ae80a554426 100644 --- a/tests/codegen-llvm/vec-in-place.rs +++ b/tests/codegen-llvm/vec-in-place.rs @@ -132,8 +132,7 @@ pub fn vec_iterator_cast_unwrap_drop(vec: Vec>) -> Vec { // CHECK-NOT: br i1 %{{.*}}, label %{{.*}}, label %{{.*}} // CHECK-NOT: %{{.*}} = mul // CHECK-NOT: %{{.*}} = udiv - // CHECK: call - // CHECK-SAME: void @llvm.assume(i1 %{{.+}}) + // CHECK: call void @llvm.assume(i1 %{{.+}}) // CHECK-NOT: br i1 %{{.*}}, label %{{.*}}, label %{{.*}} // CHECK-NOT: call // CHECK-NOT: %{{.*}} = mul @@ -149,8 +148,7 @@ pub fn vec_iterator_cast_wrap_drop(vec: Vec) -> Vec> { // CHECK-NOT: br i1 %{{.*}}, label %{{.*}}, label %{{.*}} // CHECK-NOT: %{{.*}} = mul // CHECK-NOT: %{{.*}} = udiv - // CHECK: call - // CHECK-SAME: void @llvm.assume(i1 %{{.+}}) + // CHECK: void @llvm.assume(i1 %{{.+}}) // CHECK-NOT: br i1 %{{.*}}, label %{{.*}}, label %{{.*}} // CHECK-NOT: call // CHECK-NOT: %{{.*}} = mul diff --git a/tests/incremental/hashes/function_interfaces.rs b/tests/incremental/hashes/function_interfaces.rs index 87719f22b947e..4b73163b0695d 100644 --- a/tests/incremental/hashes/function_interfaces.rs +++ b/tests/incremental/hashes/function_interfaces.rs @@ -140,12 +140,12 @@ pub fn type_parameter () {} #[cfg(not(any(bpass1,bpass4)))] #[rustc_clean( cfg = "bpass2", - except = "opt_hir_owner_nodes, generics_of, type_of, predicates_of" + except = "opt_hir_owner_nodes, generics_of, type_of, predicates_of, optimized_mir" )] #[rustc_clean(cfg = "bpass3")] #[rustc_clean( cfg = "bpass5", - except = "opt_hir_owner_nodes, generics_of, type_of, predicates_of" + except = "opt_hir_owner_nodes, generics_of, type_of, predicates_of, optimized_mir" )] #[rustc_clean(cfg = "bpass6")] pub fn type_parameter() {} diff --git a/tests/incremental/hashes/inherent_impls.rs b/tests/incremental/hashes/inherent_impls.rs index da3fb1b2ee5fe..9f173683b52cf 100644 --- a/tests/incremental/hashes/inherent_impls.rs +++ b/tests/incremental/hashes/inherent_impls.rs @@ -466,12 +466,12 @@ impl Foo { // ------------------------------------------------- // ----------- // -------------- - // ---------------------------------------------------------------- + // ------------------------------------------------------------------------------ // // ------------------------- // ----------- // -------------- - // ---------------------------------------------------------------- + // ------------------------------------------------------------------------------ // // ------------------------- pub fn add_type_parameter_to_method (&self) { } @@ -494,12 +494,12 @@ impl Foo { // appear dirty, that might be the cause. -nmatsakis #[rustc_clean( cfg="bpass2", - except="opt_hir_owner_nodes,generics_of,predicates_of,type_of", + except="opt_hir_owner_nodes,generics_of,predicates_of,type_of,optimized_mir", )] #[rustc_clean(cfg="bpass3")] #[rustc_clean( cfg="bpass5", - except="opt_hir_owner_nodes,generics_of,predicates_of,type_of", + except="opt_hir_owner_nodes,generics_of,predicates_of,type_of,optimized_mir", )] #[rustc_clean(cfg="bpass6")] pub fn add_type_parameter_to_method(&self) { } diff --git a/tests/mir-opt/const_prop/trivial_const.rs b/tests/mir-opt/const_prop/trivial_const.rs index e9134c1ebdeff..1753b989b78fd 100644 --- a/tests/mir-opt/const_prop/trivial_const.rs +++ b/tests/mir-opt/const_prop/trivial_const.rs @@ -1,5 +1,5 @@ //@ test-mir-pass: SimplifyConstCondition-after-inst-simplify -//@ compile-flags: -Zmir-enable-passes=+InstSimplify-after-simplifycfg -Zub_checks=false -Zinline-mir +//@ compile-flags: -Zmir-enable-passes=+SimplifyUbChecks-after-simplifycfg -Zub_checks=false -Zinline-mir #![crate_type = "lib"] diff --git a/tests/mir-opt/instsimplify/ub_check.rs b/tests/mir-opt/instsimplify/ub_check.rs index db890c1840798..5b678d1280dcd 100644 --- a/tests/mir-opt/instsimplify/ub_check.rs +++ b/tests/mir-opt/instsimplify/ub_check.rs @@ -1,7 +1,7 @@ -//@ test-mir-pass: InstSimplify-after-simplifycfg +//@ test-mir-pass: SimplifyUbChecks-after-simplifycfg //@ compile-flags: -Cdebug-assertions=no -Zinline-mir -// EMIT_MIR ub_check.unwrap_unchecked.InstSimplify-after-simplifycfg.diff +// EMIT_MIR ub_check.unwrap_unchecked.SimplifyUbChecks-after-simplifycfg.diff pub fn unwrap_unchecked(x: Option) -> i32 { // CHECK-LABEL: fn unwrap_unchecked( // CHECK-NOT: UbChecks() diff --git a/tests/mir-opt/instsimplify/ub_check.unwrap_unchecked.InstSimplify-after-simplifycfg.diff b/tests/mir-opt/instsimplify/ub_check.unwrap_unchecked.SimplifyUbChecks-after-simplifycfg.diff similarity index 89% rename from tests/mir-opt/instsimplify/ub_check.unwrap_unchecked.InstSimplify-after-simplifycfg.diff rename to tests/mir-opt/instsimplify/ub_check.unwrap_unchecked.SimplifyUbChecks-after-simplifycfg.diff index c4fa526a7af99..235b04744ddae 100644 --- a/tests/mir-opt/instsimplify/ub_check.unwrap_unchecked.InstSimplify-after-simplifycfg.diff +++ b/tests/mir-opt/instsimplify/ub_check.unwrap_unchecked.SimplifyUbChecks-after-simplifycfg.diff @@ -1,5 +1,5 @@ -- // MIR for `unwrap_unchecked` before InstSimplify-after-simplifycfg -+ // MIR for `unwrap_unchecked` after InstSimplify-after-simplifycfg +- // MIR for `unwrap_unchecked` before SimplifyUbChecks-after-simplifycfg ++ // MIR for `unwrap_unchecked` after SimplifyUbChecks-after-simplifycfg fn unwrap_unchecked(_1: Option) -> i32 { debug x => _1; diff --git a/tests/ui/codegen/normalization-overflow/recursion-issue-117696-1.rs b/tests/ui/codegen/normalization-overflow/recursion-issue-117696-1.rs index 2109afbd84237..da6b8ba6564fd 100644 --- a/tests/ui/codegen/normalization-overflow/recursion-issue-117696-1.rs +++ b/tests/ui/codegen/normalization-overflow/recursion-issue-117696-1.rs @@ -23,8 +23,8 @@ where { if () == () { T::count(it); - //~^ ERROR: reached the recursion limit while instantiating } else { rec(identity(&mut it)) + //~^ ERROR: reached the recursion limit while instantiating } } diff --git a/tests/ui/codegen/normalization-overflow/recursion-issue-117696-1.stderr b/tests/ui/codegen/normalization-overflow/recursion-issue-117696-1.stderr index b261530c010d5..74fa052379022 100644 --- a/tests/ui/codegen/normalization-overflow/recursion-issue-117696-1.stderr +++ b/tests/ui/codegen/normalization-overflow/recursion-issue-117696-1.stderr @@ -1,11 +1,16 @@ -error: reached the recursion limit while instantiating `<&mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut Empty as Iterator>::count` - --> $DIR/recursion-issue-117696-1.rs:25:9 +error: reached the recursion limit while instantiating `rec::<&mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut Empty>` + --> $DIR/recursion-issue-117696-1.rs:27:9 | -LL | T::count(it); - | ^^^^^^^^^^^^ +LL | rec(identity(&mut it)) + | ^^^^^^^^^^^^^^^^^^^^^^ | -note: `count` defined here - --> $SRC_DIR/core/src/iter/traits/iterator.rs:LL:COL +note: `rec` defined here + --> $DIR/recursion-issue-117696-1.rs:20:1 + | +LL | / fn rec(mut it: T) +LL | | where +LL | | T: Iterator, + | |________________^ error: aborting due to 1 previous error diff --git a/tests/ui/mir/validate/critical-edge.rs b/tests/ui/mir/validate/critical-edge.rs index 7fe3891d6424e..98bdc4cbec4ca 100644 --- a/tests/ui/mir/validate/critical-edge.rs +++ b/tests/ui/mir/validate/critical-edge.rs @@ -1,15 +1,14 @@ // Optimized MIR shouldn't have critical call edges // -//@ build-fail //@ edition: 2021 -//@ compile-flags: --crate-type=lib +//@ compile-flags: -Zvalidate-mir //@ failure-status: 101 //@ dont-check-compiler-stderr #![feature(custom_mir, core_intrinsics)] -use core::intrinsics::mir::*; +use std::intrinsics::mir::*; -#[custom_mir(dialect = "runtime", phase = "optimized")] +#[custom_mir(dialect = "mono", phase = "optimized")] #[inline(always)] pub fn f(a: u32) -> u32 { mir! { @@ -31,3 +30,7 @@ pub fn f(a: u32) -> u32 { } } } + +fn main() { + f(5); +}