CI: Retry powershell commands on transient AccessViolationExceptions

flowey/flowey_lib_hvlite/src/_jobs/consume_and_test_nextest_vmm_tests_archive.rs

@@ -77,6 +77,7 @@ impl SimpleFlowNode for Node {
         ctx.import::<crate::artifact_openhcl_igvm_from_recipe::resolve::Node>();
         ctx.import::<crate::download_openvmm_vmm_tests_artifacts::Node>();
         ctx.import::<crate::download_release_igvm_files_from_gh::resolve::Node>();
+        ctx.import::<crate::harden_powershell_event_log::Node>();
         ctx.import::<crate::init_openvmm_magicpath_uefi_mu_msvm::Node>();
         ctx.import::<crate::install_vmm_tests_deps::Node>();
         ctx.import::<crate::init_vmm_tests_env::Node>();
@@ -168,6 +169,15 @@ impl SimpleFlowNode for Node {
 
         let mut pre_run_deps = vec![ctx.reqv(crate::install_vmm_tests_deps::Request::Install)];
 
+        // On Windows CI runners, disable Windows PowerShell engine event
+        // logging before running tests. This avoids transient
+        // AccessViolationException crashes in powershell.exe startup that
+        // surface as `exit code 0xDEAD` from petri's Hyper-V cmdlet calls.
+        if matches!(ctx.platform(), FlowPlatform::Windows) {
+            pre_run_deps
+                .push(ctx.reqv(|done| crate::harden_powershell_event_log::Request { done }));
+        }
+
         let (test_log_path, get_test_log_path) = ctx.new_var();
 
         let extra_env = ctx.reqv(|v| crate::init_vmm_tests_env::Request {

flowey/flowey_lib_hvlite/src/harden_powershell_event_log.rs

@@ -0,0 +1,92 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! Harden the Windows CI runner against transient PowerShell startup crashes.
+//!
+//! Windows PowerShell 5.1 (`powershell.exe`) initializes an
+//! `EventLogLogProvider` during session startup that writes to the
+//! `Windows PowerShell` Event Log source. On some CI runners the source is
+//! either unregistered or has corrupted ACLs, which causes
+//! `EventLog.SourceExists` / `EventLog.WriteEvent` to throw
+//! `System.AccessViolationException` *before* any user command is dispatched.
+//!
+//! This node disables Windows PowerShell engine event logging via the
+//! per-machine registry switch documented for the PowerShell engine. Once
+//! disabled, `EventLogLogProvider.LogEvent` becomes a no-op and the AV no
+//! longer occurs. The change is local to the runner and has no effect on
+//! production hosts.
+
+use flowey::node::prelude::*;
+
+flowey_request! {
+    pub struct Request {
+        /// Completion indicator.
+        pub done: WriteVar<SideEffect>,
+    }
+}
+
+new_simple_flow_node!(struct Node);
+
+impl SimpleFlowNode for Node {
+    type Request = Request;
+
+    fn imports(_ctx: &mut ImportCtx<'_>) {}
+
+    fn process_request(request: Self::Request, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> {
+        let Request { done } = request;
+
+        // Only meaningful on Windows CI runners.
+        if !matches!(ctx.platform(), FlowPlatform::Windows) {
+            ctx.emit_side_effect_step([], [done]);
+            return Ok(());
+        }
+
+        ctx.emit_rust_step("harden Windows PowerShell event log", |ctx| {
+            done.claim(ctx);
+            move |rt| {
+                if matches!(rt.backend(), FlowBackend::Local) {
+                    // Don't tamper with a developer's local machine.
+                    return Ok(());
+                }
+
+                // Disable Windows PowerShell engine event logging by writing
+                // the per-machine registry switch directly via `reg.exe`. We
+                // intentionally avoid `powershell.exe` here because the whole
+                // point of this step is to harden against PowerShell startup
+                // crashes -- using PowerShell to apply the mitigation would
+                // re-introduce the very flake we're trying to avoid.
+                //
+                // `reg.exe add ... /f` creates the key if it doesn't already
+                // exist and overwrites the value if it does. Failures are
+                // logged but non-fatal: the worst case is we fall back to the
+                // original (occasionally flaky) behavior.
+                let output = std::process::Command::new("reg.exe")
+                    .args([
+                        "add",
+                        r"HKLM\Software\Microsoft\PowerShell\1\PowerShellEngine",
+                        "/v",
+                        "EnableEventLogging",
+                        "/t",
+                        "REG_DWORD",
+                        "/d",
+                        "0",
+                        "/f",
+                    ])
+                    .output()?;
+
+                if !output.status.success() {
+                    log::warn!(
+                        "failed to harden PowerShell event log (continuing): {}",
+                        String::from_utf8_lossy(&output.stderr)
+                    );
+                } else {
+                    log::info!("disabled Windows PowerShell engine event logging on this runner");
+                }
+
+                Ok(())
+            }
+        });
+
+        Ok(())
+    }
+}

flowey/flowey_lib_hvlite/src/lib.rs

@@ -42,6 +42,7 @@ pub mod download_openvmm_vmm_tests_artifacts;
 pub mod download_release_igvm_files_from_gh;
 pub mod download_uefi_mu_msvm;
 pub mod git_checkout_openvmm_repo;
+pub mod harden_powershell_event_log;
 pub mod init_cross_build;
 pub mod init_openvmm_cargo_config_deny_warnings;
 pub mod init_openvmm_magicpath_openhcl_sysroot;