diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 6be89c9b..d9e5c7c7 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -75,7 +75,7 @@ jobs: pre_test: powershell -file .\bin\process_monitor.Tests\win-x64\Install-eBpfForWindows.ps1 1.4.0 test_command: .\ntosebpfext_unit.exe -d yes build_artifact: Build-x64 - environment: windows-2025 + environment: '["windows-2025"]' code_coverage: true gather_dumps: true capture_etw: true @@ -92,42 +92,52 @@ jobs: pre_test: powershell -file .\bin\process_monitor.Tests\win-x64\Install-eBpfForWindows.ps1 1.4.0 && powershell -file .\bin\process_monitor.Tests\win-x64\Setup-ProcessMonitorTests.ps1 -ArtifactsRoot . test_command: dotnet test .\bin\process_monitor.Tests\win-x64\process_monitor.Tests.dll build_artifact: Build-x64 - environment: windows-2025 + environment: '["windows-2025"]' capture_etw: true - # Run the neteventebpfext driver tests in GitHub. + # Run the neteventebpfext driver tests on 1ES runners. neteventebpfext_driver_tests: # Always run this job. needs: regular - if: github.event_name == 'schedule' || github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' + if: github.repository == 'microsoft/ntosebpfext' && (github.event_name == 'schedule' || github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch') uses: ./.github/workflows/reusable-test.yml + strategy: + matrix: + image: + - server2022 + - server2025 with: - name: neteventebpfext driver tests - pre_test: powershell -file .\bin\process_monitor.Tests\win-x64\Install-eBpfForWindows.ps1 1.4.0 - test_command: .\neteventebpfext_driver_test.exe -d yes + name: neteventebpfext_driver_${{ matrix.image }} + pre_test: .\setup_ebpf_cicd_tests.ps1 -KmTracing $false + test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "CI/CD" -Options @("neteventebpfext") + post_test: .\cleanup_ebpf_cicd_tests.ps1 -KmTracing $false build_artifact: Build-x64 - environment: windows-2025 - code_coverage: true - gather_dumps: true - capture_etw: true - leak_detection: true + environment: '["self-hosted", "1ES.Pool=ebpf-extensions-cicd-runner-pool", "1ES.ImageOverride=${{ matrix.image }}_v2", "JobId=netevent-driver-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}-${{ strategy.job-index }}-{0}"]' + code_coverage: false + gather_dumps: false + capture_etw: false - # Run the ntosebpfext driver tests in GitHub. + # Run the ntosebpfext driver tests on 1ES runners. ntosebpfext_driver_tests: # Always run this job. needs: regular - if: github.event_name == 'schedule' || github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' + if: github.repository == 'microsoft/ntosebpfext' && (github.event_name == 'schedule' || github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch') uses: ./.github/workflows/reusable-test.yml + strategy: + matrix: + image: + - server2022 + - server2025 with: - name: ntosebpfext driver tests - pre_test: powershell -file .\bin\process_monitor.Tests\win-x64\Install-eBpfForWindows.ps1 1.4.0 - test_command: .\ntosebpfext_driver_test.exe -d yes + name: ntosebpfext_driver_${{ matrix.image }} + pre_test: .\setup_ebpf_cicd_tests.ps1 -KmTracing $false + test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "CI/CD" -Options @("ntosebpfext") + post_test: .\cleanup_ebpf_cicd_tests.ps1 -KmTracing $false build_artifact: Build-x64 - environment: windows-2025 - code_coverage: true - gather_dumps: true - capture_etw: true - leak_detection: true + environment: '["self-hosted", "1ES.Pool=ebpf-extensions-cicd-runner-pool", "1ES.ImageOverride=${{ matrix.image }}_v2", "JobId=ntos-driver-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}-${{ strategy.job-index }}-{0}"]' + code_coverage: false + gather_dumps: false + capture_etw: false ossar: # Always run this job. diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml index 8b1b363d..e9f623d9 100644 --- a/.github/workflows/reusable-build.yml +++ b/.github/workflows/reusable-build.yml @@ -227,6 +227,45 @@ jobs: working-directory: ${{env.GITHUB_WORKSPACE}} run: msbuild /m /p:Configuration=${{env.BUILD_CONFIGURATION}} /p:Platform=${{env.BUILD_PLATFORM}} /bl:${{env.BUILD_PLATFORM}}_${{env.BUILD_CONFIGURATION}}\build_logs\build.binlog ${{env.BUILD_OPTIONS}} ${{env.SOLUTION_FILE_PATH}} + - name: Add eBPF for Windows installer to VM test payload + if: steps.skip_check.outputs.should_skip != 'true' && inputs.build_artifact == 'Build-x64' + working-directory: ${{github.workspace}} + run: | + .\scripts\Install-eBpfForWindows.ps1 ` + -version 1.4.0 ` + -DestinationPath "${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}}\ebpf-for-windows.msi" ` + -DownloadOnly + + - name: Verify VM test payload + if: steps.skip_check.outputs.should_skip != 'true' && inputs.build_artifact == 'Build-x64' + working-directory: ${{github.workspace}} + run: | + $output = "${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}}" + $requiredFiles = @( + "ntosebpfext_driver_test.exe", + "neteventebpfext_driver_test.exe", + "ntosebpfext.sys", + "neteventebpfext.sys", + "netevent_sim.sys", + "NtosEbpfExt.inf", + "NetEventEbpfExt.inf", + "ebpf-for-windows.msi", + "common.psm1", + "config_test_vm.psm1", + "tracing_utils.psm1", + "setup_ebpf_cicd_tests.ps1", + "execute_ebpf_cicd_tests.ps1", + "cleanup_ebpf_cicd_tests.ps1", + "install_ebpf.psm1", + "run_driver_tests.psm1", + "vm_run_tests.psm1", + "test_execution.json" + ) + $missingFiles = $requiredFiles | Where-Object { -not (Test-Path (Join-Path $output $_)) } + if ($missingFiles) { + throw "Missing required VM test payload files: $($missingFiles -join ', ')" + } + - name: Zip Build Output if: always() && (steps.skip_check.outputs.should_skip != 'true') working-directory: ${{github.workspace}} diff --git a/.github/workflows/reusable-test.yml b/.github/workflows/reusable-test.yml index 18aa5dc2..3f46b5e7 100644 --- a/.github/workflows/reusable-test.yml +++ b/.github/workflows/reusable-test.yml @@ -63,13 +63,13 @@ permissions: jobs: run_test: - # Due to app-verifier, unit tests take a lot longer to execute. Increase the timeout to 90 minutes. - timeout-minutes: 90 + # VM-backed driver tests can include VM recovery and dump collection. + timeout-minutes: 150 strategy: matrix: configurations: ${{ fromJSON(inputs.configurations) }} - runs-on: ${{inputs.environment}} + runs-on: ${{ fromJSON(format(inputs.environment, strategy.job-index)) }} env: # Configuration type to build. SOURCE_ROOT: ${{github.workspace}} @@ -101,11 +101,16 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # Check out source code if code coverage is being gathered or if crash dumps # (which require ProcDump downloaded via the repo scripts) are being gathered. - if: ((inputs.code_coverage == true) || (inputs.gather_dumps == true)) && (steps.skip_check.outputs.should_skip != 'true') + if: ((inputs.code_coverage == true) || (inputs.gather_dumps == true)) && !contains(inputs.environment, '1ES') && (steps.skip_check.outputs.should_skip != 'true') with: submodules: 'recursive' ref: ${{ github.event.workflow_run.head_branch }} + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + if: contains(inputs.environment, '1ES') && (steps.skip_check.outputs.should_skip != 'true') + with: + ref: ${{ github.event.workflow_run.head_branch }} + - name: Install ProcDump id: install_procdump if: (inputs.gather_dumps == true) && (steps.skip_check.outputs.should_skip != 'true') @@ -143,7 +148,7 @@ jobs: - name: Set up OpenCppCoverage and add to PATH id: set_up_opencppcoverage - if: (inputs.code_coverage == true) && (inputs.environment != 'ebpf_cicd_tests_ws2019' && inputs.environment != 'ebpf_cicd_tests_ws2022' && inputs.environment != 'ebpf_cicd_perf_ws2022') && (steps.skip_check.outputs.should_skip != 'true') + if: (inputs.code_coverage == true) && !contains(inputs.environment, '1ES') && (inputs.environment != 'ebpf_cicd_tests_ws2019' && inputs.environment != 'ebpf_cicd_tests_ws2022' && inputs.environment != 'ebpf_cicd_perf_ws2022') && (steps.skip_check.outputs.should_skip != 'true') run: | $curlDependencies = Get-Content -Raw "${{github.workspace}}\.github\curl-dependencies.json" | ConvertFrom-Json $version = $curlDependencies.opencppcoverage.version @@ -176,7 +181,7 @@ jobs: New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" -Name "DumpFolder" -Value "$dump_path" -PropertyType ExpandString -ErrorAction SilentlyContinue - name: Remove existing artifacts - if: (inputs.environment == 'ebpf_cicd_tests_ws2019' || inputs.environment == 'ebpf_cicd_tests_ws2022' || inputs.environment == 'ebpf_cicd_perf_ws2022') && (steps.skip_check.outputs.should_skip != 'true') + if: (contains(inputs.environment, 'ebpf_cicd') || contains(inputs.environment, '1ES')) && (steps.skip_check.outputs.should_skip != 'true') run: | Remove-Item -Path ${{github.workspace}}\${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}} -Recurse -Force -ErrorAction SilentlyContinue @@ -221,12 +226,75 @@ jobs: .\netevent_ebpf_ext_export_program_info.exe - name: Run pre test command - if: steps.skip_check.outputs.should_skip != 'true' && (inputs.environment != 'ebpf_cicd_tests_ws2019' && inputs.environment != 'ebpf_cicd_tests_ws2022' && inputs.environment != 'ebpf_cicd_perf_ws2022') + if: steps.skip_check.outputs.should_skip != 'true' && !contains(inputs.environment, '1ES') && (inputs.environment != 'ebpf_cicd_tests_ws2019' && inputs.environment != 'ebpf_cicd_tests_ws2022' && inputs.environment != 'ebpf_cicd_perf_ws2022') id: run_pre_test_command working-directory: ./${{env.BUILD_PLATFORM}}/${{env.BUILD_CONFIGURATION}} run: | ${{env.PRE_COMMAND}} + - name: Diagnose nested Hyper-V + if: steps.skip_check.outputs.should_skip != 'true' && contains(inputs.environment, '1ES') + continue-on-error: true + run: | + $ErrorActionPreference = 'Continue' + + Get-ComputerInfo | + Select-Object CsName, WindowsProductName, OsBuildNumber, HyperVisorPresent | + Format-List + + Get-CimInstance Win32_Processor | + Select-Object Name, VirtualizationFirmwareEnabled, VMMonitorModeExtensions, SecondLevelAddressTranslationExtensions | + Format-List + + Get-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V-All | + Format-List FeatureName, State + Get-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V-Management-PowerShell | + Format-List FeatureName, State + + Get-Service vmms, vmcompute -ErrorAction Continue | + Format-Table Name, Status, StartType + + Get-CimInstance Win32_SystemDriver -Filter "Name = 'Vid'" | + Format-List Name, State, Status, StartMode, PathName + + bcdedit /enum '{current}' | + Select-String hypervisorlaunchtype + + Get-VMHost | Format-List * + Get-VM -Name runner_vm | Format-List * + Get-VMProcessor -VMName runner_vm | Format-List * + Get-VMSnapshot -VMName runner_vm | Format-Table Name, SnapshotType + + - name: Run pre test command on 1ES runner + if: steps.skip_check.outputs.should_skip != 'true' && contains(inputs.environment, '1ES') + id: run_pre_test_command_1es + working-directory: ./${{env.BUILD_PLATFORM}}/${{env.BUILD_CONFIGURATION}} + run: | + ${{env.PRE_COMMAND}} -LogFileName ${{ runner.name }}.log -SelfHostedRunnerName '1ESRunner' -RegressionArtifactsConfiguration ${{env.BUILD_CONFIGURATION}} + + - name: Diagnose nested Hyper-V failure + if: failure() && contains(inputs.environment, '1ES') && steps.run_pre_test_command_1es.outcome == 'failure' + continue-on-error: true + run: | + $ErrorActionPreference = 'Continue' + $startTime = (Get-Date).AddMinutes(-15) + $logs = @( + 'Microsoft-Windows-Hyper-V-Hypervisor-Admin', + 'Microsoft-Windows-Hyper-V-VMMS-Admin', + 'Microsoft-Windows-Hyper-V-Worker-Admin', + 'System' + ) + + foreach ($log in $logs) { + Write-Host "`n=== $log ===" + Get-WinEvent -FilterHashtable @{ + LogName = $log + StartTime = $startTime + } -ErrorAction Continue | + Select-Object -First 50 TimeCreated, Id, LevelDisplayName, ProviderName, Message | + Format-List + } + - name: Run test with Code Coverage and low resource simulation if: (inputs.code_coverage == true) && (inputs.fault_injection == true) && (steps.skip_check.outputs.should_skip != 'true') id: run_test_with_code_coverage_in_fault_injection @@ -253,7 +321,7 @@ jobs: OpenCppCoverage.exe -q --sources %CD% --excluded_sources %CD%\external\Catch2 --export_type cobertura:ebpf_for_windows.xml --working_dir ${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}} -- powershell .\Run-Test.ps1 ${{env.DUMP_PATH}} ${{env.TEST_TIMEOUT}} ${{env.TEST_COMMAND}} - name: Run test without codecoverage - if: (inputs.code_coverage != true) && (inputs.vs_dev != true) && (inputs.fault_injection != true) && (steps.skip_check.outputs.should_skip != 'true') + if: (inputs.code_coverage != true) && (inputs.vs_dev != true) && (inputs.fault_injection != true) && !contains(inputs.environment, '1ES') && (steps.skip_check.outputs.should_skip != 'true') working-directory: ./${{env.BUILD_PLATFORM}}/${{env.BUILD_CONFIGURATION}} id: run_test_without_code_coverage shell: cmd @@ -261,14 +329,28 @@ jobs: set EBPF_ENABLE_WER_REPORT=yes ${{env.TEST_COMMAND}} + - name: Run test on 1ES runner + if: (inputs.code_coverage == false) && (steps.skip_check.outputs.should_skip != 'true') && contains(inputs.environment, '1ES') && (inputs.fault_injection != true) + id: run_test_1es + working-directory: ./${{env.BUILD_PLATFORM}}/${{env.BUILD_CONFIGURATION}} + run: | + ${{env.TEST_COMMAND}} -LogFileName ${{ runner.name }}.log -SelfHostedRunnerName '1ESRunner' -TestJobTimeout ${{env.TEST_TIMEOUT}} + - name: Run post test command # Run the post test command even if the workflow has failed. - if: (success() || failure()) && (steps.skip_check.outputs.should_skip != 'true') && (inputs.environment != 'ebpf_cicd_tests_ws2019' && inputs.environment != 'ebpf_cicd_tests_ws2022' && inputs.environment != 'ebpf_cicd_perf_ws2022') + if: (success() || failure()) && (steps.skip_check.outputs.should_skip != 'true') && !contains(inputs.environment, '1ES') && (inputs.environment != 'ebpf_cicd_tests_ws2019' && inputs.environment != 'ebpf_cicd_tests_ws2022' && inputs.environment != 'ebpf_cicd_perf_ws2022') id: run_post_test_command working-directory: ./${{env.BUILD_PLATFORM}}/${{env.BUILD_CONFIGURATION}} run: | ${{env.POST_COMMAND}} + - name: Run post test command on 1ES runner + if: (success() || failure()) && (steps.skip_check.outputs.should_skip != 'true') && contains(inputs.environment, '1ES') + id: run_post_test_command_1es + working-directory: ./${{env.BUILD_PLATFORM}}/${{env.BUILD_CONFIGURATION}} + run: | + ${{env.POST_COMMAND}} -LogFileName ${{ runner.name }}.log -SelfHostedRunnerName '1ESRunner' + - name: Check for CodeCoverage if: steps.skip_check.outputs.should_skip != 'true' uses: andstor/file-existence-action@558493d6c74bf472d87c84eab196434afc2fa029 @@ -335,10 +417,10 @@ jobs: - name: Fail if code coverage upload fails if: (steps.skip_check.outputs.should_skip != 'true') run: | - if ($${{ steps.test.upload_code_coverage_report_1!='failure' }}) { exit 0 } - if ($${{ steps.test.upload_code_coverage_report_2!='failure' }}) { exit 0 } - if ($${{ steps.test.upload_code_coverage_report_3!='failure' }}) { exit 0 } - if ($${{ steps.test.upload_code_coverage_report_4!='failure' }}) { exit 0 } + if ($${{ steps.upload_code_coverage_report_1.outcome != 'failure' }}) { exit 0 } + if ($${{ steps.upload_code_coverage_report_2.outcome != 'failure' }}) { exit 0 } + if ($${{ steps.upload_code_coverage_report_3.outcome != 'failure' }}) { exit 0 } + if ($${{ steps.upload_code_coverage_report_4.outcome != 'failure' }}) { exit 0 } exit 1 - name: Stop ETW tracing @@ -385,6 +467,14 @@ jobs: path: ./${{env.BUILD_PLATFORM}}/${{env.BUILD_CONFIGURATION}}/TestLogs retention-days: 5 + - name: Check for generated artifacts + # Check for artifacts even if the workflow failed. + if: (success() || failure()) && (steps.skip_check.outputs.should_skip != 'true') + uses: andstor/file-existence-action@558493d6c74bf472d87c84eab196434afc2fa029 + id: check_artifacts + with: + files: ${{github.workspace}}\${{env.BUILD_PLATFORM}}\${{env.BUILD_CONFIGURATION}}\Artifacts\* + - name: Upload generated artifacts # Upload artifacts even if the workflow failed. if: (success() || failure()) && (steps.skip_check.outputs.should_skip != 'true') && (steps.check_artifacts.outputs.files_exists == 'true') diff --git a/scripts/Install-eBpfForWindows.ps1 b/scripts/Install-eBpfForWindows.ps1 index 58a19f64..871f83f6 100644 --- a/scripts/Install-eBpfForWindows.ps1 +++ b/scripts/Install-eBpfForWindows.ps1 @@ -3,7 +3,12 @@ # Define one parameter that takes the version of eBPF for Windows to install param( - [string]$version + [Parameter(Mandatory = $true)] + [string]$version, + + [string]$DestinationPath = "$env:TEMP\ebpf-for-windows.$version.msi", + + [switch]$DownloadOnly ) # Define the URL to download the eBPF for Windows installer @@ -11,10 +16,21 @@ $installer_url = "https://github.com/microsoft/ebpf-for-windows/releases/downloa $installer_url = $installer_url -replace "%%VER%%", $version # Define the path to download the eBPF for Windows installer -$installer_path = "$env:TEMP\ebpf-for-windows.$version.msi" +$installer_path = $DestinationPath # Download the eBPF for Windows installer Invoke-WebRequest -Uri $installer_url -OutFile $installer_path +if (-not (Test-Path $installer_path -PathType Leaf)) { + throw "Failed to download the eBPF for Windows installer to '$installer_path'." +} + +if ($DownloadOnly) { + return +} + # Install eBPF for Windows -Start-Process -FilePath msiexec -ArgumentList "/i $installer_path /quiet" -Wait +$process = Start-Process -FilePath msiexec -ArgumentList "/i `"$installer_path`" /quiet /norestart" -Wait -PassThru +if ($process.ExitCode -notin @(0, 3010)) { + throw "eBPF for Windows installation failed with exit code $($process.ExitCode)." +} diff --git a/scripts/cleanup_ebpf_cicd_tests.ps1 b/scripts/cleanup_ebpf_cicd_tests.ps1 new file mode 100644 index 00000000..fd189d2c --- /dev/null +++ b/scripts/cleanup_ebpf_cicd_tests.ps1 @@ -0,0 +1,119 @@ +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: MIT + +param ([parameter(Mandatory=$false)][bool] $KmTracing = $true, + [parameter(Mandatory=$false)][string] $LogFileName = "TestLog.log", + [parameter(Mandatory=$false)][string] $WorkingDirectory = $pwd.ToString(), + [parameter(Mandatory=$false)][string] $TestExecutionJsonFileName = "test_execution.json", + [parameter(Mandatory=$false)][string] $SelfHostedRunnerName = [System.Net.Dns]::GetHostName(), + [Parameter(Mandatory = $false)][int] $TestJobTimeout = (30*60), + [Parameter(Mandatory = $false)][switch] $ExecuteOnHost, + [Parameter(Mandatory = $false)][switch] $VMIsRemote) + +$ExecuteOnHost = [bool]$ExecuteOnHost +$ExecuteOnVM = (-not $ExecuteOnHost) +$VMIsRemote = [bool]$VMIsRemote + +Push-Location $WorkingDirectory + +Import-Module .\common.psm1 -Force -ArgumentList ($LogFileName) -WarningAction SilentlyContinue + +# Read the test execution json. +$Config = Get-Content ("{0}\{1}" -f $PSScriptRoot, $TestExecutionJsonFileName) | ConvertFrom-Json + +$Job = Start-Job -ScriptBlock { + param ([Parameter(Mandatory = $True)] [bool] $ExecuteOnHost, + [Parameter(Mandatory = $True)] [bool] $ExecuteOnVM, + [Parameter(Mandatory = $True)] [bool] $VMIsRemote, + [Parameter(Mandatory = $true)] [PSCustomObject] $Config, + [Parameter(Mandatory = $true)] [string] $SelfHostedRunnerName, + [parameter(Mandatory = $true)] [string] $LogFileName, + [parameter(Mandatory = $true)] [string] $WorkingDirectory = $pwd.ToString(), + [parameter(Mandatory = $true)] [bool] $KmTracing + ) + Push-Location $WorkingDirectory + + # Load other utility modules. + Import-Module .\common.psm1 -Force -ArgumentList ($LogFileName) -WarningAction SilentlyContinue + Import-Module .\config_test_vm.psm1 -Force -ArgumentList ($WorkingDirectory, $LogFileName) -WarningAction SilentlyContinue + + if ($ExecuteOnVM) { + $VMList = $Config.VMMap.$SelfHostedRunnerName + # Wait for all VMs to be in ready state, in case the test run caused any VM to crash. + Wait-AllVMsToInitialize -VMList $VMList -VMIsRemote $VMIsRemote + + # Import logs from VMs. + Import-ResultsFromVM -VMList $VMList -KmTracing $KmTracing + + # Stop the VMs. + Stop-AllVMs -VMList $VMList + } else { + try { + Import-ResultsFromHost -KmTracing $KmTracing + } catch { + Write-Log "Failed to obtain results. Treating as non-fatal error. Error: $_" + } + } + + Pop-Location + +} -ArgumentList ( + $ExecuteOnHost, + $ExecuteOnVM, + $VMIsRemote, + $Config, + $SelfHostedRunnerName, + $LogFileName, + $WorkingDirectory, + $KmTracing) + + +# Wait for the job to complete +$JobTimedOut = ` + Wait-TestJobToComplete -Job $Job ` + -Config $Config ` + -SelfHostedRunnerName $SelfHostedRunnerName ` + -TestJobTimeout $TestJobTimeout ` + -CheckpointPrefix "Cleanup" ` + -ExecuteOnHost $ExecuteOnHost ` + -ExecuteOnVM $ExecuteOnVM ` + -VMIsRemote $VMIsRemote ` + -TestWorkingDirectory $(if ($ExecuteOnVM) { "C:\ebpf" } else { $WorkingDirectory }) ` + -LogFileName $LogFileName ` + -TestMode "CI/CD" ` + -Options @("None") ` + -TestHangTimeout (10*60) ` + -UserModeDumpFolder "C:\Dumps" + +# Re-import common.psm1 in case Wait-TestJobToComplete's timeout handler +# forcefully re-imported it (via vm_run_tests.psm1), removing it from this scope. +Import-Module $WorkingDirectory\common.psm1 -Force -ArgumentList ($LogFileName) -ErrorAction SilentlyContinue + +# Check job result before cleanup. +$JobFailed = $Job.State -eq 'Failed' +if ($JobFailed) { + try { + $childJob = $Job.ChildJobs[0] + if ($childJob -and $childJob.JobStateInfo.Reason) { + Write-Log "Cleanup job failed: $($childJob.JobStateInfo.Reason.Message)" + } + } catch { + Write-Log "Warning: Failed to read cleanup job state: $($_.Exception.Message)" + } + try { + Receive-Job -Job $Job -ErrorAction SilentlyContinue + } catch { + Write-Log "Warning: Failed to receive cleanup job output: $($_.Exception.Message)" + } +} + +# Safe cleanup (bounded to prevent hangs on stuck transports). +Remove-JobSafely -Job $Job + +Pop-Location + +if ($JobTimedOut -or $JobFailed) { + if ($JobTimedOut) { Write-Log "Cleanup timed out" } + if ($JobFailed) { Write-Log "Cleanup job failed" } + exit 1 +} \ No newline at end of file diff --git a/scripts/common.psm1 b/scripts/common.psm1 new file mode 100644 index 00000000..9bf586f6 --- /dev/null +++ b/scripts/common.psm1 @@ -0,0 +1,1159 @@ +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: MIT + +param ([parameter(Mandatory=$True)] [string] $LogFileName) + +# +# Common helper functions. +# + + +function Write-Log +{ + [CmdletBinding()] + param([parameter(Mandatory=$False, ValueFromPipeline=$true)]$TraceMessage=$null, + [parameter(Mandatory=$False)]$ForegroundColor = [System.ConsoleColor]::White) + + process + { + if (![System.String]::IsNullOrEmpty($TraceMessage)) { + $timestamp = (Get-Date).ToString('HH:mm:ss') + Write-Host "[$timestamp] :: $TraceMessage"-ForegroundColor $ForegroundColor + Write-Output "[$timestamp] :: $TraceMessage" | Out-File "$env:TEMP\$LogFileName" -Append + } + } +} + +function ThrowWithErrorMessage +{ + Param( + [Parameter(Mandatory = $True)] [string] $ErrorMessage + ) + + Write-Log $ErrorMessage -ForegroundColor Red + Start-Sleep -Milliseconds 100 + throw $ErrorMessage +} + +function Start-ProcessWithTimeout +{ + param( + [Parameter(Mandatory=$true)] [string] $FilePath, + [Parameter(Mandatory=$false)] [string[]] $ArgumentList = @(), + [Parameter(Mandatory=$false)] [int] $TimeoutSeconds = 60 + ) + + # Create temp files for output redirection + $tempOut = [System.IO.Path]::GetTempFileName() + $tempErr = [System.IO.Path]::GetTempFileName() + + try { + Write-Log "Starting process with timeout: $TimeoutSeconds seconds" + Write-Log "Executing: $FilePath $($ArgumentList -join ' ')" + + $process = Start-Process -FilePath $FilePath -ArgumentList $ArgumentList -NoNewWindow -PassThru -RedirectStandardOutput $tempOut -RedirectStandardError $tempErr + $handle = $process.Handle # Important: this ensures we can access the process info later + Write-Log "Process started with ID: $($process.Id)" + + if ($process.WaitForExit($TimeoutSeconds * 1000)) { + $exitCode = $process.ExitCode + Write-Log "Process completed with exit code: $exitCode" + return $exitCode + } else { + Write-Log "Process timed out after $TimeoutSeconds seconds, terminating process" + if (-not $process.HasExited) { + $process.Kill() + $process.WaitForExit(5000) + } + Write-Log "Process terminated due to timeout" + return -1 # Indicate timeout + } + } catch { + Write-Log "Exception running process: $_" -ForegroundColor Red + return -2 # Indicate exception + } finally { + # Clean up temp files + if (Test-Path $tempOut) { Remove-Item $tempOut -Force -ErrorAction SilentlyContinue } + if (Test-Path $tempErr) { Remove-Item $tempErr -Force -ErrorAction SilentlyContinue } + } +} + + +function Compress-File +{ + param ([Parameter(Mandatory = $True)] [string] $SourcePath, + [Parameter(Mandatory = $True)] [string] $DestinationPath + ) + + Write-Log "Compressing $SourcePath -> $DestinationPath" + + # Use System.IO.Compression directly instead of Compress-Archive + # to avoid the 2GB MemoryStream limitation in Compress-Archive (affects files > 2GB). + Add-Type -AssemblyName System.IO.Compression -ErrorAction SilentlyContinue + Add-Type -AssemblyName System.IO.Compression.FileSystem -ErrorAction SilentlyContinue + + # Retry 5 times to ensure compression operation succeeds. + # To mitigate error message: "The process cannot access the file because it is being used by another process." + $retryCount = 1 + while ($retryCount -le 5) { + try { + $ErrorActionPreference = "Stop" + if (Test-Path $DestinationPath) { + Remove-Item $DestinationPath -Force + } + + $zipArchive = [System.IO.Compression.ZipFile]::Open( + $DestinationPath, + [System.IO.Compression.ZipArchiveMode]::Create) + + try { + $sourceFiles = Get-ChildItem -Path $SourcePath -ErrorAction Stop + foreach ($file in $sourceFiles) { + Write-Log "Adding $($file.Name) ($((($file.Length) / 1MB).ToString('F2')) MB) to archive..." + [System.IO.Compression.ZipFileExtensions]::CreateEntryFromFile( + $zipArchive, + $file.FullName, + $file.Name, + [System.IO.Compression.CompressionLevel]::Fastest) | Out-Null + } + } finally { + $zipArchive.Dispose() + } + + # Verify the compressed file was actually created. + if (Test-Path $DestinationPath) { + Write-Log "Compression completed successfully." + return $true + } else { + throw "Compressed file was not created at $DestinationPath" + } + } catch { + $ErrorMessage = "*** ERROR *** Failed to compress files (attempt $retryCount of 5): $($_.Exception.Message)" + Write-Log $ErrorMessage + # Clean up partial zip file on failure. + if (Test-Path $DestinationPath) { + Remove-Item $DestinationPath -Force -ErrorAction Ignore + } + if ($retryCount -lt 5) { + Start-Sleep -seconds (5 * $retryCount) + } + $retryCount++ + } + } + + # All retries failed. + Write-Log "*** ERROR *** Failed to compress after 5 attempts. Compression failed." + return $false +} + +<# +.SYNOPSIS + Attempts to compress files and copies them to a destination, with fallback to uncompressed files. + +.DESCRIPTION + This function standardizes the pattern of trying to compress files and copying the result, + with automatic fallback to copying uncompressed files if compression fails. + +.PARAMETER SourcePath + The source files to compress (supports wildcards like *.dmp). + +.PARAMETER DestinationDirectory + The directory where the final files should be copied. + +.PARAMETER CompressedFileName + The name for the compressed file. If not provided, auto-generates based on source. + +.OUTPUTS + Returns a hashtable with the following properties: + - CompressedCopySucceeded: Boolean indicating if the compressed copy was used ($true) or fell back to uncompressed ($false) + - CompressedPath: Path to compressed file (if compressed copy succeeded) + - UncompressedPath: Path to uncompressed file (if fell back to uncompressed) + - FinalPath: Path to the final file that was copied + +.EXAMPLE + $result = CompressOrCopy-File -SourcePath "C:\dumps\*.dmp" -DestinationDirectory "C:\output" -CompressedFileName "dumps.zip" + if ($result.CompressedCopySucceeded) { + Write-Host "Compression succeeded: $($result.FinalPath)" + } else { + Write-Host "Compression failed, using uncompressed: $($result.FinalPath)" + } +#> +function CompressOrCopy-File +{ + param( + [Parameter(Mandatory=$True)][string] $SourcePath, + [Parameter(Mandatory=$True)][string] $DestinationDirectory, + [Parameter(Mandatory=$False)][string] $CompressedFileName + ) + + # Ensure destination directory exists. + if (-not (Test-Path $DestinationDirectory -PathType Container)) { + New-Item -Path $DestinationDirectory -ItemType Directory -Force | Out-Null + } + + # Auto-generate compressed filename if not provided. + if (-not $CompressedFileName) { + $sourceName = Split-Path $SourcePath -Leaf + $CompressedFileName = "$sourceName.zip" + } + + # Create temporary path for compression. + $tempCompressedPath = Join-Path $env:TEMP $CompressedFileName + $finalCompressedPath = Join-Path $DestinationDirectory $CompressedFileName + + # Attempt compression. + $compressionSucceeded = Compress-File -SourcePath $SourcePath -DestinationPath $tempCompressedPath + + if ($compressionSucceeded -and (Test-Path $tempCompressedPath)) { + # Compression succeeded - copy compressed file. + Copy-Item -Path $tempCompressedPath -Destination $finalCompressedPath -Force + Remove-Item -Path $tempCompressedPath -Force -ErrorAction Ignore + + $compressedFile = Get-ChildItem -Path $finalCompressedPath + Write-Log "Copied compressed file: $($compressedFile.Name), Size: $((($compressedFile.Length) / 1MB).ToString("F2")) MB" + + return @{ + CompressedCopySucceeded = $true + CompressedPath = $finalCompressedPath + UncompressedPath = "" + FinalPath = $finalCompressedPath + } + } else { + # Compression failed - copy uncompressed files as fallback. + Write-Log "*** WARNING *** Compression failed. Copying uncompressed files instead." + + $sourceFiles = Get-ChildItem -Path $SourcePath -ErrorAction SilentlyContinue + $copiedPaths = @() + + foreach ($file in $sourceFiles) { + $destinationPath = Join-Path $DestinationDirectory $file.Name + Copy-Item -Path $file.FullName -Destination $destinationPath -Force + Write-Log "Copied uncompressed file: $($file.Name)" + $copiedPaths += $destinationPath + } + + # Clean up temporary compressed file if it exists. + if (Test-Path $tempCompressedPath) { + Remove-Item -Path $tempCompressedPath -Force -ErrorAction Ignore + } + + return @{ + CompressedCopySucceeded = $false + CompressedPath = "" + UncompressedPath = if ($copiedPaths.Count -gt 0) { $copiedPaths -join "; " } else { "" } + FinalPath = if ($copiedPaths.Count -gt 0) { $copiedPaths -join "; " } else { "" } + } + } +} + +<# +.SYNOPSIS + Copies compressed files from a remote session, with fallback to uncompressed files. + +.DESCRIPTION + This function standardizes the pattern of trying to copy compressed files from a remote session + and falling back to uncompressed files if the compressed copy fails. + +.PARAMETER VMSession + The remote PowerShell session to copy from. + +.PARAMETER CompressedSourcePath + The path to the compressed file on the remote session. + +.PARAMETER UncompressedSourcePath + The path to the uncompressed file on the remote session. + +.PARAMETER DestinationDirectory + The local directory where files should be copied. + +.OUTPUTS + Returns a hashtable with the following properties: + - CompressedCopySucceeded: Boolean indicating if the compressed copy was used ($true) or fell back to uncompressed ($false) + - CompressedPath: Path to compressed file (if compressed copy succeeded) + - UncompressedPath: Path to uncompressed file (if fell back to uncompressed) + - FinalPath: Path to the final file that was copied + +.EXAMPLE + $result = CopyCompressedOrUncompressed-FileFromSession -VMSession $session -CompressedSourcePath "C:\eBPF\trace.zip" -UncompressedSourcePath "C:\eBPF\trace.etl" -DestinationDirectory ".\Logs" + if ($result.CompressedCopySucceeded) { + Write-Host "Compressed copy succeeded: $($result.FinalPath)" + } else { + Write-Host "Using uncompressed fallback: $($result.FinalPath)" + } +#> +function CopyCompressedOrUncompressed-FileFromSession +{ + param( + [Parameter(Mandatory=$True)] $VMSession, + [Parameter(Mandatory=$True)][string] $CompressedSourcePath, + [Parameter(Mandatory=$True)][string] $UncompressedSourcePath, + [Parameter(Mandatory=$True)][string] $DestinationDirectory, + [Parameter(Mandatory=$False)][int] $MaxRetries = 3 + ) + + # Ensure destination directory exists. + if (-not (Test-Path $DestinationDirectory -PathType Container)) { + New-Item -Path $DestinationDirectory -ItemType Directory -Force | Out-Null + } + + $compressedFileName = Split-Path $CompressedSourcePath -Leaf + $uncompressedFileName = Split-Path $UncompressedSourcePath -Leaf + $compressedDestPath = Join-Path $DestinationDirectory $compressedFileName + + # Try to copy compressed file first with retry logic. + Write-Log "Copy $CompressedSourcePath to $DestinationDirectory" + $compressedCopySucceeded = $false + + for ($attempt = 1; $attempt -le $MaxRetries; $attempt++) { + try { + Write-Log "Attempting compressed file copy (attempt $attempt of $MaxRetries)" + $copyResult = Copy-FromSessionWithTimeout ` + -Session $VMSession ` + -Path $CompressedSourcePath ` + -Destination $DestinationDirectory ` + -OperationName "Compressed file copy" ` + -Recurse ` + -CopyErrorAction 'Stop' + + # Check if compressed copy succeeded. + if (Test-Path $compressedDestPath) { + Write-Log "Successfully copied compressed file: $compressedFileName" + $compressedCopySucceeded = $true + break + } + } catch { + Write-Log "Compressed file copy attempt $attempt failed: $($_.Exception.Message)" + if ($attempt -lt $MaxRetries) { + Start-Sleep -Seconds (2 * $attempt) # Progressive delay + } + } + } + + if ($compressedCopySucceeded) { + return @{ + CompressedCopySucceeded = $true + CompressedPath = $compressedDestPath + UncompressedPath = "" + FinalPath = $compressedDestPath + } + } else { + # Compressed copy failed, try uncompressed with retry logic. + Write-Log "Compressed file not found or copy failed, trying uncompressed: Copy $UncompressedSourcePath to $DestinationDirectory" + $uncompressedDestPath = Join-Path $DestinationDirectory $uncompressedFileName + $uncompressedCopySucceeded = $false + + # Handle wildcard paths by expanding them on the remote session first + $sourceFiles = @() + if ($UncompressedSourcePath -like "*\*.*") { + try { + $sourceFiles = Invoke-WithTimeout -OperationName "Expand wildcard path" -TimeoutMs 30000 -ScriptBlock { + param($session, $path) + Invoke-Command -Session $session -ScriptBlock { + param($Path) + Get-ChildItem -Path $Path -ErrorAction SilentlyContinue | ForEach-Object { $_.FullName } + } -ArgumentList $path -ErrorAction SilentlyContinue + } -ArgumentList @($VMSession, $UncompressedSourcePath) + + if (-not $sourceFiles -or $sourceFiles.Count -eq 0) { + Write-Log "No files found matching pattern: $UncompressedSourcePath" + } + } catch { + Write-Log "Failed to expand wildcard path $UncompressedSourcePath : $($_.Exception.Message)" + } + } else { + $sourceFiles = @($UncompressedSourcePath) + } + + # Copy each file individually with retry logic + $anyCopySucceeded = $false + foreach ($sourceFile in $sourceFiles) { + $fileName = Split-Path $sourceFile -Leaf + $destPath = Join-Path $DestinationDirectory $fileName + + for ($attempt = 1; $attempt -le $MaxRetries; $attempt++) { + try { + Write-Log "Attempting to copy $sourceFile (attempt $attempt of $MaxRetries)" + $copyResult = Copy-FromSessionWithTimeout ` + -Session $VMSession ` + -Path $sourceFile ` + -Destination $DestinationDirectory ` + -OperationName "Copy $fileName" ` + -CopyErrorAction 'Stop' + + # Check if copy succeeded. + if (Test-Path $destPath) { + Write-Log "Successfully copied uncompressed file: $fileName" + $anyCopySucceeded = $true + $uncompressedDestPath = $destPath # Update to last successful file + break + } + } catch { + Write-Log "Failed to copy $sourceFile (attempt $attempt): $($_.Exception.Message)" + if ($attempt -lt $MaxRetries) { + Start-Sleep -Seconds (2 * $attempt) # Progressive delay + } + } + } + } + + if (-not $anyCopySucceeded) { + Write-Log "*** WARNING *** Failed to copy any files from $UncompressedSourcePath after $MaxRetries attempts each" + } + + return @{ + CompressedCopySucceeded = $false + CompressedPath = "" + UncompressedPath = $uncompressedDestPath + FinalPath = $uncompressedDestPath + } + } +} + +<# +.SYNOPSIS + Safely stops and removes a PowerShell job with bounded timeouts. +.DESCRIPTION + Wraps Stop-Job and Remove-Job in bounded async calls to prevent hangs + when the job's transport (e.g. VMBus/PS Direct) is stuck. +.PARAMETER Job + The job to stop and remove. +.PARAMETER TimeoutMs + Maximum time in milliseconds to wait for each operation. Default 30000 (30s). +#> +function Remove-JobSafely { + param( + [Parameter(Mandatory=$true)] [System.Management.Automation.Job] $Job, + [Parameter(Mandatory=$false)] [int] $TimeoutMs = 30000 + ) + # Attempt Stop-Job with a timeout (it can hang on stuck transports). + if ($Job.State -eq 'Running') { + $ps = $null + try { + $ps = [powershell]::Create().AddScript({ param($j) Stop-Job -Job $j -ErrorAction SilentlyContinue }).AddArgument($Job) + $async = $ps.BeginInvoke() + if (-not $async.AsyncWaitHandle.WaitOne($TimeoutMs)) { + Write-Log "Warning: Stop-Job timed out after $($TimeoutMs / 1000)s" + } + try { $ps.EndInvoke($async) } catch {} + } catch {} finally { + if ($ps) { $ps.Dispose() } + } + } + # Attempt Remove-Job with a timeout. + $ps = $null + try { + $ps = [powershell]::Create().AddScript({ param($j) Remove-Job -Job $j -Force -ErrorAction SilentlyContinue }).AddArgument($Job) + $async = $ps.BeginInvoke() + if (-not $async.AsyncWaitHandle.WaitOne($TimeoutMs)) { + Write-Log "Warning: Remove-Job timed out after $($TimeoutMs / 1000)s" + } + try { $ps.EndInvoke($async) } catch {} + } catch {} finally { + if ($ps) { $ps.Dispose() } + } +} + +<# +.SYNOPSIS + Runs a script block with a bounded timeout using a separate PowerShell runspace. +.DESCRIPTION + Executes the given script block in a new runspace. If the operation does not complete + within TimeoutMs, it is stopped and $null is returned. This prevents any single + operation (e.g. Copy-Item -FromSession over a stuck PS Direct transport) from + blocking indefinitely. +.PARAMETER ScriptBlock + The script block to execute. +.PARAMETER ArgumentList + Arguments to pass to the script block. +.PARAMETER TimeoutMs + Maximum time in milliseconds to wait. Default 300000 (5 minutes). +.PARAMETER OperationName + Descriptive name for log messages. +#> +function Invoke-WithTimeout { + param( + [Parameter(Mandatory=$true)] [ScriptBlock] $ScriptBlock, + [Parameter(Mandatory=$false)] [object[]] $ArgumentList = @(), + [Parameter(Mandatory=$false)] [int] $TimeoutMs = 300000, + [Parameter(Mandatory=$false)] [string] $OperationName = "Operation" + ) + try { + $ps = [powershell]::Create() + $null = $ps.AddScript($ScriptBlock) + foreach ($arg in $ArgumentList) { + $null = $ps.AddArgument($arg) + } + $async = $ps.BeginInvoke() + if (-not $async.AsyncWaitHandle.WaitOne($TimeoutMs)) { + Write-Log "Warning: $OperationName timed out after $($TimeoutMs / 1000)s" + try { $ps.Stop() } catch {} + return $null + } + return $ps.EndInvoke($async) + } catch { + Write-Log "Warning: $OperationName failed: $($_.Exception.Message)" + return $null + } finally { + if ($ps) { $ps.Dispose() } + } +} + +<# +.SYNOPSIS + Copies a file from a remote PS session with a bounded timeout. +.DESCRIPTION + Wraps Copy-Item -FromSession inside Invoke-WithTimeout to prevent hangs + on stuck PS Direct transports. +.PARAMETER Session + The PSSession to copy from. +.PARAMETER Path + Source path on the remote session. +.PARAMETER Destination + Local destination path. +.PARAMETER OperationName + Descriptive name for log messages. +.PARAMETER TimeoutMs + Maximum time in milliseconds to wait. Default 300000 (5 minutes). +.PARAMETER Recurse + Whether to copy recursively. +.PARAMETER CopyErrorAction + ErrorAction for the inner Copy-Item call. Default 'Ignore'. +#> +function Copy-FromSessionWithTimeout { + param( + [Parameter(Mandatory=$true)] [System.Management.Automation.Runspaces.PSSession] $Session, + [Parameter(Mandatory=$true)] [string] $Path, + [Parameter(Mandatory=$true)] [string] $Destination, + [Parameter(Mandatory=$false)] [string] $OperationName = "Copy file from session", + [Parameter(Mandatory=$false)] [int] $TimeoutMs = 300000, + [Parameter(Mandatory=$false)] [switch] $Recurse, + [Parameter(Mandatory=$false)] [string] $CopyErrorAction = 'Ignore' + ) + Invoke-WithTimeout -OperationName $OperationName -TimeoutMs $TimeoutMs -ScriptBlock { + param($session, $src, $dst, $recurse, $errorAction) + $params = @{ + FromSession = $session + Path = $src + Destination = $dst + Force = $true + ErrorAction = $errorAction + } + if ($recurse) { $params['Recurse'] = $true } + Copy-Item @params + } -ArgumentList @($Session, $Path, $Destination, [bool]$Recurse, $CopyErrorAction) +} + +function Wait-TestJobToComplete +{ + param([Parameter(Mandatory = $true)] [System.Management.Automation.Job] $Job, + [Parameter(Mandatory = $true)] [PSCustomObject] $Config, + [Parameter(Mandatory = $true)] [string] $SelfHostedRunnerName, + [Parameter(Mandatory = $true)] [int] $TestJobTimeout, + [Parameter(Mandatory = $true)] [string] $CheckpointPrefix, + [Parameter(Mandatory = $false)] [bool] $ExecuteOnHost=$false, + [Parameter(Mandatory = $false)] [bool] $ExecuteOnVM=$true, + [Parameter(Mandatory = $false)] [bool] $VMIsRemote=$false, + [Parameter(Mandatory = $false)] [string] $TestWorkingDirectory="C:\ebpf", + [Parameter(Mandatory = $false)] [string] $LogFileName="timeout_kernel_dump.log", + [Parameter(Mandatory = $false)] [string] $TestMode="CI/CD", + [Parameter(Mandatory = $false)] [string[]] $Options=@("None"), + [Parameter(Mandatory = $false)] [int] $TestHangTimeout=(10*60), + [Parameter(Mandatory = $false)] [string] $UserModeDumpFolder="C:\Dumps", + [Parameter(Mandatory = $false)] [bool] $GenerateKernelDumpOnTimeout=$false) + $TimeElapsed = 0 + # Loop to fetch and print job output in near real-time. + while ($Job.State -eq 'Running') { + try { + $JobOutput = Receive-Job -Job $job -ErrorAction SilentlyContinue + $JobOutput | ForEach-Object { Write-Host $_ } + } catch { + Write-Log "Warning: Failed to receive job output (remote session may have ended): $($_.Exception.Message)" + } + + Start-Sleep -Seconds 2 + $TimeElapsed += 2 + + if ($TimeElapsed -gt $TestJobTimeout) { + if ($Job.State -eq "Running") { + Write-Host "Job on $CheckpointPrefix has timed out after $($TestJobTimeout / 60) minutes." -ForegroundColor Yellow + if ($GenerateKernelDumpOnTimeout -and $ExecuteOnVM) { + $VMList = $Config.VMMap.$SelfHostedRunnerName + # Currently one VM runs per runner. + $TestVMName = $VMList[0].Name + + try { + Write-Log "Generating kernel dump due to test timeout on $TestVMName" + + Import-Module "$PSScriptRoot\vm_run_tests.psm1" -ArgumentList @( + $false, # ExecuteOnHost + $true, # ExecuteOnVM + $VMIsRemote, # VMIsRemote + $TestVMName, # VMName + $TestWorkingDirectory, # WorkingDirectory + $LogFileName, # LogFileName + $TestMode, # TestMode + $Options, # Options + $TestHangTimeout, # TestHangTimeout + $UserModeDumpFolder # UserModeDumpFolder + ) -Force -WarningAction SilentlyContinue + + # Generate kernel dump on the VM + Generate-KernelDumpOnVM + } catch { + # Do nothing - this is expected as the VM will crash. + } + + # Restore common.psm1 globally. The Import-Module -Force of + # vm_run_tests.psm1 above causes common.psm1 to be forcefully + # re-imported into that module's scope, which removes it from the + # session-wide module table and breaks callers that previously + # imported it (e.g. execute_ebpf_cicd_tests.ps1 loses Write-Log + # and Remove-JobSafely). + Import-Module "$PSScriptRoot\common.psm1" -Force ` + -ArgumentList ($LogFileName) ` + -WarningAction SilentlyContinue ` + -Scope Global + } + $JobTimedOut = $true + break + } + } + } + + # Print any remaining output after the job completes. + try { + $JobOutput = Receive-Job -Job $job -ErrorAction SilentlyContinue + $JobOutput | ForEach-Object { Write-Host $_ } + } catch { + Write-Log "Warning: Failed to receive final job output (remote session may have ended): $($_.Exception.Message)" + } + + # Check the job's final state and report failures as errors. + if ($Job.State -eq 'Failed') { + $jobError = $Job.ChildJobs[0].JobStateInfo.Reason + if ($jobError) { + Write-Log "*** ERROR *** Job failed with error: $($jobError.Message)" + Write-Log "Stack trace: $($jobError.ScriptStackTrace)" + } else { + Write-Log "*** ERROR *** Job failed with unknown error." + } + return $true # Signal failure to the caller. + } + + return $JobTimedOut +} + +<# +.SYNOPSIS + Helper function to create a directory if it does not already exist. + +.DESCRIPTION + This function checks if a directory exists at the specified path. If it does not exist, it creates the directory. + +.PARAMETER Path + The path of the directory to create. + +.EXAMPLE + Create-DirectoryIfNotExists -Path "C:\MyDirectory" +#> +function Create-DirectoryIfNotExists { + param ( + [Parameter(Mandatory=$True)][string]$Path + ) + + # Create the directory if it does not already exist. + if (-not (Test-Path -Path $Path -PathType Container)) { + New-Item -Path $Path -ItemType Directory -Force + } + + # Check if the directory was successfully created. + if (-not (Test-Path -PathType Container $Path)) { + throw "Failed to create directory: $Path" + } +} + +<# +.SYNOPSIS + Helper function to replace placeholder strings in a file. + +.DESCRIPTION + This function replaces all occurrences of a specified search string with a replacement string in a file. + +.PARAMETER FilePath + The path to the file in which to replace the placeholder strings. + +.PARAMETER SearchString + The string to search for in the file. + +.PARAMETER ReplaceString + The string to replace the search string with. + +.EXAMPLE + Replace-PlaceholderStrings -FilePath "C:\MyFile.txt" -SearchString "PLACEHOLDER" -ReplaceString "ActualValue" +#> +function Replace-PlaceholderStrings { + param ( + [Parameter(Mandatory=$True)][string]$FilePath, + [Parameter(Mandatory=$True)][string]$SearchString, + [Parameter(Mandatory=$True)][string]$ReplaceString + ) + + try { + $content = Get-Content -Path $FilePath + $content = $content -replace $SearchString, $ReplaceString + Set-Content -Path $FilePath -Value $content + } catch { + throw "Failed to replace placeholder strings in file: $FilePath. Error: $_" + } +} + +<# +.SYNOPSIS + Helper function to invoke a script using PsExec. + +.DESCRIPTION + This function uses PsExec to run a PowerShell script in the LocalSystem account context. + +.PARAMETER Script + The script to execute using PsExec. + +.EXAMPLE + Invoke-PsExecScript -Script "Get-Process" +#> +function Invoke-PsExecScript { + param ( + [Parameter(Mandatory=$true)][string]$Script + ) + $PSExecPath = Get-PSExec + if (($null -eq $PSExecPath) -or (-not (Test-Path $PSExecPath))) { + throw "Failed to retrieve PsExec path." + } + + $outputFile = [System.IO.Path]::GetTempFileName() + $errorFile = [System.IO.Path]::GetTempFileName() + + try { + $process = Start-Process -FilePath $PsExecPath -ArgumentList "-accepteula -nobanner -s powershell.exe -command `"$Script`"" -NoNewWindow -PassThru -Wait -RedirectStandardOutput $outputFile -RedirectStandardError $errorFile + $output = Get-Content $outputFile + $err = Get-Content $errorFile + + if ($process.ExitCode -ne 0) { + throw "PsExec failed with exit code $($process.ExitCode). Output: $output Error: $err" + } + + return $output + } finally { + Remove-Item $outputFile -Force -ErrorAction Ignore + Remove-Item $errorFile -Force -ErrorAction Ignore + } +} + +<# +.SYNOPSIS + Returns the well-known password used for inner test VM accounts. + +.DESCRIPTION + Single source of truth for the VM password used by both the Administrator and + VMStandardUser accounts on the inner test VMs. All scripts that need the password + should call this function rather than hardcoding the value. + + A simple well-known password is acceptable here because these are ephemeral nested + test VMs that are not network-accessible outside the host. They are created, used + for CI/CD test execution, and destroyed within a single pipeline run. The password + only needs to meet Windows complexity requirements and remain consistent between + the unattend.xml provisioning (via PLACEHOLDER_PASSWORD substitution in Create-VM) + and the PowerShell Direct credentials used by the test scripts. + +.OUTPUTS + [String] The VM password. +#> +function Get-VMPassword { + return 'eBPF4W!n' +} + +<# +.SYNOPSIS + Imports the CredentialManager, and installs it if necessary. + +.DESCRIPTION + This function imports the CredentialManager module and installs it if it is not already installed. It also ensures that any dependencies are installed. +#> +function Get-CredentialManager { + # Import the CredentialManager module. Ensure any dependencies are installed. + Install-PackageProvider -Name NuGet -Force -ErrorAction Stop *> $null 2>&1 + Import-PackageProvider -Name NuGet -Force -ErrorAction Stop *> $null 2>&1 + if (-not (Get-Module -ListAvailable -Name CredentialManager)) { + Install-Module -Name CredentialManager -Force -ErrorAction Stop *> $null 2>&1 + } + Import-Module CredentialManager -ErrorAction Stop +} + +<# +.SYNOPSIS + Retrieves a credential from the Windows Credential Manager using PsExec. + +.PARAMETER Target + The name of the stored credential. Default is "MyStoredCredential". + +.DESCRIPTION + This function uses PsExec to run a PowerShell script in the LocalSystem account context to retrieve a credential from the Windows Credential Manager. + +.EXAMPLE + $credential = Retrieve-StoredCredential -Target "MyStoredCredential" +#> +function Retrieve-StoredCredential { + param ( + [Parameter(Mandatory=$True)][string]$Target + ) + Get-CredentialManager + + $Script = @" + Import-Module CredentialManager -ErrorAction Stop; + `$Credential = Get-StoredCredential -Target '$Target'; + `$UserName = `$Credential.UserName; + `$Password = `$Credential.GetNetworkCredential().Password; + \"`$UserName`n`$Password\" +"@ + + # PSExec sometimes fails to fetch the output. Retry up to 3 times to improve reliability. + $attempt = 0 + $maxRetries = 5 + while ($attempt -lt $maxRetries) { + try { + $output = Invoke-PsExecScript -Script $Script + $lines = $output -split "`n" + $Username = $lines[0].Trim() + $Password = ConvertTo-SecureString -String $lines[1].Trim() -AsPlainText -Force + if ($null -eq $Username -or $null -eq $Password) { + throw "Failed to retrieve the stored credential." + } + return [System.Management.Automation.PSCredential]::new($Username, $Password) + } catch { + $attempt++ + if ($attempt -lt $maxRetries) { + Start-Sleep -Seconds 5 + } else { + throw "Failed to retrieve the stored credential after $maxRetries attempts." + } + } + } +} + +<# +.SYNOPSIS + Creates a PSCredential for the specified VM user. + +.DESCRIPTION + For local VMs (PowerShell Direct), creates a PSCredential using the well-known + password from Get-VMPassword. + + For remote VMs (WinRM), retrieves the credential from Windows Credential Manager + using the specified target name. The credential must have been previously stored + with New-StoredCredential using `-Persist LocalMachine`. + +.PARAMETER Username + The username for the credential. + +.PARAMETER VMIsRemote + When true, retrieves the credential from Credential Manager instead of using + the hardcoded password. Defaults to false. + +.EXAMPLE + $credential = Get-VMCredential -Username 'Administrator' + $credential = Get-VMCredential -Username 'Administrator' -VMIsRemote $true +#> +function Get-VMCredential { + param ( + [Parameter(Mandatory=$True)][string]$Username, + [Parameter(Mandatory=$false)][bool]$VMIsRemote = $false + ) + if ($VMIsRemote) { + # Determine the Credential Manager target based on username. + if ($Username -eq 'Administrator') { + $CredentialTarget = 'TEST_VM' + } else { + $CredentialTarget = 'TEST_VM_STANDARD' + } + return Retrieve-StoredCredential -Target $CredentialTarget + } else { + $securePassword = ConvertTo-SecureString -String (Get-VMPassword) -AsPlainText -Force + return [System.Management.Automation.PSCredential]::new($Username, $securePassword) + } +} + + +function Expand-ZipFile { + param( + [Parameter(Mandatory=$True)][string] $DownloadFilePath, + [Parameter(Mandatory=$True)][string] $OutputDir, + [Parameter(Mandatory=$True)][int] $maxRetries, + [Parameter(Mandatory=$True)][int] $retryDelay, + [Parameter(Mandatory=$True)][int] $timeout + ) + + for ($i = 0; $i -lt $maxRetries; $i++) { + try { + $job = Start-Job -ScriptBlock { + param ($DownloadFilePath, $OutputDir) + Expand-Archive -Path $DownloadFilePath -DestinationPath $OutputDir -Force + } -ArgumentList $DownloadFilePath, $OutputDir + + if (Wait-Job -Job $job -Timeout $timeout) { + Write-Log "Extraction completed. $DownloadFilePath -> $OutputDir" + Receive-Job -Job $job + break + } else { + Stop-Job -Job $job + Remove-Job -Job $job + if ($i -eq ($maxRetries - 1)) { + throw "Failed to extract $DownloadFilePath after $maxRetries attempts." + } else { + Start-Sleep -Seconds $retryDelay + } + } + } catch { + if ($i -eq ($maxRetries - 1)) { + throw "Failed to extract $DownloadFilePath after $maxRetries attempts." + } else { + Start-Sleep -Seconds $retryDelay + } + } + } +} + +function Get-ZipFileFromUrl { + param( + [Parameter(Mandatory=$True)][string] $Url, + [Parameter(Mandatory=$True)][string] $DownloadFilePath, + [Parameter(Mandatory=$True)][string] $OutputDir + ) + $maxRetries = 5 + $retryDelay = 5 # seconds + $timeout = 300 # seconds + + Write-Log "Downloading $Url to $DownloadFilePath" + + for ($i = 0; $i -lt $maxRetries; $i++) { + try { + $response = Invoke-WebRequest -Uri $Url -UseBasicParsing -Method Head -TimeoutSec $timeout + if ($response.StatusCode -ne 200) { + throw "Failed to reach $Url HTTP status code: $($response.StatusCode)" + } + + $ProgressPreference = 'SilentlyContinue' + + $job = Start-Job -ScriptBlock { + param ($Url, $DownloadFilePath, $timeout) + Invoke-WebRequest -Uri $Url -OutFile $DownloadFilePath -TimeoutSec $timeout + } -ArgumentList $Url, $DownloadFilePath, $timeout + + if (Wait-Job -Job $job -Timeout $timeout) { + Receive-Job -Job $job + + Write-Log "Extracting $DownloadFilePath to $OutputDir" + Expand-ZipFile -DownloadFilePath $DownloadFilePath -OutputDir $OutputDir -maxRetries $maxRetries -retryDelay $retryDelay -timeout $timeout + break + } else { + Stop-Job -Job $job + Remove-Job -Job $job + if (Test-Path $DownloadFilePath) { + Remove-Item -Path $DownloadFilePath -Force -ErrorAction Ignore + } + if ($i -eq ($maxRetries - 1)) { + throw "Failed to download $Url after $maxRetries attempts." + } else { + Start-Sleep -Seconds $retryDelay + } + } + } catch { + if (Test-Path $DownloadFilePath) { + Remove-Item -Path $DownloadFilePath -Force -ErrorAction Ignore + } + if ($i -eq ($maxRetries - 1)) { + throw "Failed to download $Url after $maxRetries attempts." + } else { + Start-Sleep -Seconds $retryDelay + } + } + } +} + +function Get-RegressionTestArtifacts +{ + param([Parameter(Mandatory=$True)][string] $Configuration, + [Parameter(Mandatory=$True)][string] $ArtifactVersion) + + $RegressionTestArtifactsPath = "$pwd\regression" + $OriginalPath = $pwd + if (Test-Path -Path $RegressionTestArtifactsPath) { + Remove-Item -Path $RegressionTestArtifactsPath -Recurse -Force + } + mkdir $RegressionTestArtifactsPath + + # Verify artifacts' folder presence + if (-not (Test-Path -Path $RegressionTestArtifactsPath)) { + $ErrorMessage = "*** ERROR *** Regression test artifacts folder not found: $RegressionTestArtifactsPath)" + Write-Log $ErrorMessage + throw $ErrorMessage + } + + # Download regression test artifacts for each version. + $DownloadPath = "$RegressionTestArtifactsPath" + $ArtifactName = "v$ArtifactVersion/Build.$Configuration.x64.zip" + $ArtifactUrl = "https://github.com/microsoft/ebpf-for-windows/releases/download/" + $ArtifactName + + if (Test-Path -Path $DownloadPath\Build-x64.$Configuration) { + Remove-Item -Path $DownloadPath\Build-x64.$Configuration -Recurse -Force + } + + Get-ZipFileFromUrl -Url $ArtifactUrl -DownloadFilePath "$DownloadPath\Build-x64.$Configuration.zip" -OutputDir $DownloadPath + $DownloadedArtifactPath = "$DownloadPath\Build $Configuration x64" + if (!(Test-Path -Path $DownloadedArtifactPath)) { + throw ("Path ""$DownloadedArtifactPath"" not found.") + } + + # Copy all the drivers, DLLs, exe and .o files to pwd. + Write-Log "Copy regression test artifacts to main folder" -ForegroundColor Green + Push-Location $DownloadedArtifactPath + Get-ChildItem -Path .\* -Include *.sys | Move-Item -Destination $OriginalPath -Force + Get-ChildItem -Path .\* -Include *.dll | Move-Item -Destination $OriginalPath -Force + Get-ChildItem -Path .\* -Include *.exe | Move-Item -Destination $OriginalPath -Force + Get-ChildItem -Path .\* -Include *.o | Move-Item -Destination $OriginalPath -Force + Pop-Location + + Remove-Item -Path $DownloadPath -Force -Recurse + + # Delete ebpfapi.dll from the artifacts. ebpfapi.dll from the MSI installation should be used instead. + Remove-Item -Path ".\ebpfapi.dll" -Force +} + +# Copied from https://github.com/microsoft/msquic/blob/main/scripts/prepare-machine.ps1 +function Get-CoreNetTools { + param( + [string] $Architecture = "x64" + ) + # Download and extract https://github.com/microsoft/corenet-ci. + $DownloadPath = "$pwd\corenet-ci" + mkdir $DownloadPath + Write-Log "Downloading CoreNet-CI to $DownloadPath" + Get-ZipFileFromUrl -Url "https://github.com/microsoft/corenet-ci/archive/refs/heads/main.zip" -DownloadFilePath "$DownloadPath\corenet-ci.zip" -OutputDir $DownloadPath + # DuoNic. + if ($Architecture -eq "arm64") { + $duoNicPath = "$DownloadPath\corenet-ci-main\vm-setup\duonic\arm64\*" + } else { + $duoNicPath = "$DownloadPath\corenet-ci-main\vm-setup\duonic\*" + } + Move-Item -Path $duoNicPath -Destination $pwd -Force + # Procdump. + Move-Item -Path "$DownloadPath\corenet-ci-main\vm-setup\procdump64.exe" -Destination $pwd -Force + # NotMyFault. + Move-Item -Path "$DownloadPath\corenet-ci-main\vm-setup\notmyfault64.exe" -Destination $pwd -Force + Remove-Item -Path $DownloadPath -Force -Recurse +} + +# Download and extract PSExec to run tests as SYSTEM. +function Get-PSExec { + $psExecPath = "$pwd\PsExec64.exe" + # Check to see if PSExec already exists + if (Test-Path -Path $psExecPath) { + return $psExecPath + } + $url = "https://download.sysinternals.com/files/PSTools.zip" + $DownloadPath = "$pwd\psexec" + + Get-ZipFileFromUrl -Url $url -DownloadFilePath "$pwd\pstools.zip" -OutputDir "$DownloadPath" + Move-Item -Path "$DownloadPath\PsExec64.exe" -Destination $pwd -Force + Remove-Item -Path $DownloadPath -Force -Recurse -ErrorAction Ignore + return $psExecPath +} + +<# +.SYNOPSIS + Invokes a command on a remote or local VM. + +.PARAMETER VMName + The name of the VM. + +.PARAMETER VMIsRemote + Indicates if the VM is remote. + +.PARAMETER Credential + The credential to use for the VM. + +.DESCRIPTION + This function invokes a command on a remote or local VM using the specified credentials. + +.PARAMETER ScriptBlock + The script block to execute on the VM. + +.PARAMETER ArgumentList + The arguments to pass to the script block. + +.EXAMPLE + Invoke-CommandOnVM -VMName "MyVM" -VMIsRemote $true -Credential $credential -ScriptBlock { Get-Process } +#> +function Invoke-CommandOnVM { + param( + [Parameter(Mandatory = $true)][string] $VMName, + [Parameter(Mandatory = $false)][bool] $VMIsRemote = $false, + [Parameter(Mandatory = $true)][PSCredential] $Credential, + [Parameter(Mandatory = $true)][ScriptBlock] $ScriptBlock, + [Parameter(Mandatory = $false)][object[]] $ArgumentList = @() + ) + Write-Log "Invoking command on VM: $VMName (IsRemote: $VMIsRemote)" + if ($VMIsRemote) { + Invoke-Command -ComputerName $VMName -Credential $Credential -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -ErrorAction Stop + } else { + $maxAttempts = 5 + for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) { + try { + Invoke-Command -VMName $VMName -Credential $Credential -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -ErrorAction Stop + break + } catch { + $isTransientPowerShellDirectFailure = + ($_.CategoryInfo.Reason -eq 'PSDirectException') -or + ($_.FullyQualifiedErrorId -like 'PSSessionStateBroken*') -or + ($_.Exception.Message -match 'credential is invalid') -or + ($_.Exception.Message -match 'remote session might have ended') + if (-not $isTransientPowerShellDirectFailure -or $attempt -eq $maxAttempts) { + throw + } + + Write-Log "Transient PowerShell Direct failure for $VMName on attempt $attempt of $maxAttempts`: $($_.Exception.Message). Retrying..." + Start-Sleep -Seconds 5 + } + } + } +} + +<# +.SYNOPSIS + Creates a new PowerShell session on a remote or local VM. +.PARAMETER VMName + The name of the VM. +.PARAMETER VMIsRemote + Indicates if the VM is remote. +.PARAMETER Credential + The credential to use for the VM. +.RETURNS + A new PowerShell session object. +.DESCRIPTION + This function creates a new PowerShell session on a remote or local VM using the specified credentials +.EXAMPLE + $session = New-SessionOnVM -VMName "MyVM" -VMIsRemote $true -Credential $credential +#> +function New-SessionOnVM { + param( + [Parameter(Mandatory = $true)][string] $VMName, + [Parameter(Mandatory = $false)][bool] $VMIsRemote = $false, + [Parameter(Mandatory = $true)][PSCredential] $Credential + ) + $session = $null + Write-Log "Creating new session on VM: $VMName (IsRemote: $VMIsRemote)" + if ($VMIsRemote) { + $session = New-PSSession -ComputerName $VMName -Credential $Credential -ErrorAction Stop + } else { + $session = New-PSSession -VMName $VMName -Credential $Credential -ErrorAction Stop + } + return $session +} diff --git a/scripts/config_test_vm.psm1 b/scripts/config_test_vm.psm1 new file mode 100644 index 00000000..de774b1f --- /dev/null +++ b/scripts/config_test_vm.psm1 @@ -0,0 +1,1176 @@ +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: MIT + +param ([Parameter(Mandatory=$True)] [string] $WorkingDirectory, + [Parameter(Mandatory=$True)] [string] $LogFileName) + +$sleepSeconds = 10 + +Push-Location $WorkingDirectory + +Import-Module .\common.psm1 -Force -ArgumentList ($LogFileName) -WarningAction SilentlyContinue +Import-Module .\tracing_utils.psm1 -Force -ArgumentList ($LogFileName, $WorkingDirectory) -WarningAction SilentlyContinue + +# +# VM Initialization functions +# + +function Wait-AllVMsReadyForCommands +{ + param([Parameter(Mandatory=$True)]$VMList, + [Parameter(Mandatory=$True)][PSCredential] $TestCredential, + [Parameter(Mandatory=$false)][bool] $VMIsRemote = $false) + + $totalSleepTime = 0 + $ReadyList = @{} + do { + foreach($VM in $VMList) { + $VMName = $VM.Name + if ($ReadyList[$VMName] -ne $True) { + Write-Log "Poking $VMName to see if it is ready to accept commands" + $ret = $False + try { + Invoke-CommandOnVM -VMName $VMName -VMIsRemote $VMIsRemote -Credential $TestCredential -ErrorAction SilentlyContinue -ScriptBlock {$True} + $ret = $True + } catch { + if (-not $VMIsRemote) { + # Transient PS Direct failures during restore/boot often look like this: + # "The credential is invalid." with PSDirectException / PSSessionStateBroken + $isTransient = + ($_.CategoryInfo.Reason -eq 'PSDirectException') -or + ($_.FullyQualifiedErrorId -eq 'PSSessionStateBroken') -or + ($_.Exception.Message -match 'credential is invalid') + + if ($isTransient) { + Write-Log "PS Direct threw transient exception for $VMName - $($_.Exception.Message). Will retry." + } else { + Write-Log "PS Direct threw fatal exception for $VMName - $($_.Exception.Message)." + break + } + } + } + if ($ret -eq $True) { + $ReadyList += @{$VMName = $True} + } else { + continue + } + Write-Log "VM $VMName is ready" -ForegroundColor Green + } + } + if ($ReadyList.Count -ne $VMList.Count) { + Write-Log "Waiting $sleepSeconds seconds for $VMName to be responsive." + # Sleep for sleepSeconds seconds. + Start-Sleep -seconds $sleepSeconds + $totalSleepTime += $sleepSeconds + } + } + until (($ReadyList.Count -eq $VMList.Count) -or ($totalSleepTime -gt 5*60)) + + if ($ReadyList.Count -ne $VMList.Count) { + throw ("One or more VMs not ready after 5 minutes.") + } +} + +function Wait-AllVMsToInitialize +{ + param([Parameter(Mandatory=$True)]$VMList, + [Parameter(Mandatory=$false)][bool] $VMIsRemote = $false) + + $totalSleepTime = 0 + $ReadyList = @{} + do { + foreach ($VM in $VMList) { + $VMName= $VM.Name + if ($ReadyList[$VMName] -ne $True) { + $HeartBeat = (Get-VMIntegrationService $VMName | ?{$_.name -eq "Heartbeat"}).PrimaryStatusDescription + if ($HeartBeat -eq "OK") { + $ReadyList += @{$VMName = $True} + } else { + continue + } + Write-Log "Heartbeat OK on $VMName" -ForegroundColor Green + } + } + if ($ReadyList.Count -ne $VMList.Count) { + Write-Log ("{0} of {1} VMs are ready." -f $ReadyList.Count, $VMList.Count) + # Sleep for sleepSeconds seconds. + Start-Sleep -seconds $sleepSeconds + $totalSleepTime += $sleepSeconds + } + } + until (($ReadyList.Count -eq $VMList.Count) -or ($totalSleepTime -gt 5*60)) + + if ($ReadyList.Count -ne $VMList.Count) { + throw ("Did not get heartbeat from one or more VMs 5 minutes after starting") + } + + $TestCredential = Get-VMCredential -Username 'Administrator' -VMIsRemote $VMIsRemote + Wait-AllVMsReadyForCommands -VMList $VMList -TestCredential $TestCredential -VMIsRemote:$VMIsRemote + + # Enable guest-services on each VM. + $ReadyList.Clear() + $totalSleepTime = 0 + + do { + foreach($VM in $VMList) { + $VMName = $VM.Name + if ((Get-VMIntegrationService $VMName | ?{$_.name -eq "Guest Service Interface"}).PrimaryStatusDescription -ne "OK") { + Write-Log "Enabling guest services on $VMName" + Enable-VMIntegrationService -VMName $VMName -Name "Guest Service Interface" + if ((Get-VMIntegrationService $VMName | ?{$_.name -eq "Guest Service Interface"}).PrimaryStatusDescription -eq "OK") { + $ReadyList += @{$VMName = $True} + } else { + break + } + Write-Log "Guest services enabled on $VMName" -ForegroundColor Green + } else { + Write-Log "Guest services already enabled on $VMName" + $ReadyList += @{$VMName = $True} + } + } + if ($ReadyList.Count -ne $VMList.Count) { + # Wait one second for guest services to start. + Start-Sleep -seconds 1 + $totalSleepTime += 1 + } + } + until (($ReadyList.Count -eq $VMList.Count) -or ($totalSleepTime -gt 1*60)) + + if ($ReadyList.Count -ne $VMList.Count) { + throw ("Guest service failed to get enabled on one or more VMs after waiting for 1 minute.") + } +} + +function Restore-AllVMs +{ + param ([Parameter(Mandatory=$True)] $VMList) + foreach ($VM in $VMList) { + $VMName = $VM.Name + Write-Log "Restoring VM $VMName" + Restore-VMSnapshot -Name 'baseline' -VMName $VMName -Confirm:$false + } +} + +function Start-AllVMs +{ + param ([Parameter(Mandatory=$True)] $VMList) + foreach ($VM in $VMList) { + $VMName = $VM.Name + Write-Log "Starting VM $VMName" + Start-VM -VMName $VMName -ErrorAction Stop 2>&1 | Write-Log + } +} + +function Initialize-AllVMs +{ + param ([Parameter(Mandatory=$True)] $VMList) + + # Restore the VMs. + Restore-AllVMs -VMList $VMList + + # Start the VMs. + Start-AllVMs -VMList $VMList + + if (-not $VMIsRemote) { + # Wait for VMs to be ready. + Write-Log "Waiting for all the VMs to be in ready state..." -ForegroundColor Yellow + Wait-AllVMsToInitialize -VMList $VMList + } else { + $TestCredential = Get-VMCredential -Username 'Administrator' -VMIsRemote $true + Wait-AllVMsReadyForCommands -VMList $VMList -TestCredential $TestCredential -VMIsRemote:$true + } +} + +# +# VM Cleanup Functions +# + +function Stop-AllVMs +{ + param ([Parameter(Mandatory=$True)] $VMList) + + foreach ($VM in $VMList) { + # Stop the VM. + $VMName = $VM.Name + Write-Log "Stopping VM $VMName" + Stop-VM -Name $VMName -Force -TurnOff -WarningAction Ignore 2>&1 | Write-Log + } +} + +# +# Export build artifacts. +# + +function Export-BuildArtifactsToVMs +{ + param( + [Parameter(Mandatory=$True)] $VMList, + [Parameter(Mandatory=$false)][bool] $VMIsRemote = $false + ) + + $tempFileName = [System.IO.Path]::GetTempFileName() + ".tgz" + try { + Write-Log "Creating $tempFileName containing files in $pwd" + &tar @("cfz", "$tempFileName", "*") + if ($LASTEXITCODE -ne 0) { + throw "Failed to create build artifact archive with exit code $LASTEXITCODE." + } + Write-Log "Created $tempFileName containing files in $pwd" + + # Copy artifacts to the given VM list. + foreach($VM in $VMList) { + $VMName = $VM.Name + Write-Log "Exporting build artifacts to $VMName" + $TestCredential = Get-VMCredential -Username 'Administrator' -VMIsRemote $VMIsRemote + $initializeGuest = { + # Create working directory c:\eBPF. + if(!(Test-Path "$Env:SystemDrive\eBPF")) { + New-Item -ItemType Directory -Path "$Env:SystemDrive\eBPF" | Out-Null + } + # Enable EULA for all SysInternals tools. + $RegistryPath = 'HKCU:\Software\Sysinternals' + if (-not (Test-Path $RegistryPath)) { + # Create the registry key if it doesn't exist + New-Item -Path $RegistryPath -Force | Out-Null + } + Set-ItemProperty -Path $RegistryPath -Name 'EulaAccepted' -Value 1 + + # Enables full memory dump. + # NOTE: This needs a VM with an explicitly created page file of *AT LEAST* (physical_memory + 1MB) in size. + # The default value of the 'CrashDumpEnabled' key is 7 ('automatic' sizing of dump file size (system determined)). + # https://learn.microsoft.com/en-us/troubleshoot/windows-server/performance/memory-dump-file-options + Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\CrashControl' -Name 'CrashDumpEnabled' -Value 1 + + return $Env:SystemDrive + } + + if ($VMIsRemote) { + $VMSession = New-SessionOnVM -VMName $VMName -VMIsRemote $true -Credential $TestCredential + try { + $VMSystemDrive = Invoke-Command -Session $VMSession -ScriptBlock $initializeGuest -ErrorAction Stop + Write-Log "Created c:\eBPF, enabled SysInternals EULA and full memory dump on $VMName" + Write-Log "VM $VMName system drive is $VMSystemDrive" + + Write-Log "Copying $tempFileName to $VMSystemDrive\eBPF on $VMName" + Copy-Item -ToSession $VMSession -Path $tempFileName -Destination "$VMSystemDrive\eBPF\ebpf.tgz" -Force -ErrorAction Stop + Write-Log "Copied $tempFileName to $VMSystemDrive\eBPF on $VMName" + + Invoke-Command -Session $VMSession -ScriptBlock { + Set-Location "$Env:SystemDrive\eBPF" + &tar @("xf", "ebpf.tgz") + if ($LASTEXITCODE -ne 0) { + throw "Failed to unpack build artifacts with exit code $LASTEXITCODE." + } + } -ErrorAction Stop + } finally { + Remove-PSSession $VMSession -ErrorAction SilentlyContinue + } + } else { + $VMSystemDrive = Invoke-CommandOnVM -VMName $VMName -Credential $TestCredential -ScriptBlock $initializeGuest + Write-Log "Created c:\eBPF, enabled SysInternals EULA and full memory dump on $VMName" + Write-Log "VM $VMName system drive is $VMSystemDrive" + + Write-Log "Copying $tempFileName to $VMSystemDrive\eBPF on $VMName with Hyper-V Guest Services" + Copy-VMFile -VMName $VMName -SourcePath $tempFileName -DestinationPath "$VMSystemDrive\eBPF\ebpf.tgz" ` + -FileSource Host -CreateFullPath -Force -ErrorAction Stop + Write-Log "Copied $tempFileName to $VMSystemDrive\eBPF on $VMName" + + Invoke-CommandOnVM -VMName $VMName -Credential $TestCredential -ScriptBlock { + Set-Location "$Env:SystemDrive\eBPF" + &tar @("xf", "ebpf.tgz") + if ($LASTEXITCODE -ne 0) { + throw "Failed to unpack build artifacts with exit code $LASTEXITCODE." + } + } + } + + Write-Log "Unpacked $tempFileName to $VMSystemDrive\eBPF on $VMName" + Write-Log "Export completed." -ForegroundColor Green + } + } finally { + Remove-Item -Force $tempFileName -ErrorAction SilentlyContinue + } +} + +# +# Install eBPF components on VM. +# +function Install-eBPFComponentsOnVM +{ + param( + [parameter(Mandatory=$true)][string] $VMName, + [parameter(Mandatory=$true)][string] $TestMode, + [parameter(Mandatory=$true)][bool] $KmTracing, + [parameter(Mandatory=$true)][string] $KmTraceType, + [Parameter(Mandatory=$false)][bool] $VMIsRemote = $false, + [parameter(Mandatory=$false)][bool] $GranularTracing = $false + ) + + Write-Log "Installing eBPF components on $VMName" + $TestCredential = Get-VMCredential -Username 'Administrator' -VMIsRemote $VMIsRemote + + $scriptBlock = { + param([Parameter(Mandatory=$True)] [string] $WorkingDirectory, + [Parameter(Mandatory=$True)] [string] $LogFileName, + [Parameter(Mandatory=$true)] [bool] $KmTracing, + [Parameter(Mandatory=$true)] [string] $KmTraceType, + [parameter(Mandatory=$true)][string] $TestMode, + [parameter(Mandatory=$false)][bool] $GranularTracing = $false) + $WorkingDirectory = "$env:SystemDrive\$WorkingDirectory" + Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue + Import-Module $WorkingDirectory\install_ebpf.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue + + Install-eBPFComponents -KmTracing $KmTracing -KmTraceType $KmTraceType -KMDFVerifier $true -TestMode $TestMode -GranularTracing $GranularTracing -ErrorAction Stop + } + + Invoke-CommandOnVM -VMName $VMName -Credential $TestCredential -VMIsRemote $VMIsRemote -ScriptBlock $scriptBlock -ArgumentList ("eBPF", $LogFileName, $KmTracing, $KmTraceType, $TestMode, $GranularTracing) -ErrorAction Stop + + Write-Log "eBPF components installed on $VMName" -ForegroundColor Green +} + +function Uninstall-eBPFComponentsOnVM +{ + param([parameter(Mandatory=$true)][string] $VMName) + + Write-Log "Unnstalling eBPF components on $VMName" + $TestCredential = Get-VMCredential -Username 'Administrator' + + Invoke-Command -VMName $VMName -Credential $TestCredential -ScriptBlock { + param([Parameter(Mandatory=$True)] [string] $WorkingDirectory, + [Parameter(Mandatory=$True)] [string] $LogFileName) + $WorkingDirectory = "$env:SystemDrive\$WorkingDirectory" + Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue + Import-Module $WorkingDirectory\install_ebpf.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue + + Uninstall-eBPFComponents + } -ArgumentList ("eBPF", $LogFileName) -ErrorAction Stop + Write-Log "eBPF components uninstalled on $VMName" -ForegroundColor Green +} + +function Stop-eBPFComponentsOnVM +{ + param([parameter(Mandatory=$true)][string] $VMName, + [parameter(Mandatory=$false)][bool] $GranularTracing = $false) + + Write-Log "Stopping eBPF components on $VMName" + $TestCredential = Get-VMCredential -Username 'Administrator' + + Invoke-Command ` + -VMName $VMName ` + -Credential $TestCredential ` + -ScriptBlock { + param([Parameter(Mandatory=$True)] [string] $WorkingDirectory, + [Parameter(Mandatory=$True)] [string] $LogFileName, + [Parameter(Mandatory=$false)] [bool] $GranularTracing = $false + ) + + $WorkingDirectory = "$env:SystemDrive\$WorkingDirectory" + Import-Module $WorkingDirectory\common.psm1 ` + -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue + + Import-Module $WorkingDirectory\install_ebpf.psm1 ` + -ArgumentList($WorkingDirectory, $LogFileName) ` + -Force -WarningAction SilentlyContinue + + Stop-eBPFServiceAndDrivers -GranularTracing $GranularTracing + + } -ArgumentList ("eBPF", $LogFileName, $GranularTracing) -ErrorAction Stop + + Write-Log "eBPF components stopped on $VMName" -ForegroundColor Green +} + +function Compress-KernelModeDumpOnVM +{ + param ( + [Parameter(Mandatory = $True)] [System.Management.Automation.Runspaces.PSSession] $Session + ) + + Invoke-Command -Session $Session -ScriptBlock { + param([Parameter(Mandatory=$True)] [string] $WorkingDirectory, + [Parameter(Mandatory=$True)] [string] $LogFileName) + + Import-Module $env:SystemDrive\$WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue + + $KernelModeDumpFileSourcePath = "$Env:WinDir" + $KernelModeDumpFileDestinationPath = "$Env:SystemDrive\KernelDumps" + + # Create the compressed dump folder if doesn't exist. + if (!(Test-Path $KernelModeDumpFileDestinationPath)) { + Write-Log "Creating $KernelModeDumpFileDestinationPath directory." + Write-Log "Current user: $($env:USERNAME), SystemDrive: $($env:SystemDrive)" + Write-Log "Working directory: $(Get-Location)" + + try { + New-Item -ItemType Directory -Path $KernelModeDumpFileDestinationPath -Force -ErrorAction Stop | Out-Null + Write-Log "Successfully created directory: $KernelModeDumpFileDestinationPath" + } catch { + $ErrorMessage = "*** ERROR *** Create compressed dump file directory failed: $KernelModeDumpFileDestinationPath. Error: $($_.Exception.Message)" + Write-Log $ErrorMessage + + # Verify if directory creation failed and directory still doesn't exist + if (!(Test-Path $KernelModeDumpFileDestinationPath)) { + Write-Log "*** ERROR *** Directory creation failed and directory does not exist. Treating as non-fatal and returning." + return + } + } + } + + if (Test-Path $KernelModeDumpFileSourcePath\*.dmp -PathType Leaf) { + Write-Log "Found kernel mode dump(s) in $($KernelModeDumpFileSourcePath):" + $DumpFiles = get-childitem -Path $KernelModeDumpFileSourcePath\*.dmp + foreach ($DumpFile in $DumpFiles) { + Write-Log "`tName:$($DumpFile.Name), Size:$((($DumpFile.Length) / 1MB).ToString("F2")) MB" + } + + Write-Log ` + "Compressing kernel dump files: $KernelModeDumpFileSourcePath -> $KernelModeDumpFileDestinationPath" + + $result = CompressOrCopy-File -SourcePath "$KernelModeDumpFileSourcePath\*.dmp" -DestinationDirectory $KernelModeDumpFileDestinationPath -CompressedFileName "km_dumps.zip" + if ($result.CompressedCopySucceeded) { + Write-Log "Successfully compressed kernel dumps: $($result.FinalPath)" + } else { + Write-Log "Used uncompressed kernel dump fallback: $($result.FinalPath)" + } + } else { + Write-Log "No kernel mode dump(s) in $($KernelModeDumpFileSourcePath)." + } + } -ArgumentList ("eBPF", $LogFileName) -ErrorAction Ignore +} + +<# +.SYNOPSIS + Ensures a PSSession to a VM is valid and usable, reconnecting if necessary. +.PARAMETER VMName + The name of the VM. +.PARAMETER CurrentSession + The current PSSession object (may be null or broken). +.PARAMETER TestCredential + Credential to use for creating a new session. +.RETURNS + A valid PSSession object, or $null if reconnection fails. +#> +function Get-ValidSession { + param( + [Parameter(Mandatory=$true)][string] $VMName, + [Parameter(Mandatory=$false)][System.Management.Automation.Runspaces.PSSession] $CurrentSession, + [Parameter(Mandatory=$true)][PSCredential] $TestCredential + ) + + # Check if current session is still usable. + if ($CurrentSession -and $CurrentSession.State -eq 'Opened' -and $CurrentSession.Availability -eq 'Available') { + return $CurrentSession + } + + # Session is broken or unavailable - clean it up and create a new one. + if ($CurrentSession) { + $state = $CurrentSession.State + $avail = $CurrentSession.Availability + Write-Log "Session to $VMName is no longer valid (State: $state, Availability: $avail). Attempting to reconnect." + try { Remove-PSSession $CurrentSession -ErrorAction SilentlyContinue } catch {} + } + + # Try to create a new session with retries. + for ($attempt = 1; $attempt -le 3; $attempt++) { + try { + $newSession = New-PSSession -VMName $VMName -Credential $TestCredential -ErrorAction Stop + if ($newSession) { + Write-Log "Successfully reconnected to $VMName (attempt $attempt)." + return $newSession + } + } catch { + Write-Log "Failed to reconnect to $VMName (attempt $attempt of 3): $($_.Exception.Message)" + if ($attempt -lt 3) { + Start-Sleep -Seconds (5 * $attempt) + } + } + } + + Write-Log "*** WARNING *** Could not establish session to $VMName after 3 attempts." + return $null +} + +# +# Import test logs and dumps from VM. +# +function Import-ResultsFromVM +{ + param([Parameter(Mandatory=$True)] $VMList, + [Parameter(Mandatory=$true)] $KmTracing) + + # Note: caller (cleanup_ebpf_cicd_tests.ps1) already calls Wait-AllVMsToInitialize. + foreach($VM in $VMList) { + $VMName = $VM.Name + Write-Log "Importing TestLogs from $VMName" + if (!(Test-Path ".\TestLogs\$VMName")) { + New-Item -ItemType Directory -Path ".\TestLogs\$VMName" + } + + $TestCredential = Get-VMCredential -Username 'Administrator' + $VMSession = New-PSSession -VMName $VMName -Credential $TestCredential + if (!$VMSession) { + Write-Log "*** WARNING *** Failed to create PowerShell session on $VMName. Skipping result import for this VM." + continue + } + $VMSystemDrive = Invoke-Command -Session $VMSession -ScriptBlock {return $Env:SystemDrive} + + # Archive and copy kernel crash dumps, if any. + Write-Log "Processing kernel mode dump (if any) on VM $VMName" + try { + Compress-KernelModeDumpOnVM -Session $VMSession + } catch { + Write-Log "*** WARNING *** Failed to compress kernel dumps on ${VMName}: $($_.Exception.Message)" + } + + $LocalKernelArchiveLocation = ".\TestLogs\$VMName\KernelDumps" + if (!(Test-Path $LocalKernelArchiveLocation)) { + New-Item -ItemType Directory -Path $LocalKernelArchiveLocation | Out-Null + } + + # Copy kernel dumps from Test VM - try compressed first, then uncompressed from Windows folder + $VMSession = Get-ValidSession -VMName $VMName -CurrentSession $VMSession -TestCredential $TestCredential + if ($VMSession) { + try { + $result = CopyCompressedOrUncompressed-FileFromSession ` + -VMSession $VMSession ` + -CompressedSourcePath "$VMSystemDrive\KernelDumps\km_dumps.zip" ` + -UncompressedSourcePath "$VMSystemDrive\Windows\*.dmp" ` + -DestinationDirectory $LocalKernelArchiveLocation + + if ($result.CompressedCopySucceeded) { + Write-Log "Successfully copied compressed kernel dumps from ${VMName}: $($result.FinalPath)" + } else { + Write-Log "Used uncompressed kernel dump fallback from ${VMName}: $($result.FinalPath)" + } + } catch { + Write-Log "*** WARNING *** Failed to copy kernel dumps from ${VMName}: $($_.Exception.Message)" + } + } else { + Write-Log "*** WARNING *** Skipping kernel dump copy from $VMName - no valid session." + } + + # Copy user mode crash dumps if any. + $VMSession = Get-ValidSession -VMName $VMName -CurrentSession $VMSession -TestCredential $TestCredential + if ($VMSession) { + Copy-FromSessionWithTimeout ` + -Session $VMSession ` + -Path "$VMSystemDrive\dumps" ` + -Destination ".\TestLogs\$VMName" ` + -OperationName "Copy user dumps from $VMName" ` + -Recurse + + # Copy performance results from Test VM. + Write-Log ("Copy performance results from eBPF on $VMName to $pwd\TestLogs\$VMName\Logs") + Copy-FromSessionWithTimeout ` + -Session $VMSession ` + -Path "$VMSystemDrive\eBPF\*.csv" ` + -Destination ".\TestLogs\$VMName\Logs" ` + -OperationName "Copy CSV results from $VMName" ` + -Recurse + } else { + Write-Log "*** WARNING *** Skipping dump/CSV copy from $VMName - no valid session." + } + + # Copy logs from Test VM. + if (!(Test-Path ".\TestLogs\$VMName\Logs")) { + New-Item -ItemType Directory -Path ".\TestLogs\$VMName\Logs" + } + $VMSession = Get-ValidSession -VMName $VMName -CurrentSession $VMSession -TestCredential $TestCredential + if ($VMSession) { + Copy-FromSessionWithTimeout ` + -Session $VMSession ` + -Path "$VMSystemDrive\eBPF\TestLogs" ` + -Destination ".\TestLogs\$VMName\Logs" ` + -OperationName "Copy driver test logs from $VMName" ` + -Recurse + + $VMTemp = Invoke-WithTimeout -OperationName "Get TEMP path from $VMName" -TimeoutMs 30000 -ScriptBlock { + param($session) + Invoke-Command -Session $session -ScriptBlock {return $Env:TEMP} -ErrorAction SilentlyContinue + } -ArgumentList @($VMSession) + if ($VMTemp) { + Write-Log ("Copy $LogFileName from $VMTemp on $VMName to $pwd\TestLogs") + Copy-FromSessionWithTimeout ` + -Session $VMSession ` + -Path "$VMTemp\$LogFileName" ` + -Destination ".\TestLogs\$VMName\Logs" ` + -OperationName "Copy log file from $VMName" ` + -Recurse + } + } else { + Write-Log "*** WARNING *** Skipping log copy from $VMName - no valid session." + } + + # Copy kernel mode traces, if enabled. + if ($KmTracing) { + $VMSession = Get-ValidSession -VMName $VMName -CurrentSession $VMSession -TestCredential $TestCredential + if ($VMSession) { + $EtlFile = $LogFileName.Substring(0, $LogFileName.IndexOf('.')) + ".etl" + # Stop KM ETW Traces. + Invoke-WithTimeout -OperationName "Stop WPR trace on $VMName" -TimeoutMs 120000 -ScriptBlock { + param($session, $wd, $logFile, $etlFile) + Invoke-Command -Session $session -ScriptBlock { + param([Parameter(Mandatory=$True)] [string] $WorkingDirectory, + [Parameter(Mandatory=$True)] [string] $LogFileName, + [Parameter(Mandatory=$True)] [string] $EtlFile) + $WorkingDirectory = "$env:SystemDrive\$WorkingDirectory" + Import-Module ` + $WorkingDirectory\common.psm1 ` + -ArgumentList ($LogFileName) ` + -Force ` + -WarningAction SilentlyContinue + Import-Module ` + $WorkingDirectory\tracing_utils.psm1 ` + -ArgumentList ($LogFileName, $WorkingDirectory) ` + -Force ` + -WarningAction SilentlyContinue + + $baseFileName = [System.IO.Path]::GetFileNameWithoutExtension($EtlFile) + Stop-WPRTrace -FileName $baseFileName + } -ArgumentList ($wd, $logFile, $etlFile) -ErrorAction Ignore + } -ArgumentList @($VMSession, "eBPF", $LogFileName, $EtlFile) + } else { + Write-Log "*** WARNING *** Skipping KM trace stop on $VMName - no valid session." + } + } + + # Copy tracing ETL files from Test VM (if any). + Write-Log ("Copy ETL files from $VMSystemDrive\eBPF\TestLogs on $VMName to $pwd\TestLogs\$VMName\Logs") + $VMSession = Get-ValidSession -VMName $VMName -CurrentSession $VMSession -TestCredential $TestCredential + if ($VMSession) { + # First, compress the ETL files on the VM + Invoke-WithTimeout -OperationName "Compress ETL files on $VMName" -TimeoutMs 120000 -ScriptBlock { + param($session, $wd, $logFile) + Invoke-Command -Session $session -ScriptBlock { + param([Parameter(Mandatory=$True)] [string] $WorkingDirectory, + [Parameter(Mandatory=$True)] [string] $LogFileName) + $WorkingDirectory = "$env:SystemDrive\$WorkingDirectory" + + # Ensure common module is loaded in the remote session so Write-Log and Compress-File are available. + Import-Module "$WorkingDirectory\common.psm1" -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue + + if (Test-Path "$WorkingDirectory\TestLogs\*.etl" -PathType Leaf) { + Write-Log "Found ETL files in $WorkingDirectory\TestLogs" + Get-ChildItem "$WorkingDirectory\TestLogs\*.etl" | ForEach-Object { + Write-Log " ETL file: $($_.Name), Size: $((($_.Length) / 1MB).ToString('F2')) MB" + } + + Write-Log "Compressing ETL files..." + $compressionSucceeded = Compress-File -SourcePath "$WorkingDirectory\TestLogs\*.etl" -DestinationPath "$WorkingDirectory\traces.zip" + if (-not $compressionSucceeded -or -not (Test-Path "$WorkingDirectory\traces.zip")) { + Write-Log "*** WARNING *** ETL compression failed on VM. Will attempt to copy uncompressed ETL files." + } else { + Write-Log "Successfully compressed ETL files to traces.zip" + } + } + } -ArgumentList ($wd, $logFile) -ErrorAction Ignore + } -ArgumentList @($VMSession, "eBPF", $LogFileName) + + # Copy compressed ETL files from Test VM - try compressed first, then uncompressed fallback + try { + $tracingResult = CopyCompressedOrUncompressed-FileFromSession ` + -VMSession $VMSession ` + -CompressedSourcePath "$VMSystemDrive\eBPF\traces.zip" ` + -UncompressedSourcePath "$VMSystemDrive\eBPF\TestLogs\*.etl" ` + -DestinationDirectory ".\TestLogs\$VMName\Logs" + } catch { + Write-Log "*** WARNING *** Failed to copy ETL files from ${VMName}: $($_.Exception.Message)" + } + + # Compress and copy the performance profile if present. + Invoke-WithTimeout -OperationName "Compress perf profile on $VMName" -TimeoutMs 120000 -ScriptBlock { + param($session) + Invoke-Command -Session $session -ScriptBlock { + if (Test-Path $Env:SystemDrive\eBPF\bpf_performance*.etl -PathType Leaf) { + tar czf $Env:SystemDrive\eBPF\bpf_perf_etls.tgz -C $Env:SystemDrive\eBPF bpf_performance*.etl + dir $Env:SystemDrive\eBPF\bpf_performance*.etl + Remove-Item -Path $Env:SystemDrive\eBPF\bpf_performance*.etl + } + } -ErrorAction Ignore + } -ArgumentList @($VMSession) + Write-Log ("Copy performance profile from eBPF on $VMName to $pwd\TestLogs\$VMName\Logs") + Copy-FromSessionWithTimeout ` + -Session $VMSession ` + -Path "$VMSystemDrive\eBPF\bpf_perf_etls.tgz" ` + -Destination ".\TestLogs\$VMName\Logs" ` + -OperationName "Copy perf profile from $VMName" ` + -Recurse + } else { + Write-Log "*** WARNING *** Skipping ETL/perf copy from $VMName - no valid session." + } + + Write-Log "Completed importing results from $VMName" + } + # Move runner test logs to TestLogs folder. + Write-Log "Copy $LogFileName from $env:TEMP on host runner to $pwd\TestLogs" + Move-Item "$env:TEMP\$LogFileName" -Destination ".\TestLogs" -Force -ErrorAction Ignore 2>&1 | Write-Log +} + +function Import-ResultsFromHost { + param( + [Parameter(Mandatory = $true)][bool] $KmTracing + ) + Write-Log "Importing results from host..." + $TestLogsDir = Join-Path $WorkingDirectory 'TestLogs' + if (!(Test-Path $TestLogsDir)) { + New-Item -ItemType Directory -Path $TestLogsDir | Out-Null + } + + # Copy user mode crash dumps if any. + if (Test-Path "$WorkingDirectory\dumps") { + Copy-Item "$WorkingDirectory\dumps\*" "$TestLogsDir" -Recurse -Force -ErrorAction Ignore | Out-Null + } + + # Stop and collect ETL trace if enabled. + if ($KmTracing) { + $EtlFile = $LogFileName.Substring(0, $LogFileName.IndexOf('.')) + ".etl" + $baseFileName = [System.IO.Path]::GetFileNameWithoutExtension($EtlFile) + Stop-WPRTrace -FileName $baseFileName + } + Write-Log "Completed Importing results from host..." +} + +# +# Configure network adapters on VMs. +# +function Initialize-NetworkInterfaces { + param( + # Initialize network interfaces on VMs if set to true. + # Initialize network interfaces directly on the host otherwise. + [Parameter(Mandatory=$false)][bool] $ExecuteOnVM = $false, + [Parameter(Mandatory=$false)] $VMList = @(), + [Parameter(Mandatory=$true)][string] $TestWorkingDirectory, + [Parameter(Mandatory=$false)][bool] $VMIsRemote = $false + ) + + $commandScriptBlock = { + param([Parameter(Mandatory=$true)] [string] $WorkingDirectory, + [Parameter(Mandatory=$true)][string] $LogFileName) + Push-Location $WorkingDirectory + Import-Module .\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue + Write-Log "Installing DuoNic driver" + .\duonic.ps1 -Install -NumNicPairs 2 + Set-NetAdapterAdvancedProperty duo? -DisplayName Checksum -RegistryValue 0 + Pop-Location + } + + $argumentList = @($TestWorkingDirectory, $LogFileName) + + if ($ExecuteOnVM) { + # Execute on VMs. + $TestCredential = Get-VMCredential -Username 'Administrator' -VMIsRemote $VMIsRemote + foreach ($VM in $VMList) { + $VMName = $VM.Name + Write-Log "Initializing network interfaces on $VMName" + Invoke-CommandOnVM -VMName $VMName -VMIsRemote $VMIsRemote -Credential $TestCredential -ScriptBlock $commandScriptBlock -ArgumentList $argumentList -ErrorAction Stop + } + } else { + Write-Log "Initializing network interfaces on host" + & $commandScriptBlock @argumentList + } + +} + +# +# Queries registry for OS build information and logs it. +# +function Log-OSBuildInformationOnVM +{ + param([parameter(Mandatory=$true)][string] $VMName, + [Parameter(Mandatory=$false)][bool] $VMIsRemote = $false) + + $TestCredential = Get-VMCredential -Username 'Administrator' -VMIsRemote $VMIsRemote + Invoke-CommandOnVM -VMName $VMName -VMIsRemote:$VMIsRemote -Credential $TestCredential -ScriptBlock { + $buildLabEx = Get-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion' -Name 'BuildLabEx' + Write-Host "OS Build Information: $($buildLabEx.BuildLabEx)" + } +} + +<# +.SYNOPSIS + Helper function to execute a command on a VM. + +.DESCRIPTION + This function executes a command on a specified VM using the provided credentials. + +.PARAMETER VMName + The name of the VM to execute the command on. + +.PARAMETER Command + The command to execute on the VM. +#> +function Execute-CommandOnVM { + param ( + [Parameter(Mandatory=$True)][string]$VMName, + [Parameter(Mandatory=$True)][string]$Command + ) + + try { + $vmCredential = Get-VMCredential -Username 'Administrator' + Write-Log "Executing command on VM: $VMName. Command: $Command" + $result = Invoke-Command -VMName $VMName -Credential $VmCredential -ScriptBlock { + param($Command) + Invoke-Expression $Command + } -ArgumentList $Command + Write-Log "Successfully executed command on VM: $VMName. Command: $Command. Result: $result" + } catch { + throw "Failed to execute command on VM: $VMName with error: $_" + } +} + + +<# +.SYNOPSIS + Helper function to create a VM. + +.DESCRIPTION + This function creates a new VM with the specified parameters. + +.PARAMETER VmName + The name of the VM to create. + +.PARAMETER VhdPath + The path to the VHD file to use for the VM. + +.PARAMETER VmStoragePath + The storage path for the VM. + +.PARAMETER VMMemory + The amount of memory to allocate for the VM. + +.PARAMETER UnattendPath + The path to the unattend file to use for the VM. This will notably be used for configuring the user accounts and passwords. + +.PARAMETER VmSwitchName + The name of the switch to use for the VM. + +.EXAMPLE + Create-VM -VmName "MyVM" -VhdPath "C:\MyVHD.vhd" -VmStoragePath "C:\VMStorage" -VMMemory 2GB -UnattendPath "C:\MyUnattend.xml" -VmSwitchName "VMInternalSwitch" +#> +function Create-VM { + param( + [Parameter(Mandatory=$True)][string]$VmName, + [Parameter(Mandatory=$True)][string]$VhdPath, + [Parameter(Mandatory=$True)][string]$VmStoragePath, + [Parameter(Mandatory=$True)][Int64]$VMMemory, + [Parameter(Mandatory=$True)][string]$UnattendPath, + [Parameter(Mandatory=$True)][string]$VmSwitchName + ) + + try { + Write-Log "Creating VM: with Name: $VmName VhdPath: $VhdPath VmStoragePath: $VmStoragePath Memory: $VMMemory UnattendPath: $UnattendPath VMSwitchName: $VmSwitchName" + ## Check for any pre-requisites + # Check that the VHD exists + if (-not (Test-Path -Path $VhdPath)) { + throw "VHD not found at $VhdPath" + } + + ## Create the VM + # Create storage directory for the VM + Create-DirectoryIfNotExists -Path $VmStoragePath + + # Move the VHD to the path + Write-Log "Moving $VhdPath to $VmStoragePath" + Move-Item -Path $VhdPath -Destination $VmStoragePath -Force + $VmVhdPath = Join-Path -Path $VmStoragePath -ChildPath (Split-Path -Path $VhdPath -Leaf) + + # Move unattend to the path and replace placeholder strings + Write-Log "Moving $UnattendPath file to $VmStoragePath" + Move-Item -Path $UnattendPath -Destination $VmStoragePath -Force + $VmUnattendPath = Join-Path -Path $VmStoragePath -ChildPath (Split-Path -Path $UnattendPath -Leaf) + + # Replace password placeholder in unattend.xml with the canonical password from Get-VMPassword. + $password = Get-VMPassword + (Get-Content -Path $VmUnattendPath -Raw).Replace('PLACEHOLDER_PASSWORD', $password) | Set-Content -Path $VmUnattendPath + + # Configure the VHD with the unattend file. + Write-Log "Mounting VHD and applying unattend file" + $VmMountPath = Join-Path -Path $VmStoragePath -ChildPath 'mountedVhd' + if (-not (Test-Path -Path $VmMountPath)) { + New-Item -ItemType Directory -Path $VmMountPath + } + Mount-WindowsImage -ImagePath $VmVhdPath -Index 1 -Path $VmMountPath -ErrorAction Stop | Out-Null + Copy-Item -Path $VmUnattendPath -Destination $VmMountPath\Unattend.xml + Apply-WindowsUnattend -Path $VmMountPath -UnattendPath $VmMountPath\Unattend.xml -ErrorAction Stop | Out-Null + Dismount-WindowsImage -Path $VmMountPath -Save -ErrorAction Stop + + # Create the VM as Generation 2 with Secure Boot disabled so unsigned test drivers can load. + Write-Log "Creating the VM (Generation 2)" + New-VM -Name $VmName -VhdPath $VmVhdPath -SwitchName $VmSwitchName -Generation 2 + Set-VMFirmware -VMName $VmName -EnableSecureBoot Off + Set-VMMemory -VMName $VmName -DynamicMemoryEnabled $false -StartupBytes $VMMemory + + if ((Get-VM -VMName $vmName) -eq $null) { + throw "Failed to create VM: $VMName" + } + + Write-Log "Successfully created VM: $VMName" -ForegroundColor Green + } catch { + throw "Failed to create VM: $VmName with error: $_" + } +} + +<# +.SYNOPSIS + Helper function to configure a VM after creation. + +.DESCRIPTION + This function configures a VM after it has been created, including setting the processor count, enabling the Guest Service Interface, and executing a setup script. + +.PARAMETER VmName + The name of the VM to configure. + +.PARAMETER VMCpuCount + The number of processors to allocate for the VM. + +.PARAMETER VMWorkingDirectory + The working directory on the VM to use for executing the setup script. Defaults to 'C:\ebpf_cicd'. + +.PARAMETER VMSetupScript + The path to the setup script to execute on the VM. Defaults to '.\configure_vm.ps1'. + +.PARAMETER FilesToCopy + An optional array of hashtables specifying files to copy to the VM before checkpointing. + Each hashtable should have 'Source' (host path) and 'Destination' (VM path) keys. + +.EXAMPLE + Initialize-VM -VmName "MyVM" -VMCpuCount 4 + +.EXAMPLE + $files = @( + @{ Source = 'C:\work\bindmonitor_x64_signed.sys'; Destination = 'C:\eBPF\bindmonitor_x64_signed.sys' } + ) + Initialize-VM -VmName "MyVM" -VMCpuCount 4 -FilesToCopy $files +#> +function Initialize-VM { + param( + [Parameter(Mandatory=$True)][string]$VmName, + [Parameter(Mandatory=$True)][int]$VMCpuCount, + [Parameter(Mandatory=$False)][string]$VMWorkingDirectory='C:\ebpf_cicd', + [Parameter(Mandatory=$False)][string]$VMSetupScript='.\configure_vm.ps1', + [Parameter(Mandatory=$False)][array]$FilesToCopy=@() + ) + + try { + Write-Log "Configuring VM: $VmName" + $vmList = @( + @{ + Name = $VmName + } + ) + + # Post VM creation configuration steps. + Write-Log "Setting VM processor count to $VMCpuCount" + Set-VMProcessor -VMName $VmName -Count $VMCpuCount + Write-Log "Enabling Guest Service Interface" + Enable-VMIntegrationService -VMName $VMName -Name 'Guest Service Interface' + + # Start the VM + Write-Log "Starting VM: $VmName" + Start-VM -Name $VmName + Wait-AllVMsToInitialize -VMList $vmList + + # Copy setup script to the VM and execute it. + Write-Log "Executing VM configuration script ($VMSetupScript) on VM: $VmName" + Copy-VMFile -VMName $VmName -FileSource Host -SourcePath $VMSetupScript -DestinationPath "$VMWorkingDirectory\$VMSetupScript" -CreateFullPath + Execute-CommandOnVM -VMName $VmName -Command "Set-ExecutionPolicy -ExecutionPolicy Bypass -Scope LocalMachine; cd $VMWorkingDirectory; .\$VMSetupScript" + Write-Log "Successfully executed VM configuration script ($VMSetupScript) on VM: $VmName" -ForegroundColor Green + + Wait-AllVMsToInitialize -VMList $vmList + + # Copy any additional files to the VM before checkpointing. + if ($FilesToCopy.Count -gt 0) { + Write-Log "Copying $($FilesToCopy.Count) additional file(s) to VM: $VmName" + foreach ($file in $FilesToCopy) { + if (Test-Path $file.Source) { + Write-Log "Copying $($file.Source) to $($file.Destination) on $VmName" + Copy-VMFile -VMName $VmName -FileSource Host -SourcePath $file.Source -DestinationPath $file.Destination -CreateFullPath -Force + Write-Log "Successfully copied $($file.Source) to $VmName" + } else { + Write-Log "Warning: Source file not found: $($file.Source)" -ForegroundColor Yellow + } + } + } + + # Checkpoint the VM. This can sometimes fail if other operations are in progress, so retry a few times to ensure a successful checkpoint. + for ($i = 0; $i -lt 5; $i += 1) { + try { + Write-Log "Checkpointing VM: $VmName" + Checkpoint-VM -Name $VMName -SnapshotName 'baseline' + $checkpoint = Get-VMSnapshot -VMName $vmName | Where-Object { $_.Name -eq 'baseline' } + if ($checkpoint -eq $null) { + throw "Failed to create checkpoint for VM: $VmName" + } + Write-Log "Successfully added 'baseline' checkpoint for VM: $VMName" -ForegroundColor Green + break + } catch { + Write-Log "Failed to checkpoint VM: $VmName. Retrying..." + Start-Sleep -Seconds 5 + continue + } + } + + Write-Log "Successfully configured VM: $VmName" -ForegroundColor Green + } catch { + throw "Failed to configure VM: $VmName with error: $_" + } +} + +########## Helpers for the host machine ########## +<# +.SYNOPSIS + Extracts .zip files in the specified directory and returns paths to .vhd and .vhdx files. + +.DESCRIPTION + This function takes an input directory as a parameter, looks inside the directory for any .zip files, extracts them, and returns a PowerShell string array of all full paths to .vhd and .vhdx files. It suppresses any output and throws errors if any exceptions are found. + +.PARAMETER InputDirectory + The directory to search for .zip files and extract them. + +.EXAMPLE + $vhdFiles = Prepare-VhdFiles -InputDirectory "C:\MyDirectory" +#> +function Prepare-VhdFiles { + param ( + [Parameter(Mandatory=$true)][string]$InputDirectory + ) + + try { + $zipFiles = Get-ChildItem -Path $InputDirectory -Filter *.zip -Recurse + foreach ($zipFile in $zipFiles) { + Expand-Archive -Path $zipFile.FullName -DestinationPath $InputDirectory *> $null 2>&1 + } + + # Get all .vhd and .vhdx files + $vhdFiles = (Get-ChildItem -Path $InputDirectory -Recurse -Include *.vhd, *.vhdx) | Select-Object -ExpandProperty FullName + + if ($vhdFiles.Count -eq 0) { + throw "No VHD files found in $InputDirectory" + } + + return [string[]]$vhdFiles + } + catch { + Get-ChildItem -Path $InputDirectory -Recurse + throw "Failed to prepare VHD files with error: $_" + } +} + +<# +.SYNOPSIS + Helper function to create a VM switch if it does not already exist. + +.DESCRIPTION + Checks if a VM switch with the given name and type already exists. If not, it creates a new switch of the specified type. + +.PARAMETER SwitchName + The name of the switch to create. + +.PARAMETER SwitchType + The type of switch to create. Can be 'External' or 'Internal'. + +.EXAMPLE + Create-VMSwitchIfNeeded -SwitchName 'VMInternalSwitch' -SwitchType 'Internal' + Create-VMSwitchIfNeeded -SwitchName 'VMExternalSwitch' -SwitchType 'External' +#> +function Create-VMSwitchIfNeeded { + param ( + [Parameter(Mandatory=$true)][string]$SwitchName, + [Parameter(Mandatory=$true)][string]$SwitchType + ) + + if ($SwitchType -eq 'External') { + # Check to see if an external switch already exists + $ExternalSwitches = (Get-VMSwitch -SwitchType External -ErrorAction Ignore) + if ($ExternalSwitches -ne $null) { + Write-Log "External switch already exists: $($ExternalSwitches[0].Name)" + return + } + + # Try to create the external switch + $NetAdapterNames = (Get-NetAdapter -Name 'Ethernet*' | Where-Object { $_.Status -eq 'Up' }).Name + $index = 0 + foreach ($NetAdapterName in $NetAdapterNames) { + try { + if ([string]::IsNullOrEmpty($NetAdapterName)) { + continue + } + $currSwitchName = $SwitchName + '-' + $index + Write-Log "Attempting to creating external switch: $currSwitchName with NetAdapter: $NetAdapterName" + New-VMSwitch -Name $currSwitchName -NetAdapterName $NetAdapterName -AllowManagementOS $true + $index += 1 + } catch { + Write-Log "Failed to create external switch for NetAdapter: $NetAdapterName with error: $_" + } + } + } elseif ($SwitchType -eq 'Internal') { + # Check to see if an internal switch already exists + $InternalSwitches = (Get-VMSwitch -SwitchType Internal -Name $SwitchName -ErrorAction Ignore) + if ($InternalSwitches -ne $null) { + Write-Log "Internal switch already exists: $($InternalSwitches[0].Name)" + return + } + + # Try to create the internal switch + try { + Write-Log "Creating internal switch" + New-VMSwitch -Name $SwitchName -SwitchType Internal + } catch { + throw "Failed to create internal switch with error: $_" + } + } else { + throw "Invalid switch type: $SwitchType" + } + + Write-Log "Successfully created $SwitchType switch with name: $SwitchName" -ForegroundColor Green +} + +<# +.SYNOPSIS + Helper function to enable HVCI on the target VM. + +.DESCRIPTION + This function enables Hypervisor-protected Code Integrity (HVCI) on the specified VM. + +.PARAMETER VmName + The name of the VM on which to enable HVCI. + +.EXAMPLE + Enable-HVCIOnVM -VmName 'MyVM' +#> +function Enable-HVCIOnVM { + param ( + [Parameter(Mandatory=$True)][string]$VmName, + [Parameter(Mandatory=$false)][bool] $VMIsRemote = $false + ) + + try { + Write-Log "Enabling HVCI on VM: $VmName" + $vmCredential = Get-VMCredential -Username 'Administrator' -VMIsRemote $VMIsRemote + $commandScriptBlock = { + # Enable HVCI + New-Item -Path "HKLM:\SYSTEM\CurrentControlSet\Control\DeviceGuard\Scenarios" -Name "HypervisorEnforcedCodeIntegrity" -ItemType Directory -Force + Set-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\DeviceGuard\Scenarios\HypervisorEnforcedCodeIntegrity" -Name "Enabled" -Value 1 -Force + # Restart the VM to apply changes + Restart-Computer -Force -ErrorAction Stop + } + Invoke-CommandOnVM -VMName $VmName -VMIsRemote $VMIsRemote -Credential $vmCredential -ScriptBlock $commandScriptBlock + } catch { + throw "Failed to enable HVCI on VM: $VmName with error: $_" + } + + # Wait 1 minute for the VM to restart + Write-Log "Waiting for 1 minute for VM: $VmName to restart" + Start-Sleep -Seconds 60 + + $VMList = @(@{ Name = $VmName }) + if (-not $VMIsRemote){ + # Wait for the VM to restart and be ready again + Write-Log "Waiting for VM: $VmName to restart and be ready again" + Wait-AllVMsToInitialize -VMList $VMList + } else { + $TestCredential = Get-VMCredential -Username 'Administrator' -VMIsRemote $VMIsRemote + Wait-AllVMsReadyForCommands -VMList $VMList -TestCredential $TestCredential -VMIsRemote:$VMIsRemote + } + + Write-Log "HVCI enabled successfully on VM: $VmName" -ForegroundColor Green +} \ No newline at end of file diff --git a/scripts/execute_ebpf_cicd_tests.ps1 b/scripts/execute_ebpf_cicd_tests.ps1 new file mode 100644 index 00000000..2090337e --- /dev/null +++ b/scripts/execute_ebpf_cicd_tests.ps1 @@ -0,0 +1,166 @@ +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: MIT + +param ([Parameter(Mandatory = $false)][string] $LogFileName = "TestLog.log", + [Parameter(Mandatory = $false)][string] $WorkingDirectory = $pwd.ToString(), + [Parameter(Mandatory = $false)][string] $TestExecutionJsonFileName = "test_execution.json", + [Parameter(Mandatory = $false)][string] $TestMode = "CI/CD", + [Parameter(Mandatory = $false)][string[]] $Options = @("None"), + [Parameter(Mandatory = $false)][string] $SelfHostedRunnerName = [System.Net.Dns]::GetHostName(), + [Parameter(Mandatory = $false)][int] $TestHangTimeout = (30*60), + [Parameter(Mandatory = $false)][string] $UserModeDumpFolder = "C:\Dumps", + [Parameter(Mandatory = $false)][int] $TestJobTimeout = (60*60), + [Parameter(Mandatory = $false)][switch] $GranularTracing = $false, + # Boolean parameter indicating if XDP tests should be run. + [Parameter(Mandatory = $false)][bool] $RunXdpTests = $false, + [Parameter(Mandatory = $false)][switch] $ExecuteOnHost, + # This parameter is only used when ExecuteOnHost is false. + [Parameter(Mandatory = $false)][switch] $VMIsRemote) + +$ExecuteOnHost = [bool]$ExecuteOnHost +$ExecuteOnVM = (-not $ExecuteOnHost) +$VMIsRemote = [bool]$VMIsRemote + +Push-Location $WorkingDirectory + +Import-Module $WorkingDirectory\common.psm1 -Force -ArgumentList ($LogFileName) -ErrorAction Stop + +# Read the test execution json. +$Config = Get-Content ("{0}\{1}" -f $PSScriptRoot, $TestExecutionJsonFileName) | ConvertFrom-Json + +if ($ExecuteOnVM) { + Write-Log "Tests will be executed on VM" +} else { + Write-Log "Executing on host" +} + +$Job = Start-Job -ScriptBlock { + param ( + [Parameter(Mandatory = $True)] [bool] $ExecuteOnHost, + [Parameter(Mandatory = $True)] [bool] $ExecuteOnVM, + [Parameter(Mandatory = $True)] [bool] $VMIsRemote, + [Parameter(Mandatory = $True)] [PSCustomObject] $Config, + [Parameter(Mandatory = $True)] [string] $SelfHostedRunnerName, + [Parameter(Mandatory = $True)] [string] $WorkingDirectory, + [Parameter(Mandatory = $True)] [string] $LogFileName, + [Parameter(Mandatory = $True)] [string] $TestMode, + [Parameter(Mandatory = $True)] [string[]] $Options, + [Parameter(Mandatory = $True)] [int] $TestHangTimeout, + [Parameter(Mandatory = $True)] [string] $UserModeDumpFolder, + [Parameter(Mandatory = $True)] [bool] $GranularTracing, + [Parameter(Mandatory = $True)] [bool] $RunXdpTests + ) + Push-Location $WorkingDirectory + # Load other utility modules. + Import-Module $WorkingDirectory\common.psm1 -Force -ArgumentList ($LogFileName) -WarningAction SilentlyContinue + + if ($ExecuteOnVM) { + Write-Log "Tests will be executed on VM" -ForegroundColor Cyan + $VMList = $Config.VMMap.$SelfHostedRunnerName + $VMName = $VMList[0].Name + $TestWorkingDirectory = "C:\ebpf" + } else { + Write-Log "Executing on host" -ForegroundColor Cyan + $VMName = $null + $TestWorkingDirectory = $WorkingDirectory + } + Import-Module $WorkingDirectory\vm_run_tests.psm1 ` + -Force ` + -ArgumentList( + $ExecuteOnHost, + $ExecuteOnVM, + $VMIsRemote, + $VMName, + $TestWorkingDirectory, + $LogFileName, + $TestMode, + $Options, + $TestHangTimeout, + $UserModeDumpFolder, + $GranularTracing, + $RunXdpTests) ` + -WarningAction SilentlyContinue + try { + Write-Log "Running kernel tests" + Run-KernelTests -Config $Config + Write-Log "Running kernel tests completed" + + Stop-eBPFComponents -GranularTracing $GranularTracing + } catch [System.Management.Automation.RemoteException] { + Write-Log $_.Exception.Message + Write-Log $_.ScriptStackTrace + if ($_.CategoryInfo.Reason -eq "TimeoutException" -or $_.Exception.Message -match "timed out|timeout") { + Generate-KernelDumpOnVM + } + throw $_.Exception.Message + } + Pop-Location +} -ArgumentList ( + $ExecuteOnHost, + $ExecuteOnVM, + $VMIsRemote, + $Config, + $SelfHostedRunnerName, + $WorkingDirectory, + $LogFileName, + $TestMode, + $Options, + $TestHangTimeout, + $UserModeDumpFolder, + $GranularTracing, + $RunXdpTests) + +# Keep track of the last received output count +$JobTimedOut = ` + Wait-TestJobToComplete -Job $Job ` + -Config $Config ` + -SelfHostedRunnerName $SelfHostedRunnerName ` + -TestJobTimeout $TestJobTimeout ` + -CheckpointPrefix "Execute" ` + -ExecuteOnHost $ExecuteOnHost ` + -ExecuteOnVM $ExecuteOnVM ` + -VMIsRemote $VMIsRemote ` + -TestWorkingDirectory $(if ($ExecuteOnVM) { "C:\ebpf" } else { $WorkingDirectory }) ` + -LogFileName $LogFileName ` + -TestMode $TestMode ` + -Options $Options ` + -TestHangTimeout $TestHangTimeout ` + -UserModeDumpFolder $UserModeDumpFolder ` + -GenerateKernelDumpOnTimeout $true + +# Re-import common.psm1 in case the kernel-dump timeout handler inside +# Wait-TestJobToComplete forcefully re-imported it (via vm_run_tests.psm1), +# which removes it from this script's scope. +Import-Module $WorkingDirectory\common.psm1 -Force -ArgumentList ($LogFileName) -ErrorAction SilentlyContinue + +# Check job result before cleanup. +$JobFailed = $Job.State -eq 'Failed' +if ($JobFailed) { + try { + $childJob = $Job.ChildJobs[0] + if ($childJob -and $childJob.JobStateInfo.Reason) { + Write-Log "Test job failed: $($childJob.JobStateInfo.Reason.Message)" + } + } catch { + Write-Log "Warning: Failed to read test job state: $($_.Exception.Message)" + } + # Drain any remaining output from the failed job. + try { + Receive-Job -Job $Job -ErrorAction SilentlyContinue + } catch { + Write-Log "Warning: Failed to receive test job output: $($_.Exception.Message)" + } +} + +# Safe cleanup (bounded to prevent hangs on stuck transports). +Remove-JobSafely -Job $Job + +Pop-Location + +if ($JobTimedOut -or $JobFailed) { + if ($JobTimedOut) { Write-Log "Exiting with error: job timed out" } + if ($JobFailed) { Write-Log "Exiting with error: job failed" } + exit 1 +} + +Write-Log "execute_ebpf_cicd_tests.ps1 completed successfully" \ No newline at end of file diff --git a/scripts/install_ebpf.psm1 b/scripts/install_ebpf.psm1 new file mode 100644 index 00000000..873474ec --- /dev/null +++ b/scripts/install_ebpf.psm1 @@ -0,0 +1,88 @@ +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: MIT + +param( + [Parameter(Mandatory = $true)][string] $WorkingDirectory, + [Parameter(Mandatory = $true)][string] $LogFileName +) + +Import-Module "$PSScriptRoot\common.psm1" -Force -ArgumentList $LogFileName -WarningAction SilentlyContinue + +function Install-eBPFComponents +{ + param( + [Parameter(Mandatory = $true)][bool] $KmTracing, + [Parameter(Mandatory = $true)][string] $KmTraceType, + [Parameter(Mandatory = $false)][bool] $KMDFVerifier = $false, + [Parameter(Mandatory = $true)][string] $TestMode, + [Parameter(Mandatory = $false)][switch] $SkipRebootOperations, + [Parameter(Mandatory = $false)][bool] $GranularTracing = $false + ) + + $msiPath = Join-Path $WorkingDirectory "ebpf-for-windows.msi" + if (-not (Test-Path $msiPath -PathType Leaf)) { + throw "Required eBPF for Windows installer was not found at '$msiPath'." + } + + Write-Log "Installing eBPF for Windows from $msiPath" + $arguments = @("/i", $msiPath, "ADDLOCAL=ALL", "/qn", "/norestart", "/l*v", "msi-install.log") + $process = Start-Process -FilePath msiexec.exe -ArgumentList $arguments -Wait -PassThru + if ($process.ExitCode -notin @(0, 3010)) { + if (Test-Path "msi-install.log") { + Get-Content "msi-install.log" | Write-Log + } + throw "eBPF for Windows installation failed with exit code $($process.ExitCode)." + } + + if ($process.ExitCode -eq 3010) { + Write-Log "eBPF for Windows installation requested a reboot; continuing because the package was installed with /norestart." + } + + $exporterPaths = @() + foreach ($exporterName in @("ntos_ebpf_ext_export_program_info.exe", "netevent_ebpf_ext_export_program_info.exe")) { + $exporterPath = Join-Path $WorkingDirectory $exporterName + if (-not (Test-Path $exporterPath -PathType Leaf)) { + throw "Required eBPF store exporter was not found at '$exporterPath'." + } + $exporterPaths += $exporterPath + } + + Write-Log "Clearing existing eBPF store information" + & $exporterPaths[0] --clear 2>&1 | Write-Log + if ($LASTEXITCODE -ne 0) { + throw "$([System.IO.Path]::GetFileName($exporterPaths[0])) --clear failed with exit code $LASTEXITCODE." + } + + foreach ($exporterPath in $exporterPaths) { + $exporterName = [System.IO.Path]::GetFileName($exporterPath) + Write-Log "Registering extension program information with $exporterName" + & $exporterPath 2>&1 | Write-Log + if ($LASTEXITCODE -ne 0) { + throw "$exporterName failed with exit code $LASTEXITCODE." + } + } + + if ($KMDFVerifier) { + Write-Log "The 1ES inner VM image controls driver verifier settings; no additional verifier configuration is applied." + } + + Write-Log "eBPF for Windows installation completed." +} + +function Stop-eBPFServiceAndDrivers +{ + param([Parameter(Mandatory = $false)][bool] $GranularTracing = $false) + + foreach ($serviceName in @("ntosebpfext", "neteventebpfext", "netevent_sim")) { + $service = Get-Service -Name $serviceName -ErrorAction SilentlyContinue + if ($service -and $service.Status -ne "Stopped") { + Write-Log "Stopping leftover test driver service $serviceName" + Stop-Service -Name $serviceName -Force -ErrorAction SilentlyContinue + } + } +} + +function Uninstall-eBPFComponents +{ + Write-Log "The inner VM is discarded after the job; eBPF for Windows is not uninstalled explicitly." +} diff --git a/scripts/run_driver_tests.psm1 b/scripts/run_driver_tests.psm1 new file mode 100644 index 00000000..f990c80b --- /dev/null +++ b/scripts/run_driver_tests.psm1 @@ -0,0 +1,91 @@ +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: MIT + +param( + [Parameter(Mandatory = $true)][string] $WorkingDirectory, + [Parameter(Mandatory = $true)][string] $LogFileName, + [Parameter(Mandatory = $false)][int] $TestHangTimeout = (30 * 60), + [Parameter(Mandatory = $false)][string] $UserModeDumpFolder = "C:\Dumps", + [Parameter(Mandatory = $false)][bool] $GranularTracing = $false +) + +Import-Module "$PSScriptRoot\common.psm1" -Force -ArgumentList $LogFileName -WarningAction SilentlyContinue + +function Invoke-NtosDriverTest +{ + param( + [Parameter(Mandatory = $true)][string] $Name, + [Parameter(Mandatory = $false)][string] $Arguments = "-d yes", + [Parameter(Mandatory = $false)][int] $Timeout = $TestHangTimeout + ) + + $testPath = Join-Path $WorkingDirectory $Name + if (-not (Test-Path $testPath -PathType Leaf)) { + throw "Driver test executable was not found at '$testPath'." + } + + $testLogs = Join-Path $WorkingDirectory "TestLogs" + New-Item -Path $testLogs -ItemType Directory -Force | Out-Null + $baseName = [System.IO.Path]::GetFileNameWithoutExtension($Name) + $stdoutPath = Join-Path $testLogs "$baseName.stdout.log" + $stderrPath = Join-Path $testLogs "$baseName.stderr.log" + + Write-Log "Executing $Name $Arguments with timeout $Timeout seconds." + $process = Start-Process ` + -FilePath $testPath ` + -ArgumentList $Arguments ` + -WorkingDirectory $WorkingDirectory ` + -RedirectStandardOutput $stdoutPath ` + -RedirectStandardError $stderrPath ` + -PassThru + + # Cache the process handle so its exit information remains available after termination. + $handle = $process.Handle + + $exited = $process.WaitForExit($Timeout * 1000) + if (-not $exited -or -not $process.HasExited) { + Write-Log "$Name exceeded its timeout. A kernel dump will be requested." + throw [System.TimeoutException]::new("$Name timed out after $Timeout seconds.") + } + + $exitCode = $process.ExitCode + + if (Test-Path $stdoutPath) { + Get-Content $stdoutPath | Write-Log + } + if ((Test-Path $stderrPath) -and (Get-Item $stderrPath).Length -gt 0) { + Get-Content $stderrPath | Write-Log + } + + if ($null -eq $exitCode) { + throw "$Name completed, but its exit code could not be read." + } + if ($exitCode -ne 0) { + throw "$Name failed with exit code $exitCode." + } + + Write-Log "$Name passed." +} + +function Invoke-NtosDriverTests +{ + param([Parameter(Mandatory = $true)][object[]] $Tests) + + foreach ($test in $Tests) { + $arguments = if ($null -ne $test.Arguments) { [string]$test.Arguments } else { "-d yes" } + $timeout = if ($null -ne $test.Timeout) { [int]$test.Timeout } else { $TestHangTimeout } + Invoke-NtosDriverTest -Name ([string]$test.Name) -Arguments $arguments -Timeout $timeout + } +} + +function Generate-KernelDump +{ + $notMyFault = Get-ChildItem -Path $WorkingDirectory -Recurse -Filter "NotMyFault64.exe" | + Select-Object -First 1 + if (-not $notMyFault) { + throw "NotMyFault64.exe was not found under '$WorkingDirectory'." + } + + Write-Log "Generating a kernel dump with $($notMyFault.FullName)." + Start-Process -FilePath $notMyFault.FullName -ArgumentList "/crash" -NoNewWindow +} diff --git a/scripts/setup_build/setup_build.vcxproj b/scripts/setup_build/setup_build.vcxproj index bb2ae6a4..69bf9ac8 100644 --- a/scripts/setup_build/setup_build.vcxproj +++ b/scripts/setup_build/setup_build.vcxproj @@ -29,6 +29,39 @@ Document + + Document + + + Document + + + Document + + + Document + + + Document + + + Document + + + Document + + + Document + + + Document + + + Document + + + Document + diff --git a/scripts/setup_ebpf_cicd_tests.ps1 b/scripts/setup_ebpf_cicd_tests.ps1 new file mode 100644 index 00000000..5a30af96 --- /dev/null +++ b/scripts/setup_ebpf_cicd_tests.ps1 @@ -0,0 +1,195 @@ +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: MIT + +param ([parameter(Mandatory = $false)][bool] $KmTracing = $true, + [parameter(Mandatory = $false)][string] $KmTraceType = "file", + [parameter(Mandatory = $false)][string] $TestMode = "CI/CD", + [parameter(Mandatory = $false)][string] $LogFileName = "TestLog.log", + [parameter(Mandatory = $false)][string] $WorkingDirectory = $pwd.ToString(), + [parameter(Mandatory = $false)][string] $RegressionArtifactsVersion = "", + [parameter(Mandatory = $false)][string] $RegressionArtifactsConfiguration = "", + [parameter(Mandatory = $false)][string] $TestExecutionJsonFileName = "test_execution.json", + [parameter(Mandatory = $false)][string] $SelfHostedRunnerName = [System.Net.Dns]::GetHostName(), + [Parameter(Mandatory = $false)][int] $TestJobTimeout = (30*60), + [Parameter(Mandatory = $false)][string] $EnableHVCI = "Off", + [Parameter(Mandatory = $false)][switch] $ExecuteOnHost, + [Parameter(Mandatory = $false)][string] $Architecture = "x64", + [Parameter(Mandatory = $false)][switch] $VMIsRemote, + [Parameter(Mandatory = $false)][switch] $GranularTracing +) + +$ExecuteOnHost = [bool]$ExecuteOnHost +$ExecuteOnVM = (-not $ExecuteOnHost) +$VMIsRemote = [bool]$VMIsRemote + +Push-Location $WorkingDirectory + +# Load other utility modules. +Import-Module .\common.psm1 -Force -ArgumentList ($LogFileName) -WarningAction SilentlyContinue + +Write-Log "ExecuteOnHost: $ExecuteOnHost" +Write-Log "ExecuteOnVM: $ExecuteOnVM" +Write-Log "VMIsRemote: $VMIsRemote" +Write-Log "TestMode: $TestMode" +Write-Log "KmTracing: $KmTracing" +Write-Log "KmTraceType: $KmTraceType" +Write-Log "EnableHVCI: $EnableHVCI" +Write-Log "GranularTracing: $GranularTracing" +Write-Log "Architecture: $Architecture" + +# Read the test execution json. +$Config = Get-Content ("{0}\{1}" -f $PSScriptRoot, $TestExecutionJsonFileName) | ConvertFrom-Json + +# Delete old log files if any. +Remove-Item "$env:TEMP\$LogFileName" -ErrorAction SilentlyContinue +Remove-Item ".\TestLogs" -Recurse -Confirm:$false -ErrorAction SilentlyContinue + +if ($TestMode -eq "Regression") { + + # Download the release artifacts for regression tests. + Get-RegressionTestArtifacts -ArtifactVersion $RegressionArtifactsVersion -Configuration $RegressionArtifactsConfiguration +} + +Get-CoreNetTools -Architecture $Architecture +Get-PSExec + +$Job = Start-Job -ScriptBlock { + param ( + [Parameter(Mandatory = $true)] [PSCustomObject] $Config, + [Parameter(Mandatory = $true)] [string] $SelfHostedRunnerName, + [parameter(Mandatory = $true)] [string] $TestMode, + [parameter(Mandatory = $true)] [string] $LogFileName, + [parameter(Mandatory = $true)] [string] $WorkingDirectory = $pwd.ToString(), + [parameter(Mandatory = $true)] [bool] $KmTracing, + [parameter(Mandatory = $true)] [string] $KmTraceType, + [parameter(Mandatory = $true)] [string] $EnableHVCI, + [parameter(Mandatory = $true)] [bool] $ExecuteOnVM, + [parameter(Mandatory = $true)] [bool] $VMIsRemote, + [parameter(Mandatory = $true)] [bool] $GranularTracing + ) + Push-Location $WorkingDirectory + + # Load other utility modules. + Import-Module .\common.psm1 -Force -ArgumentList ($LogFileName) -WarningAction SilentlyContinue + Import-Module .\config_test_vm.psm1 -Force -ArgumentList ($WorkingDirectory, $LogFileName) -WarningAction SilentlyContinue + + if ($ExecuteOnVM) { + $VMList = $Config.VMMap.$SelfHostedRunnerName + if (-not $VMIsRemote) { + # Get all local VMs to ready state. + Initialize-AllVMs -VMList $VMList -ErrorAction Stop + } + } else { + Import-Module .\install_ebpf.psm1 -Force -ArgumentList ($WorkingDirectory, $LogFileName) -WarningAction SilentlyContinue + $VMList = @() + } + + if ($ExecuteOnVM) { + # Export build artifacts to the test VMs. Attempt with a few retries. + $MaxRetryCount = 5 + for ($i = 0; $i -lt $MaxRetryCount; $i += 1) { + try { + Export-BuildArtifactsToVMs -VMList $VMList -VMIsRemote $VMIsRemote -ErrorAction Stop + break + } catch [System.Exception] { + Write-Log "Export-BuildArtifactsToVMs failed: $_" + Write-Log "Export-BuildArtifactsToVMs failed. Retrying..." + } + } + + # Enable HVCI on the test VMs if specified. + if ($EnableHVCI -eq "On") { + Write-Log "Enabling HVCI on test VMs..." + # Enable HVCI on the test VM. + foreach($VM in $VMList) { + $VMName = $VM.Name + Enable-HVCIOnVM -VMName $VMName -VMIsRemote:$VMIsRemote -ErrorAction Stop + } + } + } + + $ExecuteOnHost = -not $ExecuteOnVM + if ($ExecuteOnHost) { + # Install eBPF components on host, but skip anything that requires reboot. + # Note that installing ebpf components requires psexec which does not run in a powershell job. + Write-Log "Installing eBPF components on host" + Install-eBPFComponents -TestMode $TestMode -KmTracing $KmTracing -KmTraceType $KmTraceType -SkipRebootOperations -GranularTracing:$GranularTracing + return + } + + # The rest of the script runs only if executing on VM. + if (-not $ExecuteOnVM) { + throw "Assertion failed: ExecuteOnVM must be true." + } + + # Install eBPF Components on the test VM. + foreach($VM in $VMList) { + $VMName = $VM.Name + Install-eBPFComponentsOnVM -VMName $VMName -TestMode $TestMode -KmTracing $KmTracing -KmTraceType $KmTraceType -VMIsRemote:$VMIsRemote -GranularTracing:$GranularTracing -ErrorAction Stop + } + + # Log OS build information on the test VMs. + foreach($VM in $VMList) { + $VMName = $VM.Name + Log-OSBuildInformationOnVM -VMName $VMName -VMIsRemote:$VMIsRemote -ErrorAction Stop + } + + Pop-Location +} -ArgumentList ( + $Config, + $SelfHostedRunnerName, + $TestMode, + $LogFileName, + $WorkingDirectory, + $KmTracing, + $KmTraceType, + $EnableHVCI, + $ExecuteOnVM, + $VMIsRemote, + $GranularTracing) + +# Wait for the job to complete. +$JobTimedOut = ` + Wait-TestJobToComplete -Job $Job ` + -Config $Config ` + -SelfHostedRunnerName $SelfHostedRunnerName ` + -TestJobTimeout $TestJobTimeout ` + -CheckpointPrefix "Setup" ` + -ExecuteOnHost $ExecuteOnHost ` + -ExecuteOnVM $ExecuteOnVM ` + -VMIsRemote $VMIsRemote ` + -TestWorkingDirectory $(if ($ExecuteOnVM) { "C:\ebpf" } else { $WorkingDirectory }) ` + -LogFileName $LogFileName ` + -TestMode $TestMode ` + -Options @("None") ` + -TestHangTimeout (10*60) ` + -UserModeDumpFolder "C:\Dumps" + +# Check job result before cleanup. +$JobFailed = $Job.State -eq 'Failed' +if ($JobFailed) { + try { + $childJob = $Job.ChildJobs[0] + if ($childJob -and $childJob.JobStateInfo.Reason) { + Write-Log "Setup job failed: $($childJob.JobStateInfo.Reason.Message)" + } + } catch { + Write-Log "Warning: Failed to read setup job state: $($_.Exception.Message)" + } + try { + Receive-Job -Job $Job -ErrorAction SilentlyContinue + } catch { + Write-Log "Warning: Failed to receive setup job output: $($_.Exception.Message)" + } +} + +# Safe cleanup (bounded to prevent hangs on stuck transports). +Remove-JobSafely -Job $Job + +Pop-Location + +if ($JobTimedOut -or $JobFailed) { + if ($JobTimedOut) { Write-Log "Setup timed out" } + if ($JobFailed) { Write-Log "Setup job failed" } + exit 1 +} diff --git a/scripts/test_execution.json b/scripts/test_execution.json new file mode 100644 index 00000000..c9034879 --- /dev/null +++ b/scripts/test_execution.json @@ -0,0 +1,27 @@ +{ + "license": [ + "Copyright (c) Microsoft Corporation", + "SPDX-License-Identifier: MIT" + ], + "VMMap": { + "1ESRunner": [ + { + "Name": "runner_vm" + } + ] + }, + "Tests": [ + { + "Suite": "ntosebpfext", + "Name": "ntosebpfext_driver_test.exe", + "Arguments": "-d yes", + "Timeout": 1800 + }, + { + "Suite": "neteventebpfext", + "Name": "neteventebpfext_driver_test.exe", + "Arguments": "-d yes", + "Timeout": 1800 + } + ] +} diff --git a/scripts/tracing_utils.psm1 b/scripts/tracing_utils.psm1 new file mode 100644 index 00000000..b4a9787e --- /dev/null +++ b/scripts/tracing_utils.psm1 @@ -0,0 +1,104 @@ +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: MIT + +# Simplified module for WPR ETW tracing with automatic initialization + +param( + [Parameter(Mandatory=$true)] [string] $LogFileName, + [Parameter(Mandatory=$true)] [string] $WorkingDirectory +) + +Import-Module $WorkingDirectory\common.psm1 -Force -ArgumentList ($LogFileName) -WarningAction SilentlyContinue + +<# +.SYNOPSIS + Starts WPR tracing for a specific operation. + +.PARAMETER TraceType + The type of tracing to use ("file" or "memory"). Defaults to "file". + +.PARAMETER WprpFileName + The name of the WPRP file to use. Defaults to "ebpfforwindows.wprp". + +.PARAMETER TracingProfileName + The name of the tracing profile to use. Defaults to "EbpfForWindows-Networking". +#> +function Start-WPRTrace { + param( + [Parameter(Mandatory=$false)] [string] $TraceType = "file", + [Parameter(Mandatory=$false)] [string] $WprpFileName = "ebpfforwindows.wprp", + [Parameter(Mandatory=$false)] [string] $TracingProfileName = "EbpfForWindows-Networking", + [Parameter(Mandatory=$false)] [int] $TimeoutSeconds = 60 + ) + + try { + Write-Log "Start-WPRTrace called with TraceType: $TraceType, WorkingDirectory: $WorkingDirectory" + + # Quick cleanup of any orphaned sessions + $exitCode = Start-ProcessWithTimeout -FilePath "wpr.exe" -ArgumentList @("-cancel") -TimeoutSeconds $TimeoutSeconds + + # Build profile path and check if it exists + $wprpProfilePath = Join-Path $WorkingDirectory $WprpFileName + Write-Log "Looking for WPRP profile at: $wprpProfilePath" + if (-not (Test-Path $wprpProfilePath)) { + Write-Log "Warning: WPRP profile not found at $wprpProfilePath" -ForegroundColor Yellow + } + + Write-Log "Starting WPR trace with TraceType: $TraceType WprpFileName: $WprpFileName TracingProfileName: $TracingProfileName" + + # Prepare arguments + $profileName = if ($TraceType -eq "file") { "$TracingProfileName-File" } else { "$TracingProfileName-Memory" } + $arguments = @("-start", "$wprpProfilePath!$profileName") + if ($TraceType -eq "file") { + $arguments += "-filemode" + } + + $exitCode = Start-ProcessWithTimeout -FilePath "wpr.exe" -ArgumentList $arguments -TimeoutSeconds $TimeoutSeconds + if ($exitCode -eq 0) { + Write-Log "Successfully started trace" + } else { + Write-Log "Failed to start trace with exit code $exitCode" + } + } catch { + Write-Log "Exception starting WPR trace: $_" -ForegroundColor Red + } +} + +<# +.SYNOPSIS + Stops the currently active WPR trace. + +.PARAMETER FileName + The base filename (without timestamp and extension) to use for the ETL file. +#> +function Stop-WPRTrace { + param( + [Parameter(Mandatory=$true)] [string] $FileName, + [Parameter(Mandatory=$false)] [int] $TimeoutSeconds = 600 + ) + + try { + # Create output directory if needed + $outputDir = Join-Path $WorkingDirectory "TestLogs" + if (-not (Test-Path $outputDir)) { + New-Item -ItemType Directory -Path $outputDir -Force | Out-Null + } + + # Generate unique ETL filename with timestamp + $timestamp = Get-Date -Format "yyyyMMdd_HHmmss" + $etlFileName = "${FileName}_${timestamp}.etl" + $traceFile = Join-Path $outputDir $etlFileName + + $exitCode = Start-ProcessWithTimeout -FilePath "wpr.exe" -ArgumentList @("-stop", "`"$traceFile`"") -TimeoutSeconds $TimeoutSeconds + if ($exitCode -eq 0) { + Write-Log "Successfully stopped WPR trace: $traceFile" + return $traceFile + } else { + Write-Log "Failed to stop WPR trace with exit code $exitCode (This may be expected if no trace session was in progress)" + return $null + } + } catch { + Write-Log "Exception stopping WPR trace. This may be expected if no trace session was in progress. Error: $_" -ForegroundColor Red + return $null + } +} \ No newline at end of file diff --git a/scripts/vm_run_tests.psm1 b/scripts/vm_run_tests.psm1 new file mode 100644 index 00000000..6d87762d --- /dev/null +++ b/scripts/vm_run_tests.psm1 @@ -0,0 +1,108 @@ +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: MIT + +param( + [Parameter(Mandatory = $false)][bool] $ExecuteOnHost = $false, + [Parameter(Mandatory = $false)][bool] $ExecuteOnVM = $false, + [Parameter(Mandatory = $false)][bool] $VMIsRemote = $false, + [Parameter(Mandatory = $true)][string] $VMName, + [Parameter(Mandatory = $true)][string] $WorkingDirectory, + [Parameter(Mandatory = $true)][string] $LogFileName, + [Parameter(Mandatory = $false)][string] $TestMode = "CI/CD", + [Parameter(Mandatory = $false)][string[]] $Options = @("None"), + [Parameter(Mandatory = $false)][int] $TestHangTimeout = (30 * 60), + [Parameter(Mandatory = $false)][string] $UserModeDumpFolder = "C:\Dumps", + [Parameter(Mandatory = $false)][bool] $GranularTracing = $false, + [Parameter(Mandatory = $false)][bool] $RunXdpTests = $false +) + +Import-Module "$PSScriptRoot\common.psm1" -Force -ArgumentList $LogFileName -WarningAction SilentlyContinue + +function Invoke-OnHostOrVM +{ + param( + [Parameter(Mandatory = $true)][ScriptBlock] $ScriptBlock, + [Parameter(Mandatory = $false)][object[]] $ArgumentList = @() + ) + + if ($script:ExecuteOnHost) { + & $ScriptBlock @ArgumentList + return + } + + if (-not $script:ExecuteOnVM) { + throw "Either ExecuteOnHost or ExecuteOnVM must be true." + } + + $credential = Get-VMCredential -Username "Administrator" -VMIsRemote $script:VMIsRemote + Invoke-CommandOnVM ` + -VMName $script:VMName ` + -VMIsRemote $script:VMIsRemote ` + -Credential $credential ` + -ScriptBlock $ScriptBlock ` + -ArgumentList $ArgumentList +} + +function Run-KernelTests +{ + param([Parameter(Mandatory = $true)][PSCustomObject] $Config) + + $selectedTests = @($Config.Tests) + if ($script:Options -and ($script:Options -notcontains "None")) { + $selectedTests = @($Config.Tests | Where-Object { $script:Options -contains $_.Suite }) + } + if ($selectedTests.Count -eq 0) { + throw "No driver tests matched options: $($script:Options -join ', ')." + } + + $scriptBlock = { + param($WorkingDirectory, $LogFileName, $Tests, $TestHangTimeout, $UserModeDumpFolder) + Import-Module "$WorkingDirectory\common.psm1" -Force -ArgumentList $LogFileName -WarningAction SilentlyContinue + Import-Module "$WorkingDirectory\run_driver_tests.psm1" ` + -Force ` + -ArgumentList $WorkingDirectory, $LogFileName, $TestHangTimeout, $UserModeDumpFolder ` + -WarningAction SilentlyContinue + Invoke-NtosDriverTests -Tests $Tests + } + + Invoke-OnHostOrVM ` + -ScriptBlock $scriptBlock ` + -ArgumentList @( + $script:WorkingDirectory, + $script:LogFileName, + $selectedTests, + $script:TestHangTimeout, + $script:UserModeDumpFolder) +} + +function Stop-eBPFComponents +{ + param([Parameter(Mandatory = $false)][bool] $GranularTracing = $false) + + $scriptBlock = { + param($WorkingDirectory, $LogFileName) + Import-Module "$WorkingDirectory\common.psm1" -Force -ArgumentList $LogFileName -WarningAction SilentlyContinue + Import-Module "$WorkingDirectory\install_ebpf.psm1" ` + -Force ` + -ArgumentList $WorkingDirectory, $LogFileName ` + -WarningAction SilentlyContinue + Stop-eBPFServiceAndDrivers + } + + Invoke-OnHostOrVM -ScriptBlock $scriptBlock -ArgumentList @($script:WorkingDirectory, $script:LogFileName) +} + +function Generate-KernelDumpOnVM +{ + $scriptBlock = { + param($WorkingDirectory, $LogFileName) + Import-Module "$WorkingDirectory\common.psm1" -Force -ArgumentList $LogFileName -WarningAction SilentlyContinue + Import-Module "$WorkingDirectory\run_driver_tests.psm1" ` + -Force ` + -ArgumentList $WorkingDirectory, $LogFileName ` + -WarningAction SilentlyContinue + Generate-KernelDump + } + + Invoke-OnHostOrVM -ScriptBlock $scriptBlock -ArgumentList @($script:WorkingDirectory, $script:LogFileName) +} diff --git a/tests/neteventebpfext/netevent_sim/netevent_sim.c b/tests/neteventebpfext/netevent_sim/netevent_sim.c index 2304e2d1..06a360b1 100644 --- a/tests/neteventebpfext/netevent_sim/netevent_sim.c +++ b/tests/neteventebpfext/netevent_sim/netevent_sim.c @@ -101,10 +101,18 @@ timer_dpc_routine( .PacketDescriptor = {.PacketMetaDataLength = sizeof(PKTMON_EVT_STREAM_METADATA)}}, .payload = { .event_id = NOTIFY_EVENT_TYPE_NETEVENT_LOG, - .source_ip = {192, 168, 1, 1}, - .destination_ip = {10, 11, 12, 1}, - .source_port = 12345, - .destination_port = 80, + .source_ip = + {NETEVENT_TEST_SOURCE_IP_OCTET_1, + NETEVENT_TEST_SOURCE_IP_OCTET_2, + NETEVENT_TEST_SOURCE_IP_OCTET_3, + NETEVENT_TEST_SOURCE_IP_OCTET_4}, + .destination_ip = + {NETEVENT_TEST_DESTINATION_IP_OCTET_1, + NETEVENT_TEST_DESTINATION_IP_OCTET_2, + NETEVENT_TEST_DESTINATION_IP_OCTET_3, + NETEVENT_TEST_DESTINATION_IP_OCTET_4}, + .source_port = NETEVENT_TEST_SOURCE_PORT, + .destination_port = NETEVENT_TEST_DESTINATION_PORT, .event_counter = counter}}; if (_netevent_provider_binding_context.client_dispatch->capture_type == NeteventCapture_Drop) { diff --git a/tests/neteventebpfext/netevent_sim/netevent_types.h b/tests/neteventebpfext/netevent_sim/netevent_types.h index 591faccb..6f76e591 100644 --- a/tests/neteventebpfext/netevent_sim/netevent_types.h +++ b/tests/neteventebpfext/netevent_sim/netevent_types.h @@ -14,6 +14,17 @@ typedef struct _EX_RUNDOWN_REF_CACHE_AWARE* PEX_RUNDOWN_REF_CACHE_AWARE; #define NOTIFY_EVENT_TYPE_NETEVENT_DROP 100 #define NOTIFY_EVENT_TYPE_NETEVENT_LOG 101 +#define NETEVENT_TEST_SOURCE_IP_OCTET_1 192 +#define NETEVENT_TEST_SOURCE_IP_OCTET_2 168 +#define NETEVENT_TEST_SOURCE_IP_OCTET_3 1 +#define NETEVENT_TEST_SOURCE_IP_OCTET_4 1 +#define NETEVENT_TEST_DESTINATION_IP_OCTET_1 10 +#define NETEVENT_TEST_DESTINATION_IP_OCTET_2 11 +#define NETEVENT_TEST_DESTINATION_IP_OCTET_3 12 +#define NETEVENT_TEST_DESTINATION_IP_OCTET_4 1 +#define NETEVENT_TEST_SOURCE_PORT 12345 +#define NETEVENT_TEST_DESTINATION_PORT 80 + #pragma pack(push, 1) // Set packing to 1 byte boundary // IP address structure diff --git a/tests/neteventebpfext/neteventebpfext_driver_test/netevent_ebpfext_driver_test.cpp b/tests/neteventebpfext/neteventebpfext_driver_test/netevent_ebpfext_driver_test.cpp index 932c7878..d50642a2 100644 --- a/tests/neteventebpfext/neteventebpfext_driver_test/netevent_ebpfext_driver_test.cpp +++ b/tests/neteventebpfext/neteventebpfext_driver_test/netevent_ebpfext_driver_test.cpp @@ -17,6 +17,7 @@ #include "utils.h" #include "watchdog.h" +#include #include #include #include @@ -36,80 +37,140 @@ CATCH_REGISTER_LISTENER(cxplat_passed_test_log) struct bpf_map* netevent_event_map; struct bpf_map* command_map; -static volatile uint32_t event_count = 0; -static volatile uint32_t log_event_count = 0; -static volatile uint32_t drop_event_count = 0; +static std::atomic event_count = 0; +static std::atomic log_event_count = 0; +static std::atomic drop_event_count = 0; +static constexpr uint32_t wait_interval_seconds = 5; +static constexpr uint32_t timeout_seconds = 90; -typedef struct test_netevent_event_md +struct netevent_callback_context { - EBPF_CONTEXT_HEADER; - netevent_event_md_t context; -} test_netevent_event_md_t; + std::atomic expected_event_type = 0; + bool validate_simulator_payload = false; + std::atomic malformed_event_count = 0; + std::atomic invalid_version_count = 0; + std::atomic unknown_event_type_count = 0; + std::atomic unexpected_event_type_count = 0; + std::atomic mismatched_pktmon_event_id_count = 0; + std::atomic invalid_payload_count = 0; + std::atomic lost_event_count = 0; +}; + +template +static bool +_wait_for_condition(condition_t condition) +{ + for (uint32_t elapsed_seconds = 0; elapsed_seconds < timeout_seconds; elapsed_seconds += wait_interval_seconds) { + if (condition()) { + return true; + } + std::this_thread::sleep_for(std::chrono::seconds(wait_interval_seconds)); + } + return condition(); +} -void -_dump_event(uint8_t event_type, const char* event_descr, void* data, size_t size) +static bool +_wait_for_events_to_drain() { - netevent_data_header_t* header_ptr = reinterpret_cast(data); - if ((event_type == NETEVENT_EVENT_TYPE_PKTMON_DROP || event_type == NETEVENT_EVENT_TYPE_PKTMON_FLOW)) { - // Cast the event and print its details - netevent_message_t* test_message = - reinterpret_cast((static_cast(data) + sizeof(netevent_data_header_t))); - netevent_payload_t* test_payload = static_cast(&test_message->payload); - std::cout << "\rNetwork event [" << test_payload->event_counter << "]: {" - << "src: " << (int)test_payload->source_ip.octet1 << "." << (int)test_payload->source_ip.octet2 << "." - << (int)test_payload->source_ip.octet3 << "." << (int)test_payload->source_ip.octet4 << ":" - << test_payload->source_port << ", " - << "dst: " << (int)test_payload->destination_ip.octet1 << "." - << (int)test_payload->destination_ip.octet2 << "." << (int)test_payload->destination_ip.octet3 << "." - << (int)test_payload->destination_ip.octet4 << ":" << test_payload->destination_port; - std::cout << "}" << std::flush; - } else { - // Simply dump the event data as hex bytes. - std::cout << std::endl - << "\r>>> " << event_descr << " - type[" << (int)event_type << "], " << size << " bytes: { "; - for (size_t i = 0; i < size; ++i) { - std::cout << std::setw(2) << std::setfill('0') << std::hex - << static_cast(reinterpret_cast(data)[i]) << " "; + uint32_t previous_event_count = event_count.load(); + for (uint32_t elapsed_seconds = 0; elapsed_seconds < timeout_seconds; elapsed_seconds += wait_interval_seconds) { + std::this_thread::sleep_for(std::chrono::seconds(wait_interval_seconds)); + uint32_t current_event_count = event_count.load(); + if (current_event_count == previous_event_count) { + return true; } - std::cout << "}" << std::flush; - std::cout << std::dec; // Reset to decimal. + previous_event_count = current_event_count; } + return false; } +static void +_require_no_callback_errors(const netevent_callback_context& context) +{ + REQUIRE(context.malformed_event_count.load() == 0); + REQUIRE(context.invalid_version_count.load() == 0); + REQUIRE(context.unknown_event_type_count.load() == 0); + REQUIRE(context.unexpected_event_type_count.load() == 0); + REQUIRE(context.mismatched_pktmon_event_id_count.load() == 0); + REQUIRE(context.invalid_payload_count.load() == 0); + REQUIRE(context.lost_event_count.load() == 0); +} + +typedef struct test_netevent_event_md +{ + EBPF_CONTEXT_HEADER; + netevent_event_md_t context; +} test_netevent_event_md_t; + void netevent_monitor_event_callback(void* ctx, int cpu, void* data, uint32_t size) { - // Parameter checks. - UNREFERENCED_PARAMETER(ctx); UNREFERENCED_PARAMETER(cpu); - if (data == nullptr || size == 0 || size < sizeof(netevent_data_header_t)) { - std::cout << "empty event fired" << std::flush; + auto& context = *static_cast(ctx); + + if (data == nullptr || size < sizeof(netevent_data_header_t) + PKTMON_EVENT_HEADER_LENGTH) { + context.malformed_event_count++; return; } - // Check if this event is actually a netevent event (i.e. first byte is NETEVENT_EVENT_TYPE_PKTMON_DROP). - netevent_data_header_t* header_ptr = reinterpret_cast(data); + const netevent_data_header_t* header_ptr = reinterpret_cast(data); uint8_t event_type = header_ptr->type; event_count++; - std::cout << "event type fired" << (int)event_type << std::flush; + + if (header_ptr->version != NETEVENT_PKTMON_EVENT_CURRENT_VERSION) { + context.invalid_version_count++; + } + if (event_type == NETEVENT_EVENT_TYPE_PKTMON_FLOW) { log_event_count++; } else if (event_type == NETEVENT_EVENT_TYPE_PKTMON_DROP) { drop_event_count++; } else { + context.unknown_event_type_count++; + return; + } + + uint8_t expected_event_type = context.expected_event_type.load(); + if (expected_event_type != 0 && event_type != expected_event_type) { + context.unexpected_event_type_count++; + } + + const netevent_message_t* message = + reinterpret_cast(static_cast(data) + sizeof(netevent_data_header_t)); + if (message->header.EventId != event_type) { + context.mismatched_pktmon_event_id_count++; + } + + if (!context.validate_simulator_payload) { + return; + } + + if (size < sizeof(netevent_data_header_t) + sizeof(netevent_message_t)) { + context.malformed_event_count++; return; } - _dump_event(event_type, "netevent_event", data, size); - return; + const netevent_payload_t& payload = message->payload; + if (payload.event_id != event_type || payload.source_ip.octet1 != NETEVENT_TEST_SOURCE_IP_OCTET_1 || + payload.source_ip.octet2 != NETEVENT_TEST_SOURCE_IP_OCTET_2 || + payload.source_ip.octet3 != NETEVENT_TEST_SOURCE_IP_OCTET_3 || + payload.source_ip.octet4 != NETEVENT_TEST_SOURCE_IP_OCTET_4 || + payload.destination_ip.octet1 != NETEVENT_TEST_DESTINATION_IP_OCTET_1 || + payload.destination_ip.octet2 != NETEVENT_TEST_DESTINATION_IP_OCTET_2 || + payload.destination_ip.octet3 != NETEVENT_TEST_DESTINATION_IP_OCTET_3 || + payload.destination_ip.octet4 != NETEVENT_TEST_DESTINATION_IP_OCTET_4 || + payload.source_port != NETEVENT_TEST_SOURCE_PORT || + payload.destination_port != NETEVENT_TEST_DESTINATION_PORT) { + context.invalid_payload_count++; + } } void netevent_monitor_lost_event_callback(void* ctx, int cpu, __u64 cnt) { - UNREFERENCED_PARAMETER(ctx); UNREFERENCED_PARAMETER(cpu); - UNREFERENCED_PARAMETER(cnt); + auto& context = *static_cast(ctx); + context.lost_event_count.fetch_add(cnt); } TEST_CASE("netevent_attach_opt_simulation", "[neteventebpfext]") @@ -146,13 +207,16 @@ TEST_CASE("netevent_attach_opt_simulation", "[neteventebpfext]") // Attach to the eBPF perf buffer event map. bpf_map* netevent_events_map = bpf_object__find_map_by_name(object, "netevent_events_map"); REQUIRE(netevent_events_map != nullptr); + netevent_callback_context callback_context = {}; + callback_context.expected_event_type = NETEVENT_EVENT_TYPE_PKTMON_FLOW; + callback_context.validate_simulator_payload = true; ebpf_perf_buffer_opts perf_opts = {.sz = sizeof(ebpf_perf_buffer_opts), .flags = EBPF_PERFBUF_FLAG_AUTO_CALLBACK}; auto netevent_perf_buff = ebpf_perf_buffer__new( bpf_map__fd(netevent_events_map), 0, netevent_monitor_event_callback, netevent_monitor_lost_event_callback, - nullptr, + &callback_context, &perf_opts); REQUIRE(netevent_perf_buff != nullptr); @@ -182,45 +246,53 @@ TEST_CASE("netevent_attach_opt_simulation", "[neteventebpfext]") REQUIRE(netevent_monitor_link == nullptr); // Test attach with capture valid capture type - uint32_t event_count_before = event_count; - uint32_t log_event_count_before = log_event_count; - uint32_t drop_event_count_before = drop_event_count; + uint32_t event_count_before = event_count.load(); + uint32_t log_event_count_before = log_event_count.load(); attach_opts.capture_type = NeteventCapture_All; result = ebpf_program_attach( netevent_monitor, &EBPF_ATTACH_TYPE_NETEVENT, &attach_opts, sizeof(attach_opts), &netevent_monitor_link); REQUIRE(result == EBPF_SUCCESS); REQUIRE(netevent_monitor_link != nullptr); - std::this_thread::sleep_for(std::chrono::seconds(5)); + + _wait_for_condition([&]() { return log_event_count.load() > log_event_count_before; }); // Detach the program (link) from the attach point. int link_fd = bpf_link__fd(netevent_monitor_link); bpf_link_detach(link_fd); bpf_link__destroy(netevent_monitor_link); + netevent_monitor_link = nullptr; - // Test that only expected event counts have increased - std::this_thread::sleep_for(std::chrono::seconds(5)); - REQUIRE(log_event_count_before < log_event_count); - REQUIRE((event_count - event_count_before) == (log_event_count - log_event_count_before)); + REQUIRE(_wait_for_events_to_drain()); + uint32_t event_count_after = event_count.load(); + uint32_t log_event_count_after = log_event_count.load(); + REQUIRE(log_event_count_before < log_event_count_after); + REQUIRE((event_count_after - event_count_before) == (log_event_count_after - log_event_count_before)); // Test reattach with different capture type - event_count_before = event_count; + event_count_before = event_count.load(); + uint32_t drop_event_count_before = drop_event_count.load(); + callback_context.expected_event_type = NETEVENT_EVENT_TYPE_PKTMON_DROP; attach_opts.capture_type = NeteventCapture_Drop; result = ebpf_program_attach( netevent_monitor, &EBPF_ATTACH_TYPE_NETEVENT, &attach_opts, sizeof(attach_opts), &netevent_monitor_link); REQUIRE(result == EBPF_SUCCESS); REQUIRE(netevent_monitor_link != nullptr); - std::this_thread::sleep_for(std::chrono::seconds(5)); + + _wait_for_condition([&]() { return drop_event_count.load() > drop_event_count_before; }); // Detach the program (link) from the attach point. link_fd = bpf_link__fd(netevent_monitor_link); bpf_link_detach(link_fd); bpf_link__destroy(netevent_monitor_link); + netevent_monitor_link = nullptr; - // Test that only expected event counts have increased - std::this_thread::sleep_for(std::chrono::seconds(5)); - REQUIRE(drop_event_count_before < drop_event_count); - REQUIRE((event_count - event_count_before) == (drop_event_count - drop_event_count_before)); + REQUIRE(_wait_for_events_to_drain()); + event_count_after = event_count.load(); + uint32_t drop_event_count_after = drop_event_count.load(); + REQUIRE(drop_event_count_before < drop_event_count_after); + REQUIRE((event_count_after - event_count_before) == (drop_event_count_after - drop_event_count_before)); + _require_no_callback_errors(callback_context); // Close perf buffer. perf_buffer__free(netevent_perf_buff); @@ -278,13 +350,16 @@ TEST_CASE("netevent_drivers_load_unload_stress", "[neteventebpfext]") // Attach to the eBPF perf buffer event map. bpf_map* netevent_events_map = bpf_object__find_map_by_name(object, "netevent_events_map"); REQUIRE(netevent_events_map != nullptr); + netevent_callback_context callback_context = {}; + callback_context.expected_event_type = NETEVENT_EVENT_TYPE_PKTMON_FLOW; + callback_context.validate_simulator_payload = true; ebpf_perf_buffer_opts perf_opts = {.sz = sizeof(ebpf_perf_buffer_opts), .flags = EBPF_PERFBUF_FLAG_AUTO_CALLBACK}; auto netevent_perf_buff = ebpf_perf_buffer__new( bpf_map__fd(netevent_events_map), 0, netevent_monitor_event_callback, netevent_monitor_lost_event_callback, - nullptr, + &callback_context, &perf_opts); REQUIRE(netevent_perf_buff != nullptr); @@ -345,6 +420,8 @@ TEST_CASE("netevent_drivers_load_unload_stress", "[neteventebpfext]") int link_fd = bpf_link__fd(netevent_monitor_link); bpf_link_detach(link_fd); bpf_link__destroy(netevent_monitor_link); + REQUIRE(_wait_for_events_to_drain()); + _require_no_callback_errors(callback_context); // Close perf buffer. perf_buffer__free(netevent_perf_buff); @@ -395,13 +472,15 @@ TEST_CASE("netevent_bpf_prog_run_test", "[neteventebpfext]") // Attach to the eBPF perf buffer event map. bpf_map* netevent_events_map = bpf_object__find_map_by_name(object, "netevent_events_map"); REQUIRE(netevent_events_map != nullptr); + netevent_callback_context callback_context = {}; + callback_context.expected_event_type = NETEVENT_EVENT_TYPE_PKTMON_DROP; ebpf_perf_buffer_opts perf_opts = {.sz = sizeof(ebpf_perf_buffer_opts), .flags = EBPF_PERFBUF_FLAG_AUTO_CALLBACK}; auto netevent_perf_buff = ebpf_perf_buffer__new( bpf_map__fd(netevent_events_map), 0, netevent_monitor_event_callback, netevent_monitor_lost_event_callback, - nullptr, + &callback_context, &perf_opts); REQUIRE(netevent_perf_buff != nullptr); @@ -507,6 +586,8 @@ TEST_CASE("netevent_bpf_prog_run_test", "[neteventebpfext]") REQUIRE(link_fd != ebpf_fd_invalid); REQUIRE(bpf_link_detach(link_fd) == 0); REQUIRE(bpf_link__destroy(netevent_monitor_link) == 0); + REQUIRE(_wait_for_events_to_drain()); + _require_no_callback_errors(callback_context); // Free the perf buffer manager perf_buffer__free(netevent_perf_buff); diff --git a/tools/netevent_ebpf_ext_export_program_info/main.cpp b/tools/netevent_ebpf_ext_export_program_info/main.cpp index 17bfe4fa..31cbb7aa 100644 --- a/tools/netevent_ebpf_ext_export_program_info/main.cpp +++ b/tools/netevent_ebpf_ext_export_program_info/main.cpp @@ -33,18 +33,21 @@ main(int argc, char** argv) status = export_all_program_information(); if (status != ERROR_SUCCESS) { std::cout << "Failed export_all_program_information() - ERROR #" << status << std::endl; + return 1; } std::cout << "Exporting section information." << std::endl; status = export_all_section_information(); if (status != ERROR_SUCCESS) { std::cout << "Failed export_all_section_information() - ERROR #" << status << std::endl; + return 1; } } else { std::cout << "Clearing eBPF store." << std::endl; status = clear_ebpf_store(); if (status != EBPF_SUCCESS) { std::cout << "Failed clear_ebpf_store() - ERROR #" << status << std::endl; + return 1; } } diff --git a/tools/ntos_ebpf_ext_export_program_info/main.cpp b/tools/ntos_ebpf_ext_export_program_info/main.cpp index f7374a76..2e1cd967 100644 --- a/tools/ntos_ebpf_ext_export_program_info/main.cpp +++ b/tools/ntos_ebpf_ext_export_program_info/main.cpp @@ -33,18 +33,21 @@ main(int argc, char** argv) status = export_all_program_information(); if (status != ERROR_SUCCESS) { std::cout << "Failed export_all_program_information() - ERROR #" << status << std::endl; + return 1; } std::cout << "Exporting section information." << std::endl; status = export_all_section_information(); if (status != ERROR_SUCCESS) { std::cout << "Failed export_all_section_information() - ERROR #" << status << std::endl; + return 1; } } else { std::cout << "Clearing eBPF store." << std::endl; status = clear_ebpf_store(); if (status != EBPF_SUCCESS) { std::cout << "Failed clear_ebpf_store() - ERROR #" << status << std::endl; + return 1; } }