From 2b77ce120e97d8dfa9f087de8619453c4eb0e516 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Tue, 10 Mar 2026 10:00:58 +0100 Subject: [PATCH 1/3] Speed up slowest acceptance tests and add --no-build consistently - Reduce HangDumpChildProcesses TestTimeout from 15s to 10s (saves ~5s per parameterization). The child processes hang immediately, so 10s is plenty. - Add --no-build to second InvokeDotnetTest call in RunSettingsAreLoadedFromProject to skip redundant rebuild when only runsettings change. - Standardize --no-build placement right after the path argument in all InvokeDotnetTest calls for consistency. - Remove unused using directive in DotnetTestMSBuildOutputTests.cs. - Add trx-analysis skill for parsing TRX test result files. Fixes #15468 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/skills/trx-analysis/SKILL.md | 104 ++++++++++++++++++ .../BlameDataCollectorTests.cs | 2 +- .../DotnetTestMSBuildOutputTests.cs | 7 +- .../DotnetTestTests.cs | 12 +- .../RunsettingsTests.cs | 4 +- 5 files changed, 116 insertions(+), 13 deletions(-) create mode 100644 .github/skills/trx-analysis/SKILL.md diff --git a/.github/skills/trx-analysis/SKILL.md b/.github/skills/trx-analysis/SKILL.md new file mode 100644 index 0000000000..177f39a9e5 --- /dev/null +++ b/.github/skills/trx-analysis/SKILL.md @@ -0,0 +1,104 @@ +--- +name: trx-analysis +description: Parse and analyze Visual Studio TRX test result files. Use when asked about slow tests, test durations, test frequency, flaky tests, failure analysis, or test execution patterns from TRX files. +--- + +# TRX Test Results Analysis + +Parse `.trx` files (Visual Studio Test Results XML) to answer questions about test performance, frequency, failures, and patterns. + +## TRX File Format + +TRX files use XML namespace `http://microsoft.com/schemas/VisualStudio/TeamTest/2010`. Key elements: + +- `TestRun.Results.UnitTestResult` — individual test executions with `testName`, `duration` (HH:mm:ss.fffffff), `outcome` (Passed/Failed/NotExecuted) +- `TestRun.TestDefinitions.UnitTest` — test metadata including class and method info +- `TestRun.ResultSummary` — aggregate pass/fail/skip counts + +## Loading a TRX File + +```powershell +[xml]$trx = Get-Content "path/to/file.trx" +$results = $trx.TestRun.Results.UnitTestResult +``` + +## Common Queries + +### Top N slowest tests + +```powershell +$results | ForEach-Object { + [PSCustomObject]@{ + Test = $_.testName + Seconds = [TimeSpan]::Parse($_.duration).TotalSeconds + Outcome = $_.outcome + } +} | Sort-Object Seconds -Descending | Select-Object -First 25 | + Format-Table @{L='Sec';E={'{0,6:N1}' -f $_.Seconds}}, Outcome, Test -AutoSize +``` + +### Slowest test from each distinct class (top N) + +```powershell +$results | ForEach-Object { + $parts = $_.testName -split '\.' + [PSCustomObject]@{ + Test = $_.testName + ClassName = ($parts[0..($parts.Length-2)] -join '.') + Seconds = [TimeSpan]::Parse($_.duration).TotalSeconds + } +} | Sort-Object Seconds -Descending | + Group-Object ClassName | ForEach-Object { $_.Group | Select-Object -First 1 } | + Sort-Object Seconds -Descending | Select-Object -First 10 | + Format-Table @{L='Sec';E={'{0,6:N1}' -f $_.Seconds}}, ClassName, Test -AutoSize +``` + +### Most-executed tests (parameterization frequency) + +Extract the base method name before parameterization and count runs: + +```powershell +$results | ForEach-Object { + $name = $_.testName + if ($name -match '^(\S+?)[\s(]') { $base = $Matches[1] } else { $base = $name } + [PSCustomObject]@{ Base = $base; Seconds = [TimeSpan]::Parse($_.duration).TotalSeconds } +} | Group-Object Base | ForEach-Object { + [PSCustomObject]@{ + Runs = $_.Count + TotalSec = ($_.Group | Measure-Object Seconds -Sum).Sum + Test = $_.Name + } +} | Sort-Object TotalSec -Descending | Select-Object -First 20 | + Format-Table @{L='Runs';E={$_.Runs}}, @{L='TotalSec';E={'{0,7:N1}' -f $_.TotalSec}}, Test -AutoSize +``` + +### Failed tests + +```powershell +$results | Where-Object { $_.outcome -eq 'Failed' } | ForEach-Object { + [PSCustomObject]@{ + Test = $_.testName + Seconds = [TimeSpan]::Parse($_.duration).TotalSeconds + Error = $_.Output.ErrorInfo.Message + } +} | Format-Table -Wrap +``` + +### Summary statistics + +```powershell +$summary = $trx.TestRun.ResultSummary.Counters +[PSCustomObject]@{ + Total = $summary.total + Passed = $summary.passed + Failed = $summary.failed + Skipped = $summary.notExecuted + Duration = $trx.TestRun.Times.finish +} | Format-List +``` + +## Tips + +- Parameterized tests appear as separate `UnitTestResult` entries. Use regex `'^(\S+?)[\s(]'` to extract the base method name. +- Sort by **TotalSec** (runs × avg duration) to find tests that consume the most CI time overall, even if each individual run is fast. +- TRX files from CI are typically found in `TestResults/` or as pipeline artifacts. diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/BlameDataCollectorTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/BlameDataCollectorTests.cs index 7bc5dbc814..a5567d74b4 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/BlameDataCollectorTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/BlameDataCollectorTests.cs @@ -247,7 +247,7 @@ public void HangDumpChildProcesses(RunnerInfo runnerInfo) var assemblyPaths = GetAssetFullPath("child-hang.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); - arguments = string.Concat(arguments, $@" /Blame:""CollectHangDump;HangDumpType=full;TestTimeout=15s"""); + arguments = string.Concat(arguments, $@" /Blame:""CollectHangDump;HangDumpType=full;TestTimeout=10s"""); InvokeVsTest(arguments); ValidateDump(2); diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestMSBuildOutputTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestMSBuildOutputTests.cs index d61602db26..976ab65244 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestMSBuildOutputTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestMSBuildOutputTests.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.IO; -using Microsoft.TestPlatform.TestUtilities; using Microsoft.VisualStudio.TestTools.UnitTesting; namespace Microsoft.TestPlatform.AcceptanceTests; @@ -26,7 +25,7 @@ public void MSBuildLoggerCanBeEnabledByBuildPropertyAndDoesNotEatSpecialChars(Ru var projectPath = GetIsolatedTestAsset("TerminalLoggerTestProject.csproj"); // Forcing terminal logger so we can see the output when it is redirected - InvokeDotnetTest($@"{projectPath} -tl:on -nodereuse:false /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion}", workingDirectory: Path.GetDirectoryName(projectPath)); + InvokeDotnetTest($@"{projectPath} --no-build -tl:on", workingDirectory: Path.GetDirectoryName(projectPath)); // The output: // Determining projects to restore... @@ -55,7 +54,7 @@ public void MSBuildLoggerCanBeDisabledByBuildProperty(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var projectPath = GetIsolatedTestAsset("TerminalLoggerTestProject.csproj"); - InvokeDotnetTest($@"{projectPath} -nodereuse:false /p:VsTestUseMSBuildOutput=false /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion}", workingDirectory: Path.GetDirectoryName(projectPath)); + InvokeDotnetTest($@"{projectPath} --no-build -nodereuse:false /p:VsTestUseMSBuildOutput=false", workingDirectory: Path.GetDirectoryName(projectPath)); // Check that we see the summary that is printed from the console logger, meaning the new output is disabled. StdOutputContains("Failed! - Failed: 1, Passed: 1, Skipped: 1, Total: 3, Duration:"); @@ -75,7 +74,7 @@ public void MSBuildLoggerCanBeDisabledByEnvironmentVariableProperty(RunnerInfo r SetTestEnvironment(_testEnvironment, runnerInfo); var projectPath = GetIsolatedTestAsset("TerminalLoggerTestProject.csproj"); - InvokeDotnetTest($@"{projectPath} -nodereuse:false /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion}", environmentVariables: new Dictionary { ["MSBUILDENSURESTDOUTFORTASKPROCESSES"] = "1" }, workingDirectory: Path.GetDirectoryName(projectPath)); + InvokeDotnetTest($@"{projectPath} --no-build -nodereuse:false", environmentVariables: new Dictionary { ["MSBUILDENSURESTDOUTFORTASKPROCESSES"] = "1" }, workingDirectory: Path.GetDirectoryName(projectPath)); // Check that we see the summary that is printed from the console logger, meaning the new output is disabled. StdOutputContains("Failed! - Failed: 1, Passed: 1, Skipped: 1, Total: 3, Duration:"); diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestTests.cs index 36b81f4201..0d89d68e59 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestTests.cs @@ -27,7 +27,7 @@ public void RunDotnetTestWithCsproj(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var projectPath = GetIsolatedTestAsset("SimpleTestProject.csproj"); - InvokeDotnetTest($@"{projectPath} -tl:off /p:VSTestNoLogo=false --logger:""Console;Verbosity=normal"" /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion}", workingDirectory: Path.GetDirectoryName(projectPath)); + InvokeDotnetTest($@"{projectPath} --no-build -tl:off /p:VSTestNoLogo=false --logger:""Console;Verbosity=normal""", workingDirectory: Path.GetDirectoryName(projectPath)); // ensure our dev version is used StdOutputContains(GetFinalVersion(IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion)); @@ -44,7 +44,7 @@ public void RunDotnetTestWithDll(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPath = GetAssetFullPath("SimpleTestProject.dll"); - InvokeDotnetTest($@"{assemblyPath} --logger:""Console;Verbosity=normal""", workingDirectory: Path.GetDirectoryName(assemblyPath)); + InvokeDotnetTest($@"{assemblyPath} --no-build --logger:""Console;Verbosity=normal""", workingDirectory: Path.GetDirectoryName(assemblyPath)); // ensure our dev version is used StdOutputContains(GetFinalVersion(IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion)); @@ -61,7 +61,7 @@ public void RunDotnetTestWithCsprojPassInlineSettings(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var projectPath = GetIsolatedTestAsset("ParametrizedTestProject.csproj"); - InvokeDotnetTest($@"{projectPath} --logger:""Console;Verbosity=normal"" -tl:off /p:VSTestNoLogo=false /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion} -- TestRunParameters.Parameter(name =\""weburl\"", value=\""http://localhost//def\"")", workingDirectory: Path.GetDirectoryName(projectPath)); + InvokeDotnetTest($@"{projectPath} --no-build --logger:""Console;Verbosity=normal"" -tl:off /p:VSTestNoLogo=false -- TestRunParameters.Parameter(name =\""weburl\"", value=\""http://localhost//def\"")", workingDirectory: Path.GetDirectoryName(projectPath)); // ensure our dev version is used StdOutputContains(GetFinalVersion(IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion)); @@ -78,7 +78,7 @@ public void RunDotnetTestWithDllPassInlineSettings(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPath = GetAssetFullPath("ParametrizedTestProject.dll"); - InvokeDotnetTest($@"{assemblyPath} --logger:""Console;Verbosity=normal"" -- TestRunParameters.Parameter(name=\""weburl\"", value=\""http://localhost//def\"")", workingDirectory: Path.GetDirectoryName(assemblyPath)); + InvokeDotnetTest($@"{assemblyPath} --no-build --logger:""Console;Verbosity=normal"" -- TestRunParameters.Parameter(name=\""weburl\"", value=\""http://localhost//def\"")", workingDirectory: Path.GetDirectoryName(assemblyPath)); ValidateSummaryStatus(1, 0, 0); ExitCodeEquals(0); @@ -96,7 +96,7 @@ public void RunDotnetTestWithNativeDll(RunnerInfo runnerInfo) string assemblyRelativePath = @"microsoft.testplatform.testasset.nativecpp\2.0.0\contentFiles\any\any\x64\Microsoft.TestPlatform.TestAsset.NativeCPP.dll"; var assemblyAbsolutePath = Path.Combine(_testEnvironment.PackageDirectory, assemblyRelativePath); - InvokeDotnetTest($@"{assemblyAbsolutePath} --logger:""Console;Verbosity=normal"" --diag:c:\temp\logscpp\", workingDirectory: Path.GetDirectoryName(assemblyAbsolutePath)); + InvokeDotnetTest($@"{assemblyAbsolutePath} --no-build --logger:""Console;Verbosity=normal"" --diag:c:\temp\logscpp\", workingDirectory: Path.GetDirectoryName(assemblyAbsolutePath)); ValidateSummaryStatus(1, 1, 0); ExitCodeEquals(1); @@ -111,7 +111,7 @@ public void RunDotnetTestAndSeeOutputFromConsoleWriteLine(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPath = GetAssetFullPath("OutputtingTestProject.dll"); - InvokeDotnetTest($@"{assemblyPath} --logger:""Console;Verbosity=normal"" ", workingDirectory: Path.GetDirectoryName(assemblyPath)); + InvokeDotnetTest($@"{assemblyPath} --no-build --logger:""Console;Verbosity=normal""", workingDirectory: Path.GetDirectoryName(assemblyPath)); StdOutputContains("MY OUTPUT FROM TEST"); diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/RunsettingsTests.cs index 3054994bc6..ceb95d8b73 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/RunsettingsTests.cs @@ -512,7 +512,7 @@ public void RunSettingsAreLoadedFromProject(RunnerInfo runnerInfo) var projectName = "ProjectFileRunSettingsTestProject.csproj"; var projectPath = GetIsolatedTestAsset(projectName); - InvokeDotnetTest($@"{projectPath} /p:VSTestUseMSBuildOutput=false --logger:""Console;Verbosity=normal"" /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion}", workingDirectory: Path.GetDirectoryName(projectPath)); + InvokeDotnetTest($@"{projectPath} --no-build /p:VSTestUseMSBuildOutput=false --logger:""Console;Verbosity=normal""", workingDirectory: Path.GetDirectoryName(projectPath)); ValidateSummaryStatus(0, 1, 0); // make sure that we can revert the project settings back by providing a config from command line @@ -520,7 +520,7 @@ public void RunSettingsAreLoadedFromProject(RunnerInfo runnerInfo) // are honored by dotnet test, instead of just using the default, which would produce the same // result var settingsPath = GetProjectAssetFullPath(projectName, "inconclusive.runsettings"); - InvokeDotnetTest($@"{projectPath} --settings {settingsPath} /p:VSTestUseMSBuildOutput=false --logger:""Console;Verbosity=normal"" /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion}", workingDirectory: Path.GetDirectoryName(projectPath)); + InvokeDotnetTest($@"{projectPath} --no-build --settings {settingsPath} /p:VSTestUseMSBuildOutput=false --logger:""Console;Verbosity=normal""", workingDirectory: Path.GetDirectoryName(projectPath)); ValidateSummaryStatus(0, 0, 1); } From 23cc0118714b559520a778d81b5709e670234113 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Tue, 10 Mar 2026 10:39:48 +0100 Subject: [PATCH 2/3] Improve skill --- .github/skills/trx-analysis/SKILL.md | 83 ++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/.github/skills/trx-analysis/SKILL.md b/.github/skills/trx-analysis/SKILL.md index 177f39a9e5..efd9149b6c 100644 --- a/.github/skills/trx-analysis/SKILL.md +++ b/.github/skills/trx-analysis/SKILL.md @@ -97,8 +97,91 @@ $summary = $trx.TestRun.ResultSummary.Counters } | Format-List ``` +## Cross-File Duplicate Analysis + +Compare two TRX files to find tests that appear in both and ran (were not skipped) in both. Useful for identifying redundant CI work across different configurations (e.g., net9.0 x64 vs net48 x86). + +### Load and find duplicates that ran in both files + +```powershell +[xml]$trx1 = Get-Content "path/to/file1.trx" +[xml]$trx2 = Get-Content "path/to/file2.trx" + +$r1 = $trx1.TestRun.Results.UnitTestResult +$r2 = $trx2.TestRun.Results.UnitTestResult + +# Build lookup: testName -> (outcome, duration) keeping best outcome per name +function Get-TestLookup($results) { + $lookup = @{} + foreach ($r in $results) { + $name = $r.testName + $outcome = $r.outcome + $dur = [TimeSpan]::Parse($r.duration) + if (-not $lookup.ContainsKey($name) -or ($lookup[$name].Outcome -eq 'NotExecuted' -and $outcome -ne 'NotExecuted')) { + $lookup[$name] = [PSCustomObject]@{ Outcome = $outcome; Duration = $dur } + } + } + $lookup +} + +$t1 = Get-TestLookup $r1 +$t2 = Get-TestLookup $r2 + +$skipped = @('NotExecuted','Pending','Disconnected','Warning','InProgress','Inconclusive') +$common = $t1.Keys | Where-Object { $t2.ContainsKey($_) -and $t1[$_].Outcome -notin $skipped -and $t2[$_].Outcome -notin $skipped } +``` + +### Separate non-parametrized vs parametrized duplicates + +Parametrized tests contain `(` in their name (e.g., `RunAllTests (Row: 0, Runner = net10.0, ...)`). The base method name is everything before the first `(`. + +```powershell +$nonParam = $common | Where-Object { $_ -notmatch '\(' } +$param = $common | Where-Object { $_ -match '\(' } +``` + +### Non-parametrized duplicates ordered by duration + +```powershell +$nonParam | ForEach-Object { + $d1 = $t1[$_].Duration; $d2 = $t2[$_].Duration + [PSCustomObject]@{ + Test = $_ + File1Sec = $d1.TotalSeconds + File2Sec = $d2.TotalSeconds + TotalSec = $d1.TotalSeconds + $d2.TotalSeconds + } +} | Sort-Object TotalSec -Descending | + Format-Table @{L='File1';E={'{0,6:N1}' -f $_.File1Sec}}, + @{L='File2';E={'{0,6:N1}' -f $_.File2Sec}}, + @{L='Total';E={'{0,6:N1}' -f $_.TotalSec}}, Test -AutoSize +``` + +### Parametrized duplicates squashed by base method + +Tests with `(Row: ...)` or other parameterization are instances of the same test. Squash them into one row per base method, showing variant count, max single-instance duration, and total duration across all instances in both files. + +```powershell +$param | ForEach-Object { + if ($_ -match '^(.+?)\s*\(') { $base = $Matches[1] } else { $base = $_ } + $d1 = $t1[$_].Duration; $d2 = $t2[$_].Duration + [PSCustomObject]@{ Base = $base; D1 = $d1.TotalSeconds; D2 = $d2.TotalSeconds; Max = [Math]::Max($d1.TotalSeconds, $d2.TotalSeconds) } +} | Group-Object Base | ForEach-Object { + [PSCustomObject]@{ + Test = $_.Name + Variants = $_.Count + OneInstance = ($_.Group | Measure-Object Max -Maximum).Maximum + AllInstances = ($_.Group | Measure-Object { $_.D1 + $_.D2 } -Sum).Sum + } +} | Sort-Object AllInstances -Descending | + Format-Table @{L='Variants';E={$_.Variants}}, + @{L='1 Instance';E={'{0,7:N1}s' -f $_.OneInstance}}, + @{L='All Instances';E={'{0,7:N1}s' -f $_.AllInstances}}, Test -AutoSize +``` + ## Tips - Parameterized tests appear as separate `UnitTestResult` entries. Use regex `'^(\S+?)[\s(]'` to extract the base method name. - Sort by **TotalSec** (runs × avg duration) to find tests that consume the most CI time overall, even if each individual run is fast. +- When comparing files, filter out `NotExecuted` tests — many parameterized tests are skipped in one configuration but not the other, so raw name overlap overstates true duplication. - TRX files from CI are typically found in `TestResults/` or as pipeline artifacts. From bd6ef25d102e55acf30d9b33e96a33a43b0d1588 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Tue, 10 Mar 2026 10:42:11 +0100 Subject: [PATCH 3/3] Add --no-restore to all InvokeDotnetTest calls Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../DotnetTestMSBuildOutputTests.cs | 8 ++++---- .../DotnetTestTests.cs | 12 ++++++------ .../RunsettingsTests.cs | 6 +++--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestMSBuildOutputTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestMSBuildOutputTests.cs index 976ab65244..3ba0930695 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestMSBuildOutputTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestMSBuildOutputTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Collections.Generic; @@ -25,7 +25,7 @@ public void MSBuildLoggerCanBeEnabledByBuildPropertyAndDoesNotEatSpecialChars(Ru var projectPath = GetIsolatedTestAsset("TerminalLoggerTestProject.csproj"); // Forcing terminal logger so we can see the output when it is redirected - InvokeDotnetTest($@"{projectPath} --no-build -tl:on", workingDirectory: Path.GetDirectoryName(projectPath)); + InvokeDotnetTest($@"{projectPath} --no-restore --no-build -tl:on", workingDirectory: Path.GetDirectoryName(projectPath)); // The output: // Determining projects to restore... @@ -54,7 +54,7 @@ public void MSBuildLoggerCanBeDisabledByBuildProperty(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var projectPath = GetIsolatedTestAsset("TerminalLoggerTestProject.csproj"); - InvokeDotnetTest($@"{projectPath} --no-build -nodereuse:false /p:VsTestUseMSBuildOutput=false", workingDirectory: Path.GetDirectoryName(projectPath)); + InvokeDotnetTest($@"{projectPath} --no-restore --no-build -nodereuse:false /p:VsTestUseMSBuildOutput=false", workingDirectory: Path.GetDirectoryName(projectPath)); // Check that we see the summary that is printed from the console logger, meaning the new output is disabled. StdOutputContains("Failed! - Failed: 1, Passed: 1, Skipped: 1, Total: 3, Duration:"); @@ -74,7 +74,7 @@ public void MSBuildLoggerCanBeDisabledByEnvironmentVariableProperty(RunnerInfo r SetTestEnvironment(_testEnvironment, runnerInfo); var projectPath = GetIsolatedTestAsset("TerminalLoggerTestProject.csproj"); - InvokeDotnetTest($@"{projectPath} --no-build -nodereuse:false", environmentVariables: new Dictionary { ["MSBUILDENSURESTDOUTFORTASKPROCESSES"] = "1" }, workingDirectory: Path.GetDirectoryName(projectPath)); + InvokeDotnetTest($@"{projectPath} --no-restore --no-build -nodereuse:false", environmentVariables: new Dictionary { ["MSBUILDENSURESTDOUTFORTASKPROCESSES"] = "1" }, workingDirectory: Path.GetDirectoryName(projectPath)); // Check that we see the summary that is printed from the console logger, meaning the new output is disabled. StdOutputContains("Failed! - Failed: 1, Passed: 1, Skipped: 1, Total: 3, Duration:"); diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestTests.cs index 0d89d68e59..69182cebe0 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestTests.cs @@ -27,7 +27,7 @@ public void RunDotnetTestWithCsproj(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var projectPath = GetIsolatedTestAsset("SimpleTestProject.csproj"); - InvokeDotnetTest($@"{projectPath} --no-build -tl:off /p:VSTestNoLogo=false --logger:""Console;Verbosity=normal""", workingDirectory: Path.GetDirectoryName(projectPath)); + InvokeDotnetTest($@"{projectPath} --no-restore --no-build -tl:off /p:VSTestNoLogo=false --logger:""Console;Verbosity=normal""", workingDirectory: Path.GetDirectoryName(projectPath)); // ensure our dev version is used StdOutputContains(GetFinalVersion(IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion)); @@ -44,7 +44,7 @@ public void RunDotnetTestWithDll(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPath = GetAssetFullPath("SimpleTestProject.dll"); - InvokeDotnetTest($@"{assemblyPath} --no-build --logger:""Console;Verbosity=normal""", workingDirectory: Path.GetDirectoryName(assemblyPath)); + InvokeDotnetTest($@"{assemblyPath} --no-restore --no-build --logger:""Console;Verbosity=normal""", workingDirectory: Path.GetDirectoryName(assemblyPath)); // ensure our dev version is used StdOutputContains(GetFinalVersion(IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion)); @@ -61,7 +61,7 @@ public void RunDotnetTestWithCsprojPassInlineSettings(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var projectPath = GetIsolatedTestAsset("ParametrizedTestProject.csproj"); - InvokeDotnetTest($@"{projectPath} --no-build --logger:""Console;Verbosity=normal"" -tl:off /p:VSTestNoLogo=false -- TestRunParameters.Parameter(name =\""weburl\"", value=\""http://localhost//def\"")", workingDirectory: Path.GetDirectoryName(projectPath)); + InvokeDotnetTest($@"{projectPath} --no-restore --no-build --logger:""Console;Verbosity=normal"" -tl:off /p:VSTestNoLogo=false -- TestRunParameters.Parameter(name =\""weburl\"", value=\""http://localhost//def\"")", workingDirectory: Path.GetDirectoryName(projectPath)); // ensure our dev version is used StdOutputContains(GetFinalVersion(IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion)); @@ -78,7 +78,7 @@ public void RunDotnetTestWithDllPassInlineSettings(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPath = GetAssetFullPath("ParametrizedTestProject.dll"); - InvokeDotnetTest($@"{assemblyPath} --no-build --logger:""Console;Verbosity=normal"" -- TestRunParameters.Parameter(name=\""weburl\"", value=\""http://localhost//def\"")", workingDirectory: Path.GetDirectoryName(assemblyPath)); + InvokeDotnetTest($@"{assemblyPath} --no-restore --no-build --logger:""Console;Verbosity=normal"" -- TestRunParameters.Parameter(name=\""weburl\"", value=\""http://localhost//def\"")", workingDirectory: Path.GetDirectoryName(assemblyPath)); ValidateSummaryStatus(1, 0, 0); ExitCodeEquals(0); @@ -96,7 +96,7 @@ public void RunDotnetTestWithNativeDll(RunnerInfo runnerInfo) string assemblyRelativePath = @"microsoft.testplatform.testasset.nativecpp\2.0.0\contentFiles\any\any\x64\Microsoft.TestPlatform.TestAsset.NativeCPP.dll"; var assemblyAbsolutePath = Path.Combine(_testEnvironment.PackageDirectory, assemblyRelativePath); - InvokeDotnetTest($@"{assemblyAbsolutePath} --no-build --logger:""Console;Verbosity=normal"" --diag:c:\temp\logscpp\", workingDirectory: Path.GetDirectoryName(assemblyAbsolutePath)); + InvokeDotnetTest($@"{assemblyAbsolutePath} --no-restore --no-build --logger:""Console;Verbosity=normal"" --diag:c:\temp\logscpp\", workingDirectory: Path.GetDirectoryName(assemblyAbsolutePath)); ValidateSummaryStatus(1, 1, 0); ExitCodeEquals(1); @@ -111,7 +111,7 @@ public void RunDotnetTestAndSeeOutputFromConsoleWriteLine(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPath = GetAssetFullPath("OutputtingTestProject.dll"); - InvokeDotnetTest($@"{assemblyPath} --no-build --logger:""Console;Verbosity=normal""", workingDirectory: Path.GetDirectoryName(assemblyPath)); + InvokeDotnetTest($@"{assemblyPath} --no-restore --no-build --logger:""Console;Verbosity=normal""", workingDirectory: Path.GetDirectoryName(assemblyPath)); StdOutputContains("MY OUTPUT FROM TEST"); diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/RunsettingsTests.cs index ceb95d8b73..783ad15ec4 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/RunsettingsTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; @@ -512,7 +512,7 @@ public void RunSettingsAreLoadedFromProject(RunnerInfo runnerInfo) var projectName = "ProjectFileRunSettingsTestProject.csproj"; var projectPath = GetIsolatedTestAsset(projectName); - InvokeDotnetTest($@"{projectPath} --no-build /p:VSTestUseMSBuildOutput=false --logger:""Console;Verbosity=normal""", workingDirectory: Path.GetDirectoryName(projectPath)); + InvokeDotnetTest($@"{projectPath} --no-restore --no-build /p:VSTestUseMSBuildOutput=false --logger:""Console;Verbosity=normal""", workingDirectory: Path.GetDirectoryName(projectPath)); ValidateSummaryStatus(0, 1, 0); // make sure that we can revert the project settings back by providing a config from command line @@ -520,7 +520,7 @@ public void RunSettingsAreLoadedFromProject(RunnerInfo runnerInfo) // are honored by dotnet test, instead of just using the default, which would produce the same // result var settingsPath = GetProjectAssetFullPath(projectName, "inconclusive.runsettings"); - InvokeDotnetTest($@"{projectPath} --no-build --settings {settingsPath} /p:VSTestUseMSBuildOutput=false --logger:""Console;Verbosity=normal""", workingDirectory: Path.GetDirectoryName(projectPath)); + InvokeDotnetTest($@"{projectPath} --no-restore --no-build --settings {settingsPath} /p:VSTestUseMSBuildOutput=false --logger:""Console;Verbosity=normal""", workingDirectory: Path.GetDirectoryName(projectPath)); ValidateSummaryStatus(0, 0, 1); }