Skip to content

[tests] Show full test names in html failure reports. #25094

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rolfbjarne merged 2 commits into from
Apr 7, 2026
+105 −4
Merged

[tests] Show full test names in html failure reports. #25094

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
21cf5a1
[tests] Show full test names in html failure reports.
rolfbjarne Apr 6, 2026
e7a00ce
Merge origin/main into dev/rolf/tests-full-name
rolfbjarne Apr 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions tests/common/ParseTrxFile.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,4 +74,52 @@ public static bool TryParseTrxFile (string path, [NotNullWhen (true)] out IList<
return false;
}
}

public static bool TryParseNUnitXmlFile (string path, [NotNullWhen (true)] out IList<TrxTestResult>? failedTests, [NotNullWhen (true)] out string? outcome, out bool allTestsSucceeded, out Exception? exception)
{
allTestsSucceeded = false;
failedTests = null;
outcome = null;
exception = null;

if (!File.Exists (path))
return false;

var rv = new List<TrxTestResult> ();
try {
var xml = new XmlDocument ();
xml.Load (path);
outcome = xml.SelectSingleNode ("/*[local-name() = 'test-run' or local-name() = 'test-results']")?.Attributes? ["result"]?.Value;
var failedTestsQuery = xml.SelectNodes ("//*[local-name() = 'test-case'][@result = 'Failed' or @result = 'Failure' or @label = 'Error' or (@success = 'False' and @executed = 'True')]")?.Cast<XmlNode> ();
if (failedTestsQuery?.Any () == true) {
foreach (var node in failedTestsQuery) {
var testName = node.Attributes? ["fullname"]?.Value;
if (string.IsNullOrEmpty (testName))
testName = node.Attributes? ["name"]?.Value ?? "<unknown test name>";

var testOutcome = node.Attributes? ["label"]?.Value ?? node.Attributes? ["result"]?.Value ?? "<unknown test outcome>";
var testMessage = node.SelectSingleNode ("*[local-name() = 'failure']/*[local-name() = 'message'] | *[local-name() = 'reason']/*[local-name() = 'message']")?.InnerText ?? "";

rv.Add (new TrxTestResult () {
Name = testName,
Outcome = testOutcome,
Message = testMessage,
});
}
allTestsSucceeded = false;
} else if (string.Equals (outcome, "Passed", StringComparison.OrdinalIgnoreCase) || string.Equals (outcome, "Success", StringComparison.OrdinalIgnoreCase) || string.Equals (outcome, "Completed", StringComparison.OrdinalIgnoreCase)) {
allTestsSucceeded = true;
}

failedTests = rv;
if (outcome is null)
outcome = rv.Count > 0 ? "Failed" : "Passed";
Comment on lines +115 to +116
Copy link

Copilot AI Apr 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When outcome is missing in the NUnit XML and there are no failed tests, the method sets outcome to "Passed" (line 115-116) but leaves allTestsSucceeded as false. This makes allTestsSucceeded inconsistent with the computed outcome and can mislead callers that rely on the flag. Set allTestsSucceeded = (rv.Count == 0) when defaulting the outcome, or otherwise derive it from rv when outcome is null/empty.

Suggested change
if (outcome is null)
outcome = rv.Count > 0 ? "Failed" : "Passed";
if (outcome is null) {
outcome = rv.Count > 0 ? "Failed" : "Passed";
allTestsSucceeded = rv.Count == 0;
}

Copilot uses AI. Check for mistakes.
return true;
} catch (Exception e) {
outcome = "Failed to parse test results";
exception = e;
allTestsSucceeded = false;
return false;
}
}
}
47 changes: 43 additions & 4 deletions tests/xharness/Jenkins/Reports/HtmlReportWriter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
using Microsoft.DotNet.XHarness.iOS.Shared.Hardware;
using Microsoft.DotNet.XHarness.iOS.Shared.Logging;
using Microsoft.DotNet.XHarness.iOS.Shared.Utilities;
using Xamarin.Utils;
using Xharness.Jenkins.TestTasks;

#nullable enable
Expand Down Expand Up @@ -427,7 +428,9 @@ public void Write (IList<ITestTask> allTasks, StreamWriter writer)
var query = logs.
OfType<IFileBackedLog> ().
OrderBy (v => v.Description).
ThenBy (v => v.FullPath);
ThenBy (v => v.FullPath).
ToList ();
var hasStructuredTestResults = query.Any (v => (v.Description == LogType.NUnitResult.ToString () || v.Description == LogType.XmlLog.ToString () || v.Description == LogType.TrxLog.ToString ()) && File.Exists (v.FullPath) && new FileInfo (v.FullPath).Length > 0);
Copy link

Copilot AI Apr 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hasStructuredTestResults is computed purely from the presence of a non-empty structured results file, and is then used to suppress the [FAIL] lines from the execution/test log. If the structured results file exists but is malformed/unparseable (so neither TryWriteStructuredTestReport nor resultParser.GenerateTestReport emits anything), this will hide the fallback failure list and can leave the HTML report with no failure details. Consider driving this condition off whether a structured report was actually successfully written (e.g., set a flag when parsing succeeds), or only suppress the fallback once hasListedErrors becomes true.

Suggested change
var hasStructuredTestResults = query.Any (v => (v.Description == LogType.NUnitResult.ToString () || v.Description == LogType.XmlLog.ToString () || v.Description == LogType.TrxLog.ToString ()) && File.Exists (v.FullPath) && new FileInfo (v.FullPath).Length > 0);
var hasStructuredTestResults = false;

Copilot uses AI. Check for mistakes.
var hasListedErrors = false;
foreach (var fileLog in query) {
var log = fileLog;
Expand Down Expand Up @@ -497,7 +500,7 @@ public void Write (IList<ITestTask> allTasks, StreamWriter writer)
fails = data_tuple.Item2;
}
}
if (!hasListedErrors && fails.Count > 0) {
if (!hasStructuredTestResults && !hasListedErrors && fails.Count > 0) {
writer.WriteLine ("<div style='padding-left: 15px;'>");
foreach (var fail in fails)
writer.WriteLine ("{0} <br />", fail.AsHtml ());
Expand Down Expand Up @@ -548,7 +551,9 @@ public void Write (IList<ITestTask> allTasks, StreamWriter writer)
} else if (log.Description == LogType.NUnitResult.ToString () || log.Description == LogType.XmlLog.ToString ()) {
try {
if (!hasListedErrors && File.Exists (fileLog.FullPath) && new FileInfo (fileLog.FullPath).Length > 0) {
if (resultParser.IsValidXml (fileLog.FullPath, out var jargon)) {
if (TryWriteStructuredTestReport (writer, fileLog.FullPath, log.Description)) {
hasListedErrors = true;
} else if (resultParser.IsValidXml (fileLog.FullPath, out var jargon)) {
resultParser.GenerateTestReport (writer, fileLog.FullPath, jargon);
hasListedErrors = true;
}
Expand All @@ -558,7 +563,9 @@ public void Write (IList<ITestTask> allTasks, StreamWriter writer)
}
} else if (log.Description == LogType.TrxLog.ToString ()) {
try {
if (!hasListedErrors && resultParser.IsValidXml (fileLog.FullPath, out var jargon)) {
if (!hasListedErrors && TryWriteStructuredTestReport (writer, fileLog.FullPath, log.Description)) {
hasListedErrors = true;
} else if (!hasListedErrors && resultParser.IsValidXml (fileLog.FullPath, out var jargon)) {
resultParser.GenerateTestReport (writer, fileLog.FullPath, jargon);
hasListedErrors = true;
}
Expand Down Expand Up @@ -645,6 +652,38 @@ static string GetMimeMapping (string value)
return "application/octet-stream";
}

internal static bool TryWriteStructuredTestReport (StreamWriter writer, string filePath, string logDescription)
{
IList<TrxParser.TrxTestResult>? failedTests;
bool parsed;

if (logDescription == LogType.TrxLog.ToString ()) {
parsed = TrxParser.TryParseTrxFile (filePath, out failedTests, out _, out _, out _);
} else if (logDescription == LogType.NUnitResult.ToString () || logDescription == LogType.XmlLog.ToString ()) {
parsed = TrxParser.TryParseNUnitXmlFile (filePath, out failedTests, out _, out _, out _);
} else {
return false;
}

if (!parsed || failedTests?.Count is not > 0)
return false;

writer.WriteLine ("<div style='padding-left: 15px;'>");
writer.WriteLine ("<ul>");
foreach (var failedTest in failedTests) {
writer.WriteLine ("<li>");
if (string.IsNullOrEmpty (failedTest.Message)) {
writer.WriteLine ("{0}<br />", failedTest.Name.AsHtml ());
} else {
writer.WriteLine ("{0}: {1}<br />", failedTest.Name.AsHtml (), failedTest.Message.AsHtml ());
}
writer.WriteLine ("</li>");
}
writer.WriteLine ("</ul>");
writer.WriteLine ("</div>");
return true;
}

static string LinkEncode (string path)
{
return System.Web.HttpUtility.UrlEncode (path).Replace ("%2f", "/").Replace ("+", "%20");
Expand Down
Loading