diff --git a/tests/common/ParseTrxFile.cs b/tests/common/ParseTrxFile.cs index 537a223a4a1..ff9093c0f96 100644 --- a/tests/common/ParseTrxFile.cs +++ b/tests/common/ParseTrxFile.cs @@ -74,4 +74,52 @@ public static bool TryParseTrxFile (string path, [NotNullWhen (true)] out IList< return false; } } + + public static bool TryParseNUnitXmlFile (string path, [NotNullWhen (true)] out IList? failedTests, [NotNullWhen (true)] out string? outcome, out bool allTestsSucceeded, out Exception? exception) + { + allTestsSucceeded = false; + failedTests = null; + outcome = null; + exception = null; + + if (!File.Exists (path)) + return false; + + var rv = new List (); + try { + var xml = new XmlDocument (); + xml.Load (path); + outcome = xml.SelectSingleNode ("/*[local-name() = 'test-run' or local-name() = 'test-results']")?.Attributes? ["result"]?.Value; + var failedTestsQuery = xml.SelectNodes ("//*[local-name() = 'test-case'][@result = 'Failed' or @result = 'Failure' or @label = 'Error' or (@success = 'False' and @executed = 'True')]")?.Cast (); + if (failedTestsQuery?.Any () == true) { + foreach (var node in failedTestsQuery) { + var testName = node.Attributes? ["fullname"]?.Value; + if (string.IsNullOrEmpty (testName)) + testName = node.Attributes? ["name"]?.Value ?? ""; + + var testOutcome = node.Attributes? ["label"]?.Value ?? node.Attributes? ["result"]?.Value ?? ""; + var testMessage = node.SelectSingleNode ("*[local-name() = 'failure']/*[local-name() = 'message'] | *[local-name() = 'reason']/*[local-name() = 'message']")?.InnerText ?? ""; + + rv.Add (new TrxTestResult () { + Name = testName, + Outcome = testOutcome, + Message = testMessage, + }); + } + allTestsSucceeded = false; + } else if (string.Equals (outcome, "Passed", StringComparison.OrdinalIgnoreCase) || string.Equals (outcome, "Success", StringComparison.OrdinalIgnoreCase) || string.Equals (outcome, "Completed", StringComparison.OrdinalIgnoreCase)) { + allTestsSucceeded = true; + } + + failedTests = rv; + if (outcome is null) + outcome = rv.Count > 0 ? "Failed" : "Passed"; + return true; + } catch (Exception e) { + outcome = "Failed to parse test results"; + exception = e; + allTestsSucceeded = false; + return false; + } + } } diff --git a/tests/xharness/Jenkins/Reports/HtmlReportWriter.cs b/tests/xharness/Jenkins/Reports/HtmlReportWriter.cs index 532820b48fa..494d8e8dbcf 100644 --- a/tests/xharness/Jenkins/Reports/HtmlReportWriter.cs +++ b/tests/xharness/Jenkins/Reports/HtmlReportWriter.cs @@ -12,6 +12,7 @@ using Microsoft.DotNet.XHarness.iOS.Shared.Hardware; using Microsoft.DotNet.XHarness.iOS.Shared.Logging; using Microsoft.DotNet.XHarness.iOS.Shared.Utilities; +using Xamarin.Utils; using Xharness.Jenkins.TestTasks; #nullable enable @@ -428,7 +429,9 @@ public void Write (IList allTasks, StreamWriter writer) var query = logs. OfType (). OrderBy (v => v.Description). - ThenBy (v => v.FullPath); + ThenBy (v => v.FullPath). + ToList (); + var hasStructuredTestResults = query.Any (v => HasStructuredTestReport (v.FullPath, v.Description)); var hasListedErrors = false; foreach (var fileLog in query) { var log = fileLog; @@ -498,7 +501,7 @@ public void Write (IList allTasks, StreamWriter writer) fails = data_tuple.Item2; } } - if (!hasListedErrors && fails.Count > 0) { + if (!hasStructuredTestResults && !hasListedErrors && fails.Count > 0) { writer.WriteLine ("
"); foreach (var fail in fails) writer.WriteLine ("{0}
", fail.AsHtml ()); @@ -549,7 +552,9 @@ public void Write (IList allTasks, StreamWriter writer) } else if (log.Description == LogType.NUnitResult.ToString () || log.Description == LogType.XmlLog.ToString ()) { try { if (!hasListedErrors && File.Exists (fileLog.FullPath) && new FileInfo (fileLog.FullPath).Length > 0) { - if (resultParser.IsValidXml (fileLog.FullPath, out var jargon)) { + if (TryWriteStructuredTestReport (writer, fileLog.FullPath, log.Description)) { + hasListedErrors = true; + } else if (resultParser.IsValidXml (fileLog.FullPath, out var jargon)) { // Some test runs produce multiple XML files. Keep looking until we actually // render a failure summary, otherwise a wrapper XML can hide the useful one. hasListedErrors = TryWriteGeneratedTestReport (writer, fileLog.FullPath, jargon); @@ -560,7 +565,9 @@ public void Write (IList allTasks, StreamWriter writer) } } else if (log.Description == LogType.TrxLog.ToString ()) { try { - if (!hasListedErrors && resultParser.IsValidXml (fileLog.FullPath, out var jargon)) { + if (!hasListedErrors && TryWriteStructuredTestReport (writer, fileLog.FullPath, log.Description)) { + hasListedErrors = true; + } else if (!hasListedErrors && resultParser.IsValidXml (fileLog.FullPath, out var jargon)) { hasListedErrors = TryWriteGeneratedTestReport (writer, fileLog.FullPath, jargon); } } catch (Exception ex) { @@ -646,6 +653,38 @@ static string GetMimeMapping (string value) return "application/octet-stream"; } + internal static bool TryWriteStructuredTestReport (StreamWriter writer, string filePath, string? logDescription) + { + IList? failedTests; + bool parsed; + + if (logDescription == LogType.TrxLog.ToString ()) { + parsed = TrxParser.TryParseTrxFile (filePath, out failedTests, out _, out _, out _); + } else if (logDescription == LogType.NUnitResult.ToString () || logDescription == LogType.XmlLog.ToString ()) { + parsed = TrxParser.TryParseNUnitXmlFile (filePath, out failedTests, out _, out _, out _); + } else { + return false; + } + + if (!parsed || failedTests?.Count is not > 0) + return false; + + writer.WriteLine ("
"); + writer.WriteLine ("
    "); + foreach (var failedTest in failedTests) { + writer.WriteLine ("
  • "); + if (string.IsNullOrEmpty (failedTest.Message)) { + writer.WriteLine ("{0}
    ", failedTest.Name.AsHtml ()); + } else { + writer.WriteLine ("{0}: {1}
    ", failedTest.Name.AsHtml (), failedTest.Message.AsHtml ()); + } + writer.WriteLine ("
  • "); + } + writer.WriteLine ("
"); + writer.WriteLine ("
"); + return true; + } + static string LinkEncode (string path) { return System.Web.HttpUtility.UrlEncode (path).Replace ("%2f", "/").Replace ("+", "%20"); @@ -692,5 +731,19 @@ bool TryWriteGeneratedTestReport (StreamWriter writer, string path, XmlResultJar writer.Write (report); return true; } + + bool HasStructuredTestReport (string path, string? logDescription) + { + if ((logDescription != LogType.NUnitResult.ToString () && logDescription != LogType.XmlLog.ToString () && logDescription != LogType.TrxLog.ToString ()) || !File.Exists (path) || new FileInfo (path).Length == 0) + return false; + + using var ms = new MemoryStream (); + using var writer = new StreamWriter (ms, new UTF8Encoding (encoderShouldEmitUTF8Identifier: false), 1024, leaveOpen: true); + + if (TryWriteStructuredTestReport (writer, path, logDescription)) + return true; + + return resultParser.IsValidXml (path, out var jargon) && TryWriteGeneratedTestReport (writer, path, jargon); + } } }