From c4a96f8ca62a7ab904353d85cd09866e1d126ca9 Mon Sep 17 00:00:00 2001 From: Daniel Cazzulino Date: Thu, 4 Jul 2024 01:43:39 -0300 Subject: [PATCH] Add support for reporting to github Two features here: - Report summary and details as a comment on the associated PR (if any) - Issue :error: notices with the paths of the failed paths and the error message. --- .github/workflows/build.yml | 4 +- .netconfig | 4 +- src/Demo/Tests.cs | 16 ++- src/dotnet-trx/Extensions.cs | 15 +++ src/dotnet-trx/Process.cs | 64 ++++++++++ src/dotnet-trx/TrxCommand.cs | 221 ++++++++++++++++++++++++++++------- 6 files changed, 271 insertions(+), 53 deletions(-) create mode 100644 src/dotnet-trx/Extensions.cs create mode 100644 src/dotnet-trx/Process.cs diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c671ecc..54e87e5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -68,7 +68,9 @@ jobs: echo 'export PATH="/usr/local/opt/grep/libexec/gnubin:$PATH"' >> .bash_profile - name: 🧪 test - uses: ./.github/workflows/test + run: | + dotnet test -l trx src/Demo/ -v:q + dotnet run --no-build --project ./src/dotnet-trx/ --output - name: 🐛 logs uses: actions/upload-artifact@v3 diff --git a/.netconfig b/.netconfig index cd5388f..13c12ab 100644 --- a/.netconfig +++ b/.netconfig @@ -37,9 +37,7 @@ weak [file ".github/workflows/build.yml"] url = https://github.com/devlooped/oss/blob/main/.github/workflows/build.yml - sha = 14deaea5cecc64df51781d29891a2f67caf8be16 - etag = d9fa5d91dc601f10d19099abb55c86df065cd1c23b1f6fab98ad883cb443bf5c - weak + skip [file ".github/workflows/changelog.config"] url = https://github.com/devlooped/oss/blob/main/.github/workflows/changelog.config sha = 055a8b7c94b74ae139cce919d60b83976d2a9942 diff --git a/src/Demo/Tests.cs b/src/Demo/Tests.cs index ef26c55..a8ab34f 100644 --- a/src/Demo/Tests.cs +++ b/src/Demo/Tests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Threading.Tasks; using Xunit.Abstractions; @@ -7,10 +7,10 @@ namespace Demo; public class Tests(ITestOutputHelper output) { [Fact] - public void ICanHasOutput() => output.WriteLine("Hello, world from xunit ITestOutputHelper!"); + public void Test_With_Output() => output.WriteLine("Hello, world from xunit ITestOutputHelper!"); [Fact(Skip = "Shouldn't run for now :)")] - public void SampleSkipped() { } + public void Skipped_Test_Does_Not_Run() { } [Theory] [InlineData("en")] @@ -22,10 +22,16 @@ public void SampleSkipped() { } #pragma warning restore xUnit1026 // Theory methods should use all of their parameters [Fact] - public void OhNoh() => Assert.Equal(42, 22); + public void Fails_With_Output() + { + output.WriteLine("It was going so well... "); + output.WriteLine("Yet you never know"); + output.WriteLine("Which is why you sprinkle all these WriteLines :eyes:"); + Assert.Equal(42, 22); + } [Fact] - public void CleanStackTrace() + public void Fails_With_Complex_StackTrace() { Action runner = () => Run(); diff --git a/src/dotnet-trx/Extensions.cs b/src/dotnet-trx/Extensions.cs new file mode 100644 index 0000000..681eef9 --- /dev/null +++ b/src/dotnet-trx/Extensions.cs @@ -0,0 +1,15 @@ +using System; +using System.Text; + +namespace Devlooped; + +static class Extensions +{ + public static StringBuilder AppendLineIndented(this StringBuilder builder, string value, string indent) + { + foreach (var line in value.ReplaceLineEndings().Split(Environment.NewLine)) + builder.Append(indent).AppendLine(line); + + return builder; + } +} diff --git a/src/dotnet-trx/Process.cs b/src/dotnet-trx/Process.cs new file mode 100644 index 0000000..23ee2bb --- /dev/null +++ b/src/dotnet-trx/Process.cs @@ -0,0 +1,64 @@ +using System; +using System.Diagnostics; + +namespace Devlooped; + +static class Process +{ + public static bool TryExecute(string program, string arguments, out string? output) + => TryExecuteCore(program, arguments, null, out output); + + public static bool TryExecute(string program, string arguments, string input, out string? output) + => TryExecuteCore(program, arguments, input, out output); + + static bool TryExecuteCore(string program, string arguments, string? input, out string? output) + { + var info = new ProcessStartInfo(program, arguments) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + RedirectStandardInput = input != null + }; + + try + { + var proc = System.Diagnostics.Process.Start(info); + if (proc == null) + { + output = null; + return false; + } + + var gotError = false; + proc.ErrorDataReceived += (_, __) => gotError = true; + + if (input != null) + { + // Write the input to the standard input stream + proc.StandardInput.WriteLine(input); + proc.StandardInput.Close(); + } + + output = proc.StandardOutput.ReadToEnd(); + if (!proc.WaitForExit(5000)) + { + proc.Kill(); + output = null; + return false; + } + + var error = proc.StandardError.ReadToEnd(); + gotError |= error.Length > 0; + output = output.Trim(); + if (string.IsNullOrEmpty(output)) + output = null; + + return !gotError && proc.ExitCode == 0; + } + catch (Exception ex) + { + output = ex.Message; + return false; + } + } +} diff --git a/src/dotnet-trx/TrxCommand.cs b/src/dotnet-trx/TrxCommand.cs index f1f66ee..fafa38a 100644 --- a/src/dotnet-trx/TrxCommand.cs +++ b/src/dotnet-trx/TrxCommand.cs @@ -3,12 +3,16 @@ using System.ComponentModel; using System.IO; using System.Linq; +using System.Text; using System.Text.RegularExpressions; using System.Xml.Linq; using Devlooped.Web; using Humanizer; +using Newtonsoft.Json.Linq; +using NuGet.Protocol.Plugins; using Spectre.Console; using Spectre.Console.Cli; +using static Devlooped.Process; using static Spectre.Console.AnsiConsole; namespace Devlooped; @@ -53,6 +57,16 @@ public override int Execute(CommandContext context, TrxSettings settings) var failed = 0; var skipped = 0; var duration = TimeSpan.Zero; + var failures = new List(); + + // markdown details for gh comment + var details = new StringBuilder().AppendLine( + """ +
+ + :test_tube: Details + + """); // Process from newest files to oldest foreach (var trx in Directory.EnumerateFiles(path, "*.trx", search).OrderByDescending(File.GetLastWriteTime)) @@ -69,27 +83,44 @@ public override int Execute(CommandContext context, TrxSettings settings) continue; var test = result.Attribute("testName")!.Value; + string? output = settings.Output ? result.CssSelectElement("StdOut")?.Value : default; + switch (result.Attribute("outcome")?.Value) { case "Passed": passed++; MarkupLine($":check_mark_button: {test}"); + if (output == null) + details.AppendLine($":white_check_mark: {test}"); + else + details.AppendLine( + $""" +
+ + :white_check_mark: {test} + + """) + .AppendLineIndented(output, "> > ") + .AppendLine( + """ + +
+ """); break; case "Failed": failed++; MarkupLine($":cross_mark: {test}"); - if (result.CssSelectElement("Message")?.Value is string message && - result.CssSelectElement("StackTrace")?.Value is string stackTrace) - { - var error = new Panel( - $""" - [red]{message}[/] - [dim]{CleanStackTrace(path, result, stackTrace.ReplaceLineEndings())}[/] - """); - error.Padding = new Padding(5, 0, 0, 0); - error.Border = BoxBorder.None; - Write(error); - } + details.AppendLine( + $""" +
+ + :x: {test} + + """); + WriteError(path, failures, result, details); + if (output != null) + details.AppendLineIndented(output, "> > "); + details.AppendLine().AppendLine("
").AppendLine(); break; case "NotExecuted": if (!settings.Skipped) @@ -98,21 +129,29 @@ public override int Execute(CommandContext context, TrxSettings settings) skipped++; var reason = result.CssSelectElement("Output > ErrorInfo > Message")?.Value; Markup($"[dim]:white_question_mark: {test}[/]"); + details.Append($":grey_question: {test}"); + if (reason != null) + { Markup($"[dim] => {reason}[/]"); + details.Append($" => {reason}"); + } WriteLine(); + details.AppendLine(); break; default: break; } - if (settings.Output == true && result.CssSelectElement("StdOut")?.Value is { } output) + if (output != null) + { Write(new Panel($"[dim]{output.ReplaceLineEndings()}[/]") { Border = BoxBorder.None, Padding = new Padding(5, 0, 0, 0), }); + } } var times = doc.CssSelectElement("Times"); @@ -124,60 +163,154 @@ public override int Execute(CommandContext context, TrxSettings settings) duration += finish - start; } + details.AppendLine().AppendLine("
"); + + var summary = new Summary(passed, failed, skipped, duration); + WriteLine(); + MarkupSummary(summary); WriteLine(); + GitHubReport(summary, details); - Markup($":backhand_index_pointing_right: Run {passed + failed + skipped} tests in ~ {duration.Humanize()}"); + if (failures.Count > 0 && Environment.GetEnvironmentVariable("CI") == "true") + { + // Send workflow commands for each failure to be annotated in GH CI + foreach (var failure in failures) + WriteLine($"::error file={failure.File},line={failure.Line},title={failure.Message}::{failure.Message}"); + } + + return 0; + } - if (failed > 0) + static void MarkupSummary(Summary summary) + { + Markup($":backhand_index_pointing_right: Run {summary.Total} tests in ~ {summary.Duration.Humanize()}"); + + if (summary.Failed > 0) MarkupLine($" :cross_mark:"); else MarkupLine($" :check_mark_button:"); - if (passed > 0) - MarkupLine($" :check_mark_button: {passed} passed"); + if (summary.Passed > 0) + MarkupLine($" :check_mark_button: {summary.Passed} passed"); - if (failed > 0) - MarkupLine($" :cross_mark: {failed} failed"); + if (summary.Failed > 0) + MarkupLine($" :cross_mark: {summary.Failed} failed"); - if (settings.Skipped && skipped > 0) - MarkupLine($" :white_question_mark: {skipped} skipped"); + if (summary.Skipped > 0) + MarkupLine($" :white_question_mark: {summary.Skipped} skipped"); + } - WriteLine(); + static void GitHubReport(Summary summary, StringBuilder details) + { + if (TryExecute("gh", "--version", out var output) && output?.StartsWith("gh version") != true) + return; - return 0; + // See https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables + if (Environment.GetEnvironmentVariable("GITHUB_REF_NAME") is not { } branch || + !branch.EndsWith("/merge") || + !int.TryParse(branch[..^6], out var pr)) + return; + + var sb = new StringBuilder() + .AppendLine( + $""" + :point_right: Run {summary.Total} tests in ~ {summary.Duration.Humanize()} + """); + + if (summary.Passed > 0) + sb.AppendLine($"     :white_check_mark: {summary.Passed} passed"); + if (summary.Failed > 0) + sb.AppendLine($"     :x: {summary.Failed} failed"); + if (summary.Skipped > 0) + sb.AppendLine($"     :grey_question: {summary.Skipped} skipped"); + + sb.AppendLine(); + sb.Append(details); + sb.AppendLine(); + + sb.AppendLine( + $"from [dotnet-trx](https://github.com/devlooped/dotnet-trx) with [:purple_heart:](https://github.com/sponsors/devlooped)"); + + if (TryExecute("gh", $"pr comment {pr} --body-file -", sb.ToString(), out var link)) + WriteLine($"::notice title=Added summary as [pull-request comment]({link})"); } - string CleanStackTrace(string baseDir, XElement result, string stackTrace) + void WriteError(string baseDir, List failures, XElement result, StringBuilder details) { - // Stop lines when we find the last one from the test method + if (result.CssSelectElement("Message")?.Value is not string message || + result.CssSelectElement("StackTrace")?.Value is not string stackTrace) + return; + + var testName = result.Attribute("testName")!.Value; var testId = result.Attribute("testId")!.Value; var method = result.Document!.CssSelectElement($"UnitTest[id={testId}] TestMethod"); - if (method == null) - return stackTrace; + var lines = stackTrace.ReplaceLineEndings().Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries); - var fullName = $"{method.Attribute("className")?.Value}.{method.Attribute("name")?.Value}"; + if (method != null) + { + var fullName = $"{method.Attribute("className")?.Value}.{method.Attribute("name")?.Value}"; + var last = Array.FindLastIndex(lines, x => x.Contains(fullName)); + // Stop lines when we find the last one from the test method + if (last != -1) + lines = lines[..(last + 1)]; + } - var lines = stackTrace.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries); - var last = Array.FindLastIndex(lines, x => x.Contains(fullName)); - if (last == -1) - return stackTrace; + Failed? failed = null; + var cli = new StringBuilder(); + details.Append("> ```"); + if (stackTrace.Contains(".vb:line")) + details.AppendLine("vb"); + else + details.AppendLine("csharp"); - return string.Join(Environment.NewLine, lines - .Take(last + 1) - .Select(line => + foreach (var line in lines) + { + var match = ParseFile().Match(line); + if (!match.Success) { - var match = ParseFile().Match(line); - if (!match.Success) - return line; + cli.AppendLine(line); + details.AppendLineIndented(line, "> "); + continue; + } + + var file = match.Groups["file"].Value; + var pos = match.Groups["line"].Value; + var relative = Path.GetRelativePath(baseDir, file); + // NOTE: we replace whichever was last, since we want the annotation on the + // last one with a filename, which will be the test itself (see previous skip from last found). + failed = new Failed(testName, message, relative, int.Parse(pos)); + + cli.AppendLine(line.Replace(file, $"[link={file}][steelblue1_1]{relative}[/][/]")); + // TODO: can we render a useful link in comment details? + details.AppendLineIndented(line.Replace(file, relative), "> "); + } - var file = match.Groups["file"].Value; - var relative = Path.GetRelativePath(baseDir, file); - line = line.Replace(file, $"[link={file}][steelblue1_1]{relative}[/][/]"); + var error = new Panel( + $""" + [red]{message}[/] + [dim]{cli}[/] + """); + error.Padding = new Padding(5, 0, 0, 0); + error.Border = BoxBorder.None; + Write(error); - return line; - })); + // Use a blockquote for the entire error message + + details.AppendLine("> ```"); + + // Add to collected failures we may report to GH CI + if (failed != null) + failures.Add(failed); } - [GeneratedRegex(@" in (?.+):line", RegexOptions.Compiled)] + // in C:\path\to\file.cs:line 123 + [GeneratedRegex(@" in (?.+):line (?\d+)", RegexOptions.Compiled)] private static partial Regex ParseFile(); + + record Summary(int Passed, int Failed, int Skipped, TimeSpan Duration) + { + public int Total => Passed + Failed + Skipped; + } + + record Failed(string Test, string Message, string File, int Line); }