From c7e3c8fc25cab21ba9585cc34ae791ff9fdafbc5 Mon Sep 17 00:00:00 2001 From: Bartek Pacia Date: Mon, 19 Aug 2024 16:28:38 +0100 Subject: [PATCH] generate HTML output --- .../java/maestro/cli/command/TestCommand.kt | 3 +- .../maestro/cli/model/TestExecutionSummary.kt | 5 +- .../cli/report/HtmlAITestSuiteReporter.kt | 94 +++++++++++++++++++ .../maestro/cli/report/TestDebugReporter.kt | 32 ++++--- .../maestro/cli/report/TestSuiteReporter.kt | 6 +- .../cli/runner/MaestroCommandRunner.kt | 10 +- .../java/maestro/cli/runner/TestRunner.kt | 10 +- .../maestro/cli/runner/TestSuiteInteractor.kt | 12 +-- 8 files changed, 137 insertions(+), 35 deletions(-) create mode 100644 maestro-cli/src/main/java/maestro/cli/report/HtmlAITestSuiteReporter.kt diff --git a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt index c9159693ff..6456981361 100644 --- a/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt +++ b/maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt @@ -279,7 +279,8 @@ class TestCommand : Callable { if (!flattenDebugOutput) { TestDebugReporter.deleteOldFiles() } - Triple(suiteResult.passedCount, suiteResult.totalTests, suiteResult) + + return@newSession Triple(suiteResult.passedCount, suiteResult.totalTests, suiteResult) } else { // Run a single flow diff --git a/maestro-cli/src/main/java/maestro/cli/model/TestExecutionSummary.kt b/maestro-cli/src/main/java/maestro/cli/model/TestExecutionSummary.kt index 475a184ff7..632f767341 100644 --- a/maestro-cli/src/main/java/maestro/cli/model/TestExecutionSummary.kt +++ b/maestro-cli/src/main/java/maestro/cli/model/TestExecutionSummary.kt @@ -2,6 +2,8 @@ package maestro.cli.model import kotlin.time.Duration +// TODO: Some properties should be implemented as getters, but it's not possible. +// See https://github.com/Kotlin/kotlinx.serialization/issues/805 data class TestExecutionSummary( val passed: Boolean, val suites: List, @@ -27,5 +29,4 @@ data class TestExecutionSummary( data class Failure( val message: String, ) - -} \ No newline at end of file +} diff --git a/maestro-cli/src/main/java/maestro/cli/report/HtmlAITestSuiteReporter.kt b/maestro-cli/src/main/java/maestro/cli/report/HtmlAITestSuiteReporter.kt new file mode 100644 index 0000000000..f582e2c8a9 --- /dev/null +++ b/maestro-cli/src/main/java/maestro/cli/report/HtmlAITestSuiteReporter.kt @@ -0,0 +1,94 @@ +package maestro.cli.report + +import kotlinx.html.a +import kotlinx.html.body +import kotlinx.html.div +import kotlinx.html.h1 +import kotlinx.html.head +import kotlinx.html.html +import kotlinx.html.img +import kotlinx.html.lang +import kotlinx.html.main +import kotlinx.html.p +import kotlinx.html.script +import kotlinx.html.stream.appendHTML +import kotlinx.html.title +import okio.Sink +import okio.buffer + +// TODO(bartekpacia): Decide if AI output can be considered "test output", and therefore be present in e.g. JUnit report +class HtmlAITestSuiteReporter { + + fun report(summary: SingleFlowAIOutput, out: Sink) { + val bufferedOut = out.buffer() + val htmlContent = buildHtmlReport(summary) + bufferedOut.writeUtf8(htmlContent) + bufferedOut.close() + } + + private fun buildHtmlReport(summary: SingleFlowAIOutput): String { + return buildString { + appendLine("") + appendHTML().html { + lang = "en" + + head { + title { +"Maestro Test Report" } + script { src = "https://cdn.tailwindcss.com/3.4.5" } + } + + body { + div(classes = "flex min-h-screen flex-col") { + + div(classes = "container mx-auto py-6 space-y-2") { + h1(classes = "text-3xl") { + +"Flow \"${summary.flowName}\" – AI output" + } + p { + +"Flow file: " + a( + // FIXME(bartekpacia): This path will be broken when moved across machines + href = summary.flowFile.absolutePath + ) { + +summary.flowFile.name + } + } + } + main(classes = "container mx-auto flex flex-col gap-4") { + p(classes = "text-[#4f4b5c]") { +"10 possible defects found" } + + summary.screenOutputs.forEach { singleScreenOutput -> + div(classes = "flex items-start gap-4 bg-white") { + p(classes = "text-lg") { + +"${singleScreenOutput.defects.size} possible defects" + } + + singleScreenOutput.defects.forEach { defect -> + img(classes = "w-64 rounded-lg border-2 border-[#4f4b5c]") { + alt = "Screenshot of the defect" + // Use relative path, so when file is moved across machines, it still works + src = singleScreenOutput.screenshotPath.name.toString() + } + + div(classes = "flex flex-col gap-4") { + div(classes = "flex flex-col items-start gap-2 rounded-lg bg-[#f8f8f8] p-2") { + p(classes = "text-[#110c22]") { + +defect.reasoning + } + + div(classes = "rounded-lg bg-[#ececec] p-1 font-semibold text-[#4f4b5c]") { + +defect.category + } + } + } + } + } + } + } + } + } + } + } + } + +} diff --git a/maestro-cli/src/main/java/maestro/cli/report/TestDebugReporter.kt b/maestro-cli/src/main/java/maestro/cli/report/TestDebugReporter.kt index 7b87133aaf..7ba39addcb 100644 --- a/maestro-cli/src/main/java/maestro/cli/report/TestDebugReporter.kt +++ b/maestro-cli/src/main/java/maestro/cli/report/TestDebugReporter.kt @@ -2,10 +2,7 @@ package maestro.cli.report import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.annotation.JsonProperty -import com.fasterxml.jackson.core.util.DefaultIndenter -import com.fasterxml.jackson.core.util.DefaultPrettyPrinter import com.fasterxml.jackson.databind.JsonMappingException -import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import maestro.MaestroException import maestro.TreeNode @@ -17,6 +14,7 @@ import maestro.debuglog.DebugLogStore import maestro.debuglog.LogConfig import maestro.orchestra.MaestroCommand import maestro.orchestra.ai.Defect +import okio.sink import org.slf4j.LoggerFactory import java.io.File import java.nio.file.Files @@ -45,7 +43,7 @@ object TestDebugReporter { private var debugOutputPathAsString: String? = null private var flattenDebugOutput: Boolean = false - fun saveFlow(flowName: String, debugOutput: FlowDebugOutput, aiOutput: FlowAIOutput?, path: Path) { + fun saveFlow(flowName: String, debugOutput: FlowDebugOutput, aiOutput: SingleFlowAIOutput?, path: Path) { // TODO(bartekpacia): Potentially accept a single "FlowPersistentOutput" object // TODO(bartekpacia: Build output incrementally, instead of single-shot on flow completion @@ -82,21 +80,27 @@ object TestDebugReporter { aiOutput?.run { // Write AI screenshots. Paths need to be changed to the final ones. - val updatedOutputs = outputs.map { output -> + val updatedOutputs = screenOutputs.map { output -> val screenshotFilename = output.screenshotPath.name val screenshotFile = File(path.absolutePathString(), screenshotFilename) output.screenshotPath.copyTo(screenshotFile) output.copy(screenshotPath = screenshotFile) } - outputs.clear() - outputs.addAll(updatedOutputs) + screenOutputs.clear() + screenOutputs.addAll(updatedOutputs) // Write AI JSON output - val filename = "ai-(${flowName.replace("/", "_")}).json" - val file = File(path.absolutePathString(), filename) - mapper.writeValue(file, this) + val jsonFilename = "ai-(${flowName.replace("/", "_")}).json" + val jsonFile = File(path.absolutePathString(), jsonFilename) + mapper.writeValue(jsonFile, this) + + // Write HTML file + val htmlFilename = "ai-(${flowName.replace("/", "_")}).html" + val htmlFile = File(path.absolutePathString(), htmlFilename) + HtmlAITestSuiteReporter().report(aiOutput, htmlFile.sink()) } + } fun deleteOldFiles(days: Long = 14) { @@ -199,13 +203,13 @@ data class FlowDebugOutput( ) } -data class FlowAIOutput( +data class SingleFlowAIOutput( @JsonProperty("flow_name") val flowName: String, - @JsonProperty("flow_file_path") val flowFilePath: String, - val outputs: MutableList = mutableListOf(), + @JsonProperty("flow_file_path") val flowFile: File, + @JsonProperty("outputs") val screenOutputs: MutableList = mutableListOf(), ) -data class AIOutput( +data class SingleScreenFlowAIOutput( @JsonProperty("screenshot_path") val screenshotPath: File, val defects: List, ) diff --git a/maestro-cli/src/main/java/maestro/cli/report/TestSuiteReporter.kt b/maestro-cli/src/main/java/maestro/cli/report/TestSuiteReporter.kt index 40a7fe89f8..3f561b0f6f 100644 --- a/maestro-cli/src/main/java/maestro/cli/report/TestSuiteReporter.kt +++ b/maestro-cli/src/main/java/maestro/cli/report/TestSuiteReporter.kt @@ -5,6 +5,9 @@ import okio.Sink interface TestSuiteReporter { + /** + * Writes the report for [summary] to [out] in the format specified by the implementation. + */ fun report( summary: TestExecutionSummary, out: Sink, @@ -17,5 +20,4 @@ interface TestSuiteReporter { } } } - -} \ No newline at end of file +} diff --git a/maestro-cli/src/main/java/maestro/cli/runner/MaestroCommandRunner.kt b/maestro-cli/src/main/java/maestro/cli/runner/MaestroCommandRunner.kt index cdf9878304..c1b3c415a5 100644 --- a/maestro-cli/src/main/java/maestro/cli/runner/MaestroCommandRunner.kt +++ b/maestro-cli/src/main/java/maestro/cli/runner/MaestroCommandRunner.kt @@ -22,9 +22,9 @@ package maestro.cli.runner import maestro.Maestro import maestro.MaestroException import maestro.cli.device.Device -import maestro.cli.report.AIOutput +import maestro.cli.report.SingleScreenFlowAIOutput import maestro.cli.report.CommandDebugMetadata -import maestro.cli.report.FlowAIOutput +import maestro.cli.report.SingleFlowAIOutput import maestro.cli.report.FlowDebugOutput import maestro.cli.runner.resultview.ResultView import maestro.cli.runner.resultview.UiState @@ -50,7 +50,7 @@ object MaestroCommandRunner { view: ResultView, commands: List, debugOutput: FlowDebugOutput, - aiOutput: FlowAIOutput, + aiOutput: SingleFlowAIOutput, ): Result { val config = YamlCommandReader.getConfig(commands) val initFlow = config?.initFlow @@ -189,8 +189,8 @@ object MaestroCommandRunner { onCommandGeneratedOutput = { command, defects, screenshot -> logger.info("${command.description()} generated output") val screenshotPath = writeAIscreenshot(screenshot) - aiOutput.outputs.add( - AIOutput( + aiOutput.screenOutputs.add( + SingleScreenFlowAIOutput( screenshotPath = screenshotPath, defects = defects, ) diff --git a/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt b/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt index e7ec94520a..815d55b225 100644 --- a/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt +++ b/maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt @@ -8,7 +8,7 @@ import com.github.michaelbull.result.getOr import com.github.michaelbull.result.onFailure import maestro.Maestro import maestro.cli.device.Device -import maestro.cli.report.FlowAIOutput +import maestro.cli.report.SingleFlowAIOutput import maestro.cli.report.FlowDebugOutput import maestro.cli.report.TestDebugReporter import maestro.cli.runner.resultview.AnsiResultView @@ -39,9 +39,9 @@ object TestRunner { debugOutputPath: Path ): Int { val debugOutput = FlowDebugOutput() - var aiOutput = FlowAIOutput( + var aiOutput = SingleFlowAIOutput( flowName = flowFile.nameWithoutExtension, - flowFilePath = flowFile.absolutePath, + flowFile = flowFile, ) val result = runCatching(resultView, maestro) { @@ -122,9 +122,9 @@ object TestRunner { commands, FlowDebugOutput(), // TODO: bartekpacia - make AI outputs work in continuous mode - FlowAIOutput( + SingleFlowAIOutput( flowName = "TODO", - flowFilePath = flowFile.absolutePath, + flowFile = flowFile, ), ) }.get() diff --git a/maestro-cli/src/main/java/maestro/cli/runner/TestSuiteInteractor.kt b/maestro-cli/src/main/java/maestro/cli/runner/TestSuiteInteractor.kt index 57e8114c0c..b8766d4be4 100644 --- a/maestro-cli/src/main/java/maestro/cli/runner/TestSuiteInteractor.kt +++ b/maestro-cli/src/main/java/maestro/cli/runner/TestSuiteInteractor.kt @@ -6,9 +6,9 @@ import maestro.cli.CliError import maestro.cli.device.Device import maestro.cli.model.FlowStatus import maestro.cli.model.TestExecutionSummary -import maestro.cli.report.AIOutput +import maestro.cli.report.SingleScreenFlowAIOutput import maestro.cli.report.CommandDebugMetadata -import maestro.cli.report.FlowAIOutput +import maestro.cli.report.SingleFlowAIOutput import maestro.cli.report.FlowDebugOutput import maestro.cli.report.TestDebugReporter import maestro.cli.report.TestSuiteReporter @@ -138,9 +138,9 @@ class TestSuiteInteractor( var errorMessage: String? = null val debugOutput = FlowDebugOutput() - val aiOutput = FlowAIOutput( + val aiOutput = SingleFlowAIOutput( flowName = flowFile.nameWithoutExtension, - flowFilePath = flowFile.absolutePath, + flowFile = flowFile, ) fun takeDebugScreenshot(status: CommandStatus): File? { @@ -228,8 +228,8 @@ class TestSuiteInteractor( onCommandGeneratedOutput = { command, defects, screenshot -> logger.info("${command.description()} generated output") val screenshotPath = writeAIscreenshot(screenshot) - aiOutput.outputs.add( - AIOutput( + aiOutput.screenOutputs.add( + SingleScreenFlowAIOutput( screenshotPath = screenshotPath, defects = defects, )