feat(output): allow skipping failing test in output parse

cmd/hivecioutput/example-exclusions.json

@@ -0,0 +1,10 @@
+{
+    "testSuites": [
+      {
+        "name": "engine-transition",
+        "testCases": [
+          "Terminal blocks are gossiped"
+        ]
+      }
+    ]
+  }

cmd/hivecioutput/main.go

@@ -5,18 +5,19 @@ import (
 	"encoding/xml"
 	"flag"
 	"fmt"
+	"io"
 	"io/fs"
 	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strings"
 	"time"
 
-	"gopkg.in/inconshreveable/log15.v2"
-
 	"github.com/ethereum/hive/internal/libhive"
+	"gopkg.in/inconshreveable/log15.v2"
 )
 
 const (
@@ -30,6 +31,7 @@ type Testsuites struct {
 	Name      string   `xml:"name,attr"`
 	Tests     int      `xml:"tests,attr"`
 	Failures  int      `xml:"failures,attr"`
+	Skips     int      `xml:"skips,attr"`
 	Time      string   `xml:"time,attr"`
 	Testsuite []Testsuite
 }
@@ -41,6 +43,7 @@ type Testsuite struct {
 	Name     string   `xml:"name,attr"`
 	Tests    int      `xml:"tests,attr"`
 	Failures int      `xml:"failures,attr"`
+	Skips    int      `xml:"skips,attr"`
 	Time     string   `xml:"time,attr"`
 	Testcase []Testcase
 }
@@ -52,6 +55,7 @@ type Testcase struct {
 	Name    string   `xml:"name,attr"`
 	Time    string   `xml:"time,attr"`
 	Failure *Failure
+	Skipped *Skipped
 }
 
 type Failure struct {
@@ -61,52 +65,103 @@ type Failure struct {
 	Type    string   `xml:"type,attr"`
 }
 
+type Skipped struct {
+	XMLName xml.Name `xml:"skipped"`
+	Text    string   `xml:",chardata"`
+	Message string   `xml:"message,attr"`
+	Type    string   `xml:"type,attr"`
+}
+
+type Exclusions struct {
+	TestSuites []struct {
+		Name      string   `json:"name"`
+		TestCases []string `json:"testcases"`
+	} `json:"testSuites"`
+}
+
 func main() {
 	var exitcode bool
-	flag.BoolVar(&exitcode, "exitcode", true, "Return exit code 1 on failed tests")
+	flag.BoolVar(&exitcode, "exitcode", false, "Return exit code 1 on failed tests")
 
 	var (
-		resultsdir = flag.String("resultsdir", "/tmp/TestResults", "Results dir to scan")
-		outdir     = flag.String("outdir", "/tmp/", "Output dir for xunit xml")
+		resultsdir     = flag.String("resultsdir", "/tmp/TestResults", "Results dir to scan")
+		outdir         = flag.String("outdir", "/tmp/", "Output dir for xunit xml")
+		exclusionsFile = flag.String("exclusionsfile", "", "File containing list of test names to exclude")
 	)
 	flag.Parse()
 
 	log15.Info(fmt.Sprintf("loading results from %s", *resultsdir))
 	log15.Info(fmt.Sprintf("outputting xunit xml to %s", *outdir))
 	log15.Info(fmt.Sprintf("return exit code 1 on failed tests %v", exitcode))
+	log15.Info(fmt.Sprintf("exclusions file %s", *exclusionsFile))
+
+	// load exclusions
+	exclusions := &Exclusions{}
+	if *exclusionsFile != "" {
+		file, err := os.Open(*exclusionsFile)
+		if err != nil {
+			log15.Info(fmt.Sprintf("error opening exclusions file %s", *exclusionsFile))
+		}
+		bytes, err := io.ReadAll(file)
+		if err != nil {
+			log15.Info(fmt.Sprintf("error reading exclusions file %s", *exclusionsFile))
+		}
+		err = json.Unmarshal(bytes, &exclusions)
+		if err != nil {
+			log15.Info(fmt.Sprintf("error unmarshalling exclusions file %s", *exclusionsFile))
+		}
+		file.Close()
+
+		// print exclusions
+		if len(exclusions.TestSuites) > 0 {
+			log15.Info(fmt.Sprintf("excluded tests: %v", exclusions))
+		}
+	}
 
 	outputs := []*libhive.TestSuite{}
 
 	rd := os.DirFS(*resultsdir)
-	op, err := walkSummaryFiles(rd, ".", collectOutput, &outputs)
+	err := walkSummaryFiles(rd, ".", collectOutput, &outputs)
 	if err != nil {
 		log15.Info(fmt.Errorf("error reading results: %w", err).Error())
 		os.Exit(1)
 	}
 
-	outputXUnitXmlFile(&outputs, *outdir)
+	pass, run := outputXUnitXmlFile(&outputs, *outdir, exclusions)
 
-	if op {
+	// tests run and passed
+	if pass && run {
 		log15.Info("tests passed!")
 		os.Exit(0)
 	}
 
+	// no tests run
+	if !run {
+		log15.Info("no tests run!")
+		if exitcode {
+			os.Exit(1)
+		}
+	}
+
+	// tests run but failed
 	log15.Info("tests failed!")
 	if exitcode {
 		os.Exit(1)
 	}
 }
 
-func outputXUnitXmlFile(outputs *[]*libhive.TestSuite, path string) {
+func outputXUnitXmlFile(outputs *[]*libhive.TestSuite, path string, exclusions *Exclusions) (bool, bool) {
 	opTs := Testsuites{}
 	totalTests := 0
 	totalFailures := 0
+	totalSkipped := 0
 	suiteNo := 0
 
 	for _, ts := range *outputs {
 		log15.Info("test suite", "name", ts.Name)
 		tests := 0
 		failures := 0
+		skipped := 0
 		caseNo := 0
 		tsTime := time.Second * 0
 		suiteNo++
@@ -121,6 +176,7 @@ func outputXUnitXmlFile(outputs *[]*libhive.TestSuite, path string) {
 
 		for _, tc := range ts.TestCases {
 			caseNo++
+			testSkipped := false
 
 			tcTime := tc.End.Sub(tc.Start)
 			tsTime += tcTime
@@ -132,8 +188,11 @@ func outputXUnitXmlFile(outputs *[]*libhive.TestSuite, path string) {
 				Time: fmt.Sprintf("%v", tcTime.Seconds()),
 			}
 
-			tests++
-			if !tc.SummaryResult.Pass {
+			if contains(exclusions, ts.Name, tc.Name) {
+				testSkipped = true
+			}
+
+			if !tc.SummaryResult.Pass && !testSkipped {
 				failures++
 				oTc.Failure = &Failure{
 					Text:    tc.SummaryResult.Details,
@@ -142,20 +201,35 @@ func outputXUnitXmlFile(outputs *[]*libhive.TestSuite, path string) {
 				}
 			}
 
+			// only skip on failure, if we pass, might as well include in results
+			if !tc.SummaryResult.Pass && testSkipped {
+				skipped++
+				log15.Info("test skipped", "name", tc.Name)
+				oTc.Skipped = &Skipped{
+					Text:    tc.SummaryResult.Details,
+					Message: "Skipped",
+					Type:    "SKIPPED",
+				}
+			}
+
+			tests++
 			oTs.Testcase = append(oTs.Testcase, oTc)
 			oTs.Tests = tests
 			oTs.Failures = failures
+			oTs.Skips = skipped
 			oTs.Time = fmt.Sprintf("%v", tsTime.Seconds())
 		}
 
 		opTs.Testsuite = append(opTs.Testsuite, oTs)
 
 		totalTests += tests
 		totalFailures += failures
+		totalSkipped += skipped
 	}
 
 	opTs.Tests = totalTests
 	opTs.Failures = totalFailures
+	opTs.Skips = totalSkipped
 	opTs.ID = "0"
 	opTs.Name = "Hive Test Run"
 	opTs.Time = ""
@@ -166,58 +240,42 @@ func outputXUnitXmlFile(outputs *[]*libhive.TestSuite, path string) {
 	if err != nil {
 		log15.Error(err.Error())
 	}
-}
 
-func collectOutput(ts *libhive.TestSuite, outputs *[]*libhive.TestSuite) (bool, error) {
-	// collect output
-	*outputs = append(*outputs, ts)
+	testsRun := totalTests > 0
 
-	// determine whether any cases failed and break to return early if so
-	pass := true
-	for _, tc := range ts.TestCases {
-		if !tc.SummaryResult.Pass {
-			pass = false
-			break
-		}
+	if totalFailures > 0 {
+		return false, testsRun
 	}
+	return true, testsRun
+}
 
-	return pass, nil
+func collectOutput(ts *libhive.TestSuite, outputs *[]*libhive.TestSuite) {
+	*outputs = append(*outputs, ts)
 }
 
-type parseTs func(*libhive.TestSuite, *[]*libhive.TestSuite) (bool, error)
+type parseTs func(*libhive.TestSuite, *[]*libhive.TestSuite)
 
-func walkSummaryFiles(fsys fs.FS, dir string, parse parseTs, ts *[]*libhive.TestSuite) (bool, error) {
+func walkSummaryFiles(fsys fs.FS, dir string, parse parseTs, ts *[]*libhive.TestSuite) error {
 	logfiles, err := fs.ReadDir(fsys, dir)
 	if err != nil {
-		return false, err
+		return err
 	}
 	// Sort by name newest-first.
 	sort.Slice(logfiles, func(i, j int) bool {
 		return logfiles[i].Name() > logfiles[j].Name()
 	})
 
-	overallPass := true
-
 	for _, entry := range logfiles {
 		name := entry.Name()
 		if entry.IsDir() || !strings.HasSuffix(name, ".json") || skipFile(name) {
 			continue
 		}
 		suite, _ := parseSuite(fsys, filepath.Join(dir, name))
 		if suite != nil {
-			pass, err := parse(suite, ts)
-			if err != nil {
-				return false, err
-			}
-
-			// set overall pass false on failure, allow for
-			// loop to continue to collect all results
-			if !pass {
-				overallPass = false
-			}
+			parse(suite, ts)
 		}
 	}
-	return overallPass, nil
+	return nil
 }
 
 func parseSuite(fsys fs.FS, path string) (*libhive.TestSuite, fs.FileInfo) {
@@ -253,3 +311,22 @@ func suiteValid(s *libhive.TestSuite) bool {
 func skipFile(f string) bool {
 	return f == "errorReport.json" || f == "containerErrorReport.json" || strings.HasPrefix(f, ".")
 }
+
+func contains(ex *Exclusions, tsName, tcName string) bool {
+	for _, ts := range ex.TestSuites {
+		if tsName == ts.Name {
+			for _, tc := range ts.TestCases {
+				if removeImageName(tcName) == tc {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+func removeImageName(s string) string {
+	// regex to remove the last bracketed string (image name appended to test name)
+	reg := regexp.MustCompile(`\([^)]*\)$`)
+	return strings.Trim(reg.ReplaceAllString(s, ""), " ")
+}

go.mod

@@ -1,6 +1,6 @@
 module github.com/ethereum/hive
 
-go 1.17
+go 1.16
 
 require (
 	github.com/davecgh/go-spew v1.1.1
@@ -13,54 +13,13 @@ require (
 )
 
 require (
-	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
-	github.com/Microsoft/go-winio v0.5.2 // indirect
-	github.com/Microsoft/hcsshim v0.9.3 // indirect
 	github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect
-	github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
 	github.com/allegro/bigcache v1.2.1 // indirect
-	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/containerd/cgroups v1.0.3 // indirect
-	github.com/containerd/containerd v1.6.6 // indirect
-	github.com/deckarep/golang-set v1.8.0 // indirect
-	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
-	github.com/docker/docker v20.10.17+incompatible // indirect
-	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/docker/go-units v0.4.0 // indirect
-	github.com/edsrzf/mmap-go v1.0.0 // indirect
 	github.com/go-ole/go-ole v1.2.5 // indirect
 	github.com/go-stack/stack v1.8.1 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/snappy v0.0.4 // indirect
-	github.com/gorilla/websocket v1.4.2 // indirect
-	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
-	github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect
-	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
-	github.com/holiman/uint256 v1.2.0 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/mattn/go-runewidth v0.0.9 // indirect
-	github.com/moby/sys/mount v0.3.3 // indirect
-	github.com/moby/sys/mountinfo v0.6.2 // indirect
-	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
-	github.com/morikuni/aec v1.0.0 // indirect
-	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
-	github.com/opencontainers/runc v1.1.2 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/tsdb v0.10.0 // indirect
-	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
-	github.com/sirupsen/logrus v1.8.1 // indirect
-	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
-	github.com/tklauser/go-sysconf v0.3.5 // indirect
-	github.com/tklauser/numcpus v0.2.2 // indirect
-	go.opencensus.io v0.23.0 // indirect
 	golang.org/x/crypto v0.0.0-20211202192323-5770296d904e // indirect
-	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
-	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 )
 
 replace github.com/ethereum/hive/hiveproxy => ./hiveproxy