From 66344b2a043f54d3e0a949f502f42f99c95de599 Mon Sep 17 00:00:00 2001 From: Roger Peppe Date: Tue, 8 Oct 2024 15:42:33 +0100 Subject: [PATCH] encoding/jsonschema: always write external_teststats.txt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This means that it's not necessary to run `go generate` to generate the external_teststats.txt file. Now that it's not used for this purpose, make the `teststats.go` program more specialized to listing failed tests. Signed-off-by: Roger Peppe Change-Id: If67247d1226728b894e644eb6d0e3a83b28ef5de Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1202308 Unity-Result: CUE porcuepine TryBot-Result: CUEcueckoo Reviewed-by: Daniel Martí --- encoding/jsonschema/external_test.go | 65 +++++++++++++++++- encoding/jsonschema/external_teststats.txt | 2 +- encoding/jsonschema/teststats.go | 77 +++------------------- 3 files changed, 73 insertions(+), 71 deletions(-) diff --git a/encoding/jsonschema/external_test.go b/encoding/jsonschema/external_test.go index 12f2c8ed726..10f0a252cb2 100644 --- a/encoding/jsonschema/external_test.go +++ b/encoding/jsonschema/external_test.go @@ -17,6 +17,7 @@ package jsonschema_test import ( stdjson "encoding/json" "fmt" + "io" "os" "path" "sort" @@ -40,7 +41,6 @@ import ( // The commit below references the JSON schema test main branch as of Sun May 19 19:01:03 2024 +0300 //go:generate go run vendor_external.go -- 9fc880bfb6d8ccd093bc82431f17d13681ffae8e -//go:generate go run teststats.go -o external_teststats.txt const testDir = "testdata/external" @@ -74,6 +74,8 @@ func TestExternal(t *testing.T) { } err = externaltest.WriteTestDir(testDir, tests) qt.Assert(t, qt.IsNil(err)) + err = writeExternalTestStats(testDir, tests) + qt.Assert(t, qt.IsNil(err)) } func runExternalSchemaTests(t *testing.T, m *cuetdtest.M, filename string, s *externaltest.Schema) { @@ -264,3 +266,64 @@ func sortedKeys[T any](m map[string]T) []string { sort.Strings(ks) return ks } + +func writeExternalTestStats(testDir string, tests map[string][]*externaltest.Schema) error { + outf, err := os.Create("external_teststats.txt") + if err != nil { + return err + } + defer outf.Close() + fmt.Fprintf(outf, "# Generated by CUE_UPDATE=1 go test. DO NOT EDIT\n") + fmt.Fprintf(outf, "v2:\n") + showStats(outf, "v2", false, tests) + fmt.Fprintf(outf, "\n") + fmt.Fprintf(outf, "v3:\n") + showStats(outf, "v3", false, tests) + fmt.Fprintf(outf, "\nOptional tests\n\n") + fmt.Fprintf(outf, "v2:\n") + showStats(outf, "v2", true, tests) + fmt.Fprintf(outf, "\n") + fmt.Fprintf(outf, "v3:\n") + showStats(outf, "v3", true, tests) + return nil +} + +func showStats(outw io.Writer, version string, showOptional bool, tests map[string][]*externaltest.Schema) { + schemaOK := 0 + schemaTot := 0 + testOK := 0 + testTot := 0 + schemaOKTestOK := 0 + schemaOKTestTot := 0 + for filename, schemas := range tests { + isOptional := strings.Contains(filename, "/optional/") + if isOptional != showOptional { + continue + } + for _, schema := range schemas { + schemaTot++ + if schema.Skip[version] == "" { + schemaOK++ + } + for _, test := range schema.Tests { + testTot++ + if test.Skip[version] == "" { + testOK++ + } + if schema.Skip[version] == "" { + schemaOKTestTot++ + if test.Skip[version] == "" { + schemaOKTestOK++ + } + } + } + } + } + fmt.Fprintf(outw, "\tschema extract (pass / total): %d / %d = %.1f%%\n", schemaOK, schemaTot, percent(schemaOK, schemaTot)) + fmt.Fprintf(outw, "\ttests (pass / total): %d / %d = %.1f%%\n", testOK, testTot, percent(testOK, testTot)) + fmt.Fprintf(outw, "\ttests on extracted schemas (pass / total): %d / %d = %.1f%%\n", schemaOKTestOK, schemaOKTestTot, percent(schemaOKTestOK, schemaOKTestTot)) +} + +func percent(a, b int) float64 { + return (float64(a) / float64(b)) * 100.0 +} diff --git a/encoding/jsonschema/external_teststats.txt b/encoding/jsonschema/external_teststats.txt index 698509b28da..951c2abb750 100644 --- a/encoding/jsonschema/external_teststats.txt +++ b/encoding/jsonschema/external_teststats.txt @@ -1,4 +1,4 @@ -# Generated by teststats. DO NOT EDIT +# Generated by CUE_UPDATE=1 go test. DO NOT EDIT v2: schema extract (pass / total): 993 / 1363 = 72.9% tests (pass / total): 3517 / 4803 = 73.2% diff --git a/encoding/jsonschema/teststats.go b/encoding/jsonschema/teststats.go index 28c432c68b7..26f09804013 100644 --- a/encoding/jsonschema/teststats.go +++ b/encoding/jsonschema/teststats.go @@ -25,85 +25,28 @@ import ( "os" "path" "sort" - "strings" "cuelang.org/go/cue/token" "cuelang.org/go/encoding/jsonschema/internal/externaltest" ) -var ( - list = flag.String("list", "", "list all failed tests for a given evaluator version") - out = flag.String("o", "", "write output to a file") -) - const testDir = "testdata/external" func main() { + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "usage: teststats version\n") + fmt.Fprintf(os.Stderr, "\nList all failed tests for the given evaluator version (e.g. v2 or v3)\n") + os.Exit(2) + } flag.Parse() tests, err := externaltest.ReadTestDir(testDir) if err != nil { log.Fatal(err) } - outw := os.Stdout - if *out != "" { - var err error - outw, err = os.Create(*out) - if err != nil { - log.Fatal(err) - } - fmt.Fprintf(outw, "# Generated by teststats. DO NOT EDIT\n") - } - if *list != "" { - listFailures(outw, *list, tests) - } else { - fmt.Fprintf(outw, "v2:\n") - showStats(outw, "v2", false, tests) - fmt.Fprintf(outw, "\n") - fmt.Fprintf(outw, "v3:\n") - showStats(outw, "v3", false, tests) - fmt.Fprintf(outw, "\nOptional tests\n\n") - fmt.Fprintf(outw, "v2:\n") - showStats(outw, "v2", true, tests) - fmt.Fprintf(outw, "\n") - fmt.Fprintf(outw, "v3:\n") - showStats(outw, "v3", true, tests) + if flag.NArg() != 1 { + flag.Usage() } -} - -func showStats(outw io.Writer, version string, showOptional bool, tests map[string][]*externaltest.Schema) { - schemaOK := 0 - schemaTot := 0 - testOK := 0 - testTot := 0 - schemaOKTestOK := 0 - schemaOKTestTot := 0 - for filename, schemas := range tests { - isOptional := strings.Contains(filename, "/optional/") - if isOptional != showOptional { - continue - } - for _, schema := range schemas { - schemaTot++ - if schema.Skip[version] == "" { - schemaOK++ - } - for _, test := range schema.Tests { - testTot++ - if test.Skip[version] == "" { - testOK++ - } - if schema.Skip[version] == "" { - schemaOKTestTot++ - if test.Skip[version] == "" { - schemaOKTestOK++ - } - } - } - } - } - fmt.Fprintf(outw, "\tschema extract (pass / total): %d / %d = %.1f%%\n", schemaOK, schemaTot, percent(schemaOK, schemaTot)) - fmt.Fprintf(outw, "\ttests (pass / total): %d / %d = %.1f%%\n", testOK, testTot, percent(testOK, testTot)) - fmt.Fprintf(outw, "\ttests on extracted schemas (pass / total): %d / %d = %.1f%%\n", schemaOKTestOK, schemaOKTestTot, percent(schemaOKTestOK, schemaOKTestTot)) + listFailures(os.Stdout, flag.Arg(0), tests) } func listFailures(outw io.Writer, version string, tests map[string][]*externaltest.Schema) { @@ -137,10 +80,6 @@ func testdataPos(p positioner) token.Position { return pp } -func percent(a, b int) float64 { - return (float64(a) / float64(b)) * 100.0 -} - func sortedKeys[T any](m map[string]T) []string { ks := make([]string, 0, len(m)) for k := range m {