From 428a6cdd0893c59f12d413cbed3986d1fe0a5b46 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 13 Jun 2024 16:48:42 +0200 Subject: [PATCH] [TT-1218] update testlistgenerator for ccip (#990) --- tools/testlistgenerator/README.md | 101 +++++++---- tools/testlistgenerator/go.mod | 7 +- tools/testlistgenerator/go.sum | 8 + tools/testlistgenerator/main.go | 242 ++++++++++++++++++--------- tools/testlistgenerator/main_test.go | 127 +++++++++----- tools/testlistgenerator/package.json | 2 +- 6 files changed, 331 insertions(+), 156 deletions(-) diff --git a/tools/testlistgenerator/README.md b/tools/testlistgenerator/README.md index f93e02635..ead062853 100644 --- a/tools/testlistgenerator/README.md +++ b/tools/testlistgenerator/README.md @@ -5,13 +5,13 @@ This Go script builds a JSON file containing tests to be run for a given product ## Usage ```bash -go run main.go +go run main.go -t -o -p -r -f -e -d -n [-c ] [-w ] ``` ### Example ```bash -go run main.go 'test_list.json' 'ocr' 'TestOCR.*' './smoke/ocr_test.go' 'besu' 'hyperledger/besu:21.0.0,hyperledger/besu:22.0.0' +go run main.go -t "emv-test" -o "test_list.json" -p "ocr" -r "TestOCR.*" -f "./smoke/ocr_test.go" -e "besu" -d "hyperledger/besu:21.0.0,hyperledger/besu:22.0.0" -n "ubuntu-latest" ``` ## Output @@ -19,57 +19,96 @@ go run main.go 'test_list.json' 'ocr' 'TestOCR.*' './smoke/ocr_test.go' 'besu' ' The script generates or updates a JSON file with entries structured as follows: ```json -{ - "tests": [ - { - "product": "ocr", - "test_regex": "TestOCR.*", - "file": "./smoke/ocr_test.go", - "eth_implementation": "besu", - "docker_image": "hyperledger/besu:21.0.0" - }, - { - "product": "ocr", - "test_regex": "TestOCR.*", - "file": "./smoke/ocr_test.go", - "eth_implementation": "besu", - "docker_image": "hyperledger/besu:22.0.0" - } - ] -} +[ + { + "name": "emv-test-01", + "os": "ubuntu-latest", + "product": "ocr", + "eth_implementation": "besu", + "docker_image": "hyperledger/besu:21.0.0", + "run": "-run 'TestOCR.*' ./smoke/ocr_test.go" + }, + { + "name": "emv-test-02", + "os": "ubuntu-latest", + "product": "ocr", + "eth_implementation": "besu", + "docker_image": "hyperledger/besu:22.0.0", + "run": "-run 'TestOCR.*' ./smoke/ocr_test.go" + } +] +``` + +If the script is run with optional `chain_id` flag the output is slightly different: + +```bash +go run main.go -t "emv-test" -o "test_list.json" -p "ocr" -r "TestOCR.*" -f "./smoke/ocr_test.go" -e "besu" -d "hyperledger/besu:21.0.0,hyperledger/besu:22.0.0" -n "ubuntu-latest" -c 1337,2337 -w "mainnet,ropsten" +``` + +Output: + +```json +[ + { + "name": "emv-test-01", + "os": "ubuntu-latest", + "product": "ocr", + "eth_implementation": "besu", + "docker_image": "1337=hyperledger/besu:21.0.0,2337=hyperledger/besu:21.0.0", + "run": "-run 'TestOCR.*' ./smoke/ocr_test.go", + "networks": "mainnet,ropsten" + }, + { + "name": "emv-test-02", + "os": "ubuntu-latest", + "product": "ocr", + "eth_implementation": "besu", + "docker_image": "1337=hyperledger/besu:22.0.0,2337=hyperledger/besu:22.0.0", + "run": "-run 'TestOCR.*' ./smoke/ocr_test.go", + "networks": "mainnet,ropsten" + } +] ``` ## Command Line Arguments -- ``: The name of the JSON file where the test entries will be stored. -- ``: The name of the product for which the tests are being generated. -- ``: The regular expression to match test names. -- ``: The file path where the tests are defined. -- ``: The name of the Ethereum implementation. -- ``: A comma-separated list of Docker images to be used. +- `-t `: A prefix for the test name. +- `-o `: The name of the JSON file where the test entries will be stored. +- `-p `: The name of the product for which the tests are being generated. +- `-r `: The regular expression to match test names. +- `-f `: The file path where the tests are defined. +- `-e `: The name of the Ethereum implementation. +- `-d `: A comma-separated list of Docker images to be used. +- `-n `: The node label for the test environment. +- `-c `: (Optional) A comma-separated list of chain IDs to associate with each Docker image. +- `-w `: (Optional) A comma-separated list of networks. ## Error Handling The script will panic and display error messages in the following scenarios: - Insufficient command line arguments. -- Empty parameters for output_file_name, product, test_regex, file, eth_implementation, or docker_images. +- Empty parameters for test_name, output_file_name, product, test_regex, file, eth_implementation, docker_images, or node_label. - Invalid Docker image format (should include a version tag). - Invalid regular expression for test_regex. +- Invalid or non-integer chain IDs. - Errors in file operations (opening, reading, writing). ## Detailed Steps -1. **Argument Parsing**: The script expects at least 7 command line arguments. It splits the `` argument into a slice. -2. **Validation**: It validates the input parameters, ensuring none are empty and the regular expression compiles. +1. **Argument Parsing**: The script uses Cobra to parse command line arguments. +2. **Validation**: It validates the input parameters, ensuring none are empty, and the regular expression compiles. 3. **File Operations**: - If the output file exists, it reads and unmarshals the content. - If it doesn't exist, it creates a new file. -4. **Appending Entries**: For each Docker image, it creates a new `OutputEntry` and appends it to the output. +4. **Appending Entries**: For each Docker image, it creates new `OutputEntry` objects and appends them to the output. 5. **JSON Marshaling**: It marshals the updated output to JSON and writes it back to the file. 6. **Completion Message**: Prints a success message indicating the number of tests added. ## Notes - Ensure the Docker image names include version tags, e.g., `hyperledger/besu:21.0.0`. -- The script appends new entries; it does not overwrite existing entries in the output file. +- The script appends new entries; it does not overwrite existing entries in the output file unless the output file is specified with new test entries. +- Optional parameters like `chain_ids` and `networks` allow for additional customization of the test entries. If `chain_ids` are provided, they will be included in the Docker image field. If `networks` are provided, they will be included as an additional field in the output JSON. + +This update reflects the recent changes to include flags for `chain_ids` and `networks`, and ensures the documentation matches the new argument names and structure. diff --git a/tools/testlistgenerator/go.mod b/tools/testlistgenerator/go.mod index e6a83617a..e6e1079cc 100644 --- a/tools/testlistgenerator/go.mod +++ b/tools/testlistgenerator/go.mod @@ -2,10 +2,15 @@ module github.com/smartcontractkit/chainlink-testing-framework/tools/testlistgen go 1.21.7 -require github.com/stretchr/testify v1.9.0 +require ( + github.com/spf13/cobra v1.8.0 + github.com/stretchr/testify v1.9.0 +) require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/tools/testlistgenerator/go.sum b/tools/testlistgenerator/go.sum index 60ce688a0..50cc2e6f7 100644 --- a/tools/testlistgenerator/go.sum +++ b/tools/testlistgenerator/go.sum @@ -1,7 +1,15 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/tools/testlistgenerator/main.go b/tools/testlistgenerator/main.go index 370b88208..866d786b3 100644 --- a/tools/testlistgenerator/main.go +++ b/tools/testlistgenerator/main.go @@ -2,12 +2,14 @@ package main import ( "encoding/json" - "errors" "fmt" "io" "os" "regexp" + "strconv" "strings" + + "github.com/spf13/cobra" ) type Input struct { @@ -16,107 +18,181 @@ type Input struct { File string `json:"file"` EthImplementation string `json:"eth_implementation"` DockerImages []string `json:"docker_images"` + ChainIDs []int `json:"chain_ids"` + Networks string `json:"networks"` } type OutputEntry struct { - Product string `json:"product"` - TestRegex string `json:"test_regex"` - File string `json:"file"` - EthImplementationName string `json:"eth_implementation"` - DockerImage string `json:"docker_image"` + Name string `json:"name"` + OS string `json:"os"` + Product string `json:"product"` + EthImplementation string `json:"eth_implementation"` + DockerImage string `json:"docker_image"` + Run string `json:"run"` + Networks string `json:"networks,omitempty"` } -type Output struct { - Entries []OutputEntry `json:"tests"` -} +func main() { + var ciTestId, outputFile, product, regex, file, ethImplementation, dockerImagesArg, nodeLabel, chainIDsArg, networksArg string + var helpArg bool + + var rootCmd = &cobra.Command{ + Use: "main", + Short: "This script builds a JSON file with the tests to be run for a given product and Ethereum implementation", + Long: `This script builds a JSON file with the tests to be run for a given product and Ethereum implementation. The JSON file can be used to generate a test matrix for CI. + +When -c/chain_ids flag is provided, the output docker image will have following format: =. +For example if run with -c=1337,2337 and -d=hyperledger/besu:21.0.0 we would get: 1337=hyperledger/besu:21.0.0,2337=hyperledger/besu:21.0.0. This is useful for CCIP, which requires at least two chains per test. + +The -w/networks flag is optional and can be used to specify the networks to run the tests on. If not provided, 'networks' will be omitted from the output. +`, + Run: func(cmd *cobra.Command, args []string) { + if helpArg { + fmt.Println(cmd.Long) + _ = cmd.Usage() + return + } + if ciTestId == "" || outputFile == "" || product == "" || regex == "" || file == "" || ethImplementation == "" || dockerImagesArg == "" { + fmt.Println(cmd.Short) + _ = cmd.Usage() + panic("All flags are required") + } -const ( - InsufficientArgsErr = `Usage: go run main.go ' ' -Example: go run main.go 'ocr' 'TestOCR.*' './smoke/ocr_test.go' 'besu' 'hyperledger/besu:21.0.0,hyperledger/besu:22.0.0'` - EmptyParameterErr = "parameter '%s' cannot be empty" -) + if nodeLabel == "" { + nodeLabel = "ubuntu-latest" + fmt.Fprintf(os.Stderr, "Node label not provided, using default: %s\n", nodeLabel) + } -// this script builds a JSON file with the compatibility tests to be run for a given product and Ethereum implementation -func main() { - if len(os.Args) < 7 { - panic(errors.New(InsufficientArgsErr)) - } + dockerImages := strings.Split(dockerImagesArg, ",") + + var chainIDs []int + if chainIDsArg != "" { + for _, idStr := range strings.Split(chainIDsArg, ",") { + id, err := strconv.Atoi(idStr) + if err != nil { + panic(fmt.Errorf("invalid chain ID: %s", idStr)) + } + chainIDs = append(chainIDs, id) + } + } - outputFile := os.Args[1] - if outputFile == "" { - panic(fmt.Errorf(EmptyParameterErr, "output_file_name")) - } - dockerImagesArg := os.Args[6] - dockerImages := strings.Split(dockerImagesArg, ",") - - input := Input{ - Product: os.Args[2], - TestRegex: os.Args[3], - File: os.Args[4], - EthImplementation: os.Args[5], - DockerImages: dockerImages, - } + var networks string + if networksArg != "" { + spl := strings.Split(networksArg, ",") + networks = strings.Join(spl, ",") + } - validateInput(input) - - var output Output - var file *os.File - if _, err := os.Stat(outputFile); err == nil { - file, err = os.OpenFile(outputFile, os.O_RDWR, 0644) - if err != nil { - panic(fmt.Errorf("error opening file: %v", err)) - } - defer func() { _ = file.Close() }() - - bytes, err := io.ReadAll(file) - if err != nil { - panic(fmt.Errorf("error reading file: %v", err)) - } - - if len(bytes) > 0 { - if err := json.Unmarshal(bytes, &output); err != nil { - panic(fmt.Errorf("error unmarshalling JSON: %v", err)) + input := Input{ + Product: product, + TestRegex: regex, + File: file, + EthImplementation: ethImplementation, + DockerImages: dockerImages, + ChainIDs: chainIDs, + Networks: networks, } - } - } else { - file, err = os.Create(outputFile) - if err != nil { - panic(fmt.Errorf("error creating file: %v", err)) - } - } - defer func() { _ = file.Close() }() - - for _, image := range dockerImages { - if !strings.Contains(image, ":") { - panic(fmt.Errorf("docker image format is invalid: %s", image)) - } - output.Entries = append(output.Entries, OutputEntry{ - Product: input.Product, - TestRegex: input.TestRegex, - File: input.File, - EthImplementationName: input.EthImplementation, - DockerImage: image, - }) + + validateInput(input) + + var output []OutputEntry + var file *os.File + var err error + counter := 1 + + if _, err = os.Stat(outputFile); err == nil { + file, err = os.OpenFile(outputFile, os.O_RDWR, 0644) + if err != nil { + panic(fmt.Errorf("error opening file: %v", err)) + } + defer func() { _ = file.Close() }() + + bytes, err := io.ReadAll(file) + if err != nil { + panic(fmt.Errorf("error reading file: %v", err)) + } + + if len(bytes) > 0 { + if err := json.Unmarshal(bytes, &output); err != nil { + panic(fmt.Errorf("error unmarshalling JSON: %v", err)) + } + counter = len(output) + 1 + } + } else { + file, err = os.Create(outputFile) + if err != nil { + panic(fmt.Errorf("error creating file: %v", err)) + } + } + defer func() { _ = file.Close() }() + + for _, image := range dockerImages { + if !strings.Contains(image, ":") { + panic(fmt.Errorf("docker image format is invalid: %s", image)) + } + + entry := OutputEntry{ + Name: fmt.Sprintf("%s-%02d", ciTestId, counter), + OS: nodeLabel, + Product: input.Product, + EthImplementation: input.EthImplementation, + DockerImage: parseImage(image, chainIDs), + Run: fmt.Sprintf("-run '%s' %s", input.TestRegex, input.File), + Networks: networks, + } + output = append(output, entry) + counter++ + + } + + newOutput, err := json.MarshalIndent(output, "", " ") + if err != nil { + panic(fmt.Errorf("error marshalling JSON: %v", err)) + } + + if _, err := file.WriteAt(newOutput, 0); err != nil { + panic(fmt.Errorf("error writing to file: %v", err)) + } + + fmt.Printf("%d test(s) for %s and %s added successfully!\n", len(dockerImages), input.Product, input.EthImplementation) + }, } - newOutput, err := json.MarshalIndent(output, "", " ") - if err != nil { - panic(fmt.Errorf("error marshalling JSON: %v", err)) + rootCmd.Flags().StringVarP(&ciTestId, "ci_test_id", "t", "", "Test id to use in CI") + rootCmd.Flags().StringVarP(&outputFile, "output_file", "o", "", "Output file name") + rootCmd.Flags().StringVarP(&product, "product", "p", "", "Product") + rootCmd.Flags().StringVarP(®ex, "regex", "r", "", "Test regex") + rootCmd.Flags().StringVarP(&file, "file", "f", "", "File") + rootCmd.Flags().StringVarP(ðImplementation, "eth_implementation", "e", "", "Ethereum implementation (e.g. besu, geth, erigon)") + rootCmd.Flags().StringVarP(&dockerImagesArg, "docker_images", "d", "", "Docker images (comma separated)") + rootCmd.Flags().StringVarP(&nodeLabel, "node_label", "n", "", "Node label (runner to use)") + rootCmd.Flags().StringVarP(&chainIDsArg, "chain_ids", "c", "", "Chain IDs (comma separated)") + rootCmd.Flags().StringVarP(&networksArg, "networks", "w", "", "Networks (comma separated)") + rootCmd.Flags().BoolVarP(&helpArg, "help", "h", false, "Display help") + + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) } +} - if _, err := file.WriteAt(newOutput, 0); err != nil { - panic(fmt.Errorf("error writing to file: %v", err)) +func parseImage(image string, chainIDs []int) string { + if len(chainIDs) == 0 { + return image } - fmt.Printf("%d compatibility test(s) for %s and %s added successfully!\n", len(dockerImages), input.Product, input.EthImplementation) + var images []string + for _, id := range chainIDs { + images = append(images, fmt.Sprintf("%d=%s", id, image)) + } + return strings.Join(images, ",") } func validateInput(input Input) { if input.Product == "" { - panic(fmt.Errorf(EmptyParameterErr, "product")) + panic(fmt.Errorf("parameter 'product' cannot be empty")) } if input.TestRegex == "" { - panic(fmt.Errorf(EmptyParameterErr, "test_regex")) + panic(fmt.Errorf("parameter 'test_regex' cannot be empty")) } if _, err := regexp.Compile(input.TestRegex); err != nil { @@ -124,12 +200,12 @@ func validateInput(input Input) { } if input.File == "" { - panic(fmt.Errorf(EmptyParameterErr, "file")) + panic(fmt.Errorf("parameter 'file' cannot be empty")) } if input.EthImplementation == "" { - panic(fmt.Errorf(EmptyParameterErr, "eth_implementation")) + panic(fmt.Errorf("parameter 'eth_implementation' cannot be empty")) } if len(input.DockerImages) == 0 || (len(input.DockerImages) == 1 && input.DockerImages[0] == "") { - panic(fmt.Errorf(EmptyParameterErr, "docker_images")) + panic(fmt.Errorf("parameter 'docker_images' cannot be empty")) } } diff --git a/tools/testlistgenerator/main_test.go b/tools/testlistgenerator/main_test.go index ec5d1413e..a23672d41 100644 --- a/tools/testlistgenerator/main_test.go +++ b/tools/testlistgenerator/main_test.go @@ -19,123 +19,170 @@ func TestMainFunction(t *testing.T) { } } - t.Run("MissingArguments", func(t *testing.T) { + t.Run("FileCreationAndWrite", func(t *testing.T) { resetEnv() - os.Args = []string{"main", "arg1", "arg2", "arg3", "arg4"} - require.PanicsWithError(t, InsufficientArgsErr, func() { main() }) + os.Args = []string{"main", "-t", "test", "-o", OutputFile, "-p", "ocr", "-r", "TestOCR.*", "-f", "./smoke/ocr_test.go", "-e", "besu", "-d", "hyperledger/besu:21.0.0,hyperledger/besu:22.0.0"} + require.NotPanics(t, func() { main() }) + + require.FileExists(t, OutputFile) + bytes, err := os.ReadFile(OutputFile) + require.NoError(t, err) + + var output []OutputEntry + err = json.Unmarshal(bytes, &output) + require.NoError(t, err) + require.Len(t, output, 2) + require.Equal(t, "ocr", output[0].Product) + require.Equal(t, "-run 'TestOCR.*' ./smoke/ocr_test.go", output[0].Run) + require.Equal(t, "ubuntu-latest", output[0].OS) + require.Equal(t, "besu", output[0].EthImplementation) + require.Equal(t, "hyperledger/besu:21.0.0", output[0].DockerImage) }) - t.Run("InvalidDockerImageFormat", func(t *testing.T) { + t.Run("FileCreationAndWriteWithChainIDs", func(t *testing.T) { resetEnv() - os.Args = []string{"main", OutputFile, "ocr", "TestOCR.*", "./smoke/ocr_test.go", "besu", "hyperledger/besu"} - require.PanicsWithError(t, fmt.Sprintf("docker image format is invalid: %s", "hyperledger/besu"), func() { main() }) + os.Args = []string{"main", "-t", "test", "-o", OutputFile, "-p", "ocr", "-r", "TestOCR.*", "-f", "./smoke/ocr_test.go", "-e", "besu", "-d", "hyperledger/besu:21.0.0", "-c", "1337,2337", "-n", "ubuntu-latest"} + require.NotPanics(t, func() { main() }) + + require.FileExists(t, OutputFile) + bytes, err := os.ReadFile(OutputFile) + require.NoError(t, err) + + var output []OutputEntry + err = json.Unmarshal(bytes, &output) + require.NoError(t, err) + require.Len(t, output, 1) + require.Equal(t, "ocr", output[0].Product) + require.Equal(t, "-run 'TestOCR.*' ./smoke/ocr_test.go", output[0].Run) + require.Equal(t, "ubuntu-latest", output[0].OS) + require.Equal(t, "besu", output[0].EthImplementation) + require.Equal(t, "1337=hyperledger/besu:21.0.0,2337=hyperledger/besu:21.0.0", output[0].DockerImage) }) - t.Run("FileCreationAndWrite", func(t *testing.T) { + t.Run("FileCreationAndWriteWithNetworksAndNodeLabel", func(t *testing.T) { resetEnv() - os.Args = []string{"main", OutputFile, "ocr", "TestOCR.*", "./smoke/ocr_test.go", "besu", "hyperledger/besu:21.0.0,hyperledger/besu:22.0.0"} + os.Args = []string{"main", "-t", "test", "-o", OutputFile, "-p", "ocr", "-r", "TestOCR.*", "-f", "./smoke/ocr_test.go", "-e", "besu", "-d", "hyperledger/besu:21.0.0", "-w", "mainnet,ropsten", "-n", "ubuntu-latest-2core"} require.NotPanics(t, func() { main() }) require.FileExists(t, OutputFile) bytes, err := os.ReadFile(OutputFile) require.NoError(t, err) - var output Output + var output []OutputEntry err = json.Unmarshal(bytes, &output) require.NoError(t, err) - require.Len(t, output.Entries, 2) - require.Equal(t, "ocr", output.Entries[0].Product) - require.Equal(t, "TestOCR.*", output.Entries[0].TestRegex) - require.Equal(t, "./smoke/ocr_test.go", output.Entries[0].File) - require.Equal(t, "besu", output.Entries[0].EthImplementationName) - require.Equal(t, "hyperledger/besu:21.0.0", output.Entries[0].DockerImage) + require.Len(t, output, 1) + require.Equal(t, "ocr", output[0].Product) + require.Equal(t, "-run 'TestOCR.*' ./smoke/ocr_test.go", output[0].Run) + require.Equal(t, "ubuntu-latest-2core", output[0].OS) + require.Equal(t, "besu", output[0].EthImplementation) + require.Equal(t, "hyperledger/besu:21.0.0", output[0].DockerImage) + require.Equal(t, "mainnet,ropsten", output[0].Networks) }) t.Run("AppendToFile", func(t *testing.T) { resetEnv() - os.Args = []string{"main", OutputFile, "ocr", "TestOCR.*", "./smoke/ocr_test.go", "besu", "hyperledger/besu:21.0.0,hyperledger/besu:22.0.0"} + os.Args = []string{"main", "-t", "test", "-o", OutputFile, "-p", "ocr", "-r", "TestOCR.*", "-f", "./smoke/ocr_test.go", "-e", "besu", "-d", "hyperledger/besu:21.0.0,hyperledger/besu:22.0.0"} require.NotPanics(t, func() { main() }) - os.Args = []string{"main", OutputFile, "ocr", "TestOCR.*", "./smoke/ocr_test.go", "geth", "ethereum/client-go:1.10.0"} + os.Args = []string{"main", "-t", "test", "-o", OutputFile, "-p", "ocr", "-r", "TestOCR.*", "-f", "./smoke/ocr_test.go", "-e", "geth", "-d", "ethereum/client-go:1.10.0", "-n", "ubuntu-latest"} require.NotPanics(t, func() { main() }) require.FileExists(t, OutputFile) bytes, err := os.ReadFile(OutputFile) require.NoError(t, err) - var output Output + var output []OutputEntry err = json.Unmarshal(bytes, &output) require.NoError(t, err) - require.Len(t, output.Entries, 3) + require.Len(t, output, 3) }) t.Run("OverwriteFile", func(t *testing.T) { resetEnv() - os.Args = []string{"main", OutputFile, "ocr", "TestOCR.*", "./smoke/ocr_test.go", "besu", "hyperledger/besu:21.0.0,hyperledger/besu:22.0.0"} + os.Args = []string{"main", "-t", "test", "-o", OutputFile, "-p", "ocr", "-r", "TestOCR.*", "-f", "./smoke/ocr_test.go", "-e", "besu", "-d", "hyperledger/besu:21.0.0,hyperledger/besu:22.0.0", "-n", "ubuntu-latest"} require.NotPanics(t, func() { main() }) require.FileExists(t, OutputFile) bytes, err := os.ReadFile(OutputFile) require.NoError(t, err) - var initialOutput Output + var initialOutput []OutputEntry err = json.Unmarshal(bytes, &initialOutput) require.NoError(t, err) - require.Len(t, initialOutput.Entries, 2) + require.Len(t, initialOutput, 2) - os.Args = []string{"main", OutputFile, "ocr", "TestOCR.*", "./smoke/ocr_test.go", "besu", "hyperledger/besu:22.0.0,hyperledger/besu:23.0.0"} + os.Args = []string{"main", "-t", "test", "-o", OutputFile, "-p", "ocr", "-r", "TestOCR.*", "-f", "./smoke/ocr_test.go", "-e", "besu", "-d", "hyperledger/besu:22.0.0,hyperledger/besu:23.0.0", "-n", "ubuntu-latest"} require.NotPanics(t, func() { main() }) require.FileExists(t, OutputFile) bytes, err = os.ReadFile(OutputFile) require.NoError(t, err) - var output Output + var output []OutputEntry err = json.Unmarshal(bytes, &output) require.NoError(t, err) - require.Len(t, output.Entries, 4) - require.Equal(t, "hyperledger/besu:23.0.0", output.Entries[3].DockerImage) + require.Len(t, output, 4) + require.Equal(t, "hyperledger/besu:23.0.0", output[3].DockerImage) + }) + + t.Run("MissingArguments", func(t *testing.T) { + resetEnv() + os.Args = []string{"main", "arg1", "arg2", "arg3", "arg4"} + require.Panics(t, func() { main() }) + }) + + t.Run("InvalidDockerImageFormat", func(t *testing.T) { + resetEnv() + os.Args = []string{"main", "-t", "test", "-o", OutputFile, "-p", "ocr", "-r", "TestOCR.*", "-f", "./smoke/ocr_test.go", "-e", "besu", "-d", "hyperledger/besu", "-n", "ubuntu-latest"} + require.PanicsWithError(t, fmt.Sprintf("docker image format is invalid: %s", "hyperledger/besu"), func() { main() }) }) t.Run("EmptyOutputFileName", func(t *testing.T) { resetEnv() - os.Args = []string{"main", "", "ocr", "TestOCR.*", "./smoke/ocr_test.go", "besu", "hyperledger/besu:21.0.0"} - require.PanicsWithError(t, fmt.Sprintf(EmptyParameterErr, "output_file_name"), func() { main() }) + os.Args = []string{"main", "-t", "test", "-o", "", "-p", "ocr", "-r", "TestOCR.*", "-f", "./smoke/ocr_test.go", "-e", "besu", "-d", "hyperledger/besu:21.0.0", "-n", "ubuntu-latest"} + require.Panics(t, func() { main() }) }) t.Run("EmptyProduct", func(t *testing.T) { resetEnv() - os.Args = []string{"main", OutputFile, "", "TestOCR.*", "./smoke/ocr_test.go", "besu", "hyperledger/besu:21.0.0"} - require.PanicsWithError(t, fmt.Sprintf(EmptyParameterErr, "product"), func() { main() }) + os.Args = []string{"main", "-t", "test", "-o", OutputFile, "-p", "", "-r", "TestOCR.*", "-f", "./smoke/ocr_test.go", "-e", "besu", "-d", "hyperledger/besu:21.0.0", "-n", "ubuntu-latest"} + require.Panics(t, func() { main() }) }) t.Run("EmptyTestRegex", func(t *testing.T) { resetEnv() - os.Args = []string{"main", OutputFile, "ocr", "", "./smoke/ocr_test.go", "besu", "hyperledger/besu:21.0.0"} - require.PanicsWithError(t, fmt.Sprintf(EmptyParameterErr, "test_regex"), func() { main() }) + os.Args = []string{"main", "-t", "test", "-o", OutputFile, "-p", "ocr", "-r", "", "-f", "./smoke/ocr_test.go", "-e", "besu", "-d", "hyperledger/besu:21.0.0", "-n", "ubuntu-latest"} + require.Panics(t, func() { main() }) }) t.Run("InvalidTestRegex", func(t *testing.T) { resetEnv() - os.Args = []string{"main", OutputFile, "ocr", "[invalid", "./smoke/ocr_test.go", "besu", "hyperledger/besu:21.0.0"} - require.PanicsWithError(t, fmt.Sprintf("failed to compile regex: %v", "error parsing regexp: missing closing ]: `[invalid`"), func() { main() }) + os.Args = []string{"main", "-t", "test", "-o", OutputFile, "-p", "ocr", "-r", "[invalid", "-f", "./smoke/ocr_test.go", "-e", "besu", "-d", "hyperledger/besu:21.0.0", "-n", "ubuntu-latest"} + require.Panics(t, func() { main() }) }) t.Run("EmptyFile", func(t *testing.T) { resetEnv() - os.Args = []string{"main", OutputFile, "ocr", "TestOCR.*", "", "besu", "hyperledger/besu:21.0.0"} - require.PanicsWithError(t, fmt.Sprintf(EmptyParameterErr, "file"), func() { main() }) + os.Args = []string{"main", "-t", "test", "-o", OutputFile, "-p", "ocr", "-r", "TestOCR.*", "-f", "", "-e", "besu", "-d", "hyperledger/besu:21.0.0", "-n", "ubuntu-latest"} + require.Panics(t, func() { main() }) }) t.Run("EmptyEthImplementation", func(t *testing.T) { resetEnv() - os.Args = []string{"main", OutputFile, "ocr", "TestOCR.*", "./smoke/ocr_test.go", "", "hyperledger/besu:21.0.0"} - require.PanicsWithError(t, fmt.Sprintf(EmptyParameterErr, "eth_implementation"), func() { main() }) + os.Args = []string{"main", "-t", "test", "-o", OutputFile, "-p", "ocr", "-r", "TestOCR.*", "-f", "./smoke/ocr_test.go", "-e", "", "-d", "hyperledger/besu:21.0.0", "-n", "ubuntu-latest"} + require.Panics(t, func() { main() }) }) t.Run("EmptyDockerImages", func(t *testing.T) { resetEnv() - os.Args = []string{"main", OutputFile, "ocr", "TestOCR.*", "./smoke/ocr_test.go", "besu", ""} - require.PanicsWithError(t, fmt.Sprintf(EmptyParameterErr, "docker_images"), func() { main() }) + os.Args = []string{"main", "-t", "test", "-o", OutputFile, "-p", "ocr", "-r", "TestOCR.*", "-f", "./smoke/ocr_test.go", "-e", "besu", "-d", "", "-n", "ubuntu-latest"} + require.Panics(t, func() { main() }) + }) + + t.Run("ChainIdsNotInteger", func(t *testing.T) { + resetEnv() + os.Args = []string{"main", "-t", "test", "-o", OutputFile, "-p", "ocr", "-r", "TestOCR.*", "-f", "./smoke/ocr_test.go", "-e", "besu", "-d", "hyperledger/besu:21.0.0", "-n", "ubuntu-latest", "-c", "2,invalid"} + require.Panics(t, func() { main() }) }) defer func() { resetEnv() }() diff --git a/tools/testlistgenerator/package.json b/tools/testlistgenerator/package.json index 80e4492fe..35d655234 100644 --- a/tools/testlistgenerator/package.json +++ b/tools/testlistgenerator/package.json @@ -1,5 +1,5 @@ { "name": "testlistgenerator", "description": "Tool to generate a JSON list of tests", - "version": "1.0.1" + "version": "1.1.0" }