From 8b58efabf96e6305c4a59536dd6d09900285cac9 Mon Sep 17 00:00:00 2001 From: Vincent Le Goff Date: Thu, 23 Mar 2023 15:25:59 +0100 Subject: [PATCH] feat(convert): add new format --- cmd/common.go | 2 +- cmd/common_konnect.go | 2 +- cmd/convert.go | 29 +++++--- cmd/validate.go | 2 +- convert/{testdata/2 => }/.gitignore | 0 convert/convert.go | 75 +++++++++++++++++--- convert/convert_test.go | 91 +++++++++++++++++++++++-- convert/testdata/3/output.yaml | 10 --- convert/testdata/4/output.yaml | 10 --- convert/testdata/5/input.yaml | 35 ++++++++++ convert/testdata/5/output-expected.yaml | 65 ++++++++++++++++++ convert/testdata/6/input.yaml | 41 +++++++++++ convert/testdata/6/output-expected.yaml | 62 +++++++++++++++++ convert/testdata/7/input-1.yaml | 25 +++++++ convert/testdata/7/input-2.yaml | 12 ++++ convert/testdata/7/output-expected.yaml | 65 ++++++++++++++++++ convert/testdata/8/input.yaml | 9 +++ convert/testdata/8/output-expected.yaml | 13 ++++ convert/testdata/9/input.yaml | 9 +++ convert/testdata/9/output-expected.yaml | 13 ++++ file/reader.go | 4 +- file/reader_test.go | 8 +-- file/readfile.go | 62 ++++++++++++++--- file/readfile_test.go | 2 +- file/writer.go | 33 +++++---- 25 files changed, 602 insertions(+), 77 deletions(-) rename convert/{testdata/2 => }/.gitignore (100%) delete mode 100644 convert/testdata/3/output.yaml delete mode 100644 convert/testdata/4/output.yaml create mode 100644 convert/testdata/5/input.yaml create mode 100644 convert/testdata/5/output-expected.yaml create mode 100644 convert/testdata/6/input.yaml create mode 100644 convert/testdata/6/output-expected.yaml create mode 100644 convert/testdata/7/input-1.yaml create mode 100644 convert/testdata/7/input-2.yaml create mode 100644 convert/testdata/7/output-expected.yaml create mode 100644 convert/testdata/8/input.yaml create mode 100644 convert/testdata/8/output-expected.yaml create mode 100644 convert/testdata/9/input.yaml create mode 100644 convert/testdata/9/output-expected.yaml diff --git a/cmd/common.go b/cmd/common.go index ede3d0421..2ba0bdac1 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -84,7 +84,7 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism, delay int, workspace string, ) error { // read target file - targetContent, err := file.GetContentFromFiles(filenames) + targetContent, err := file.GetContentFromFiles(filenames, false) if err != nil { return err } diff --git a/cmd/common_konnect.go b/cmd/common_konnect.go index fbe4a1feb..cd38ff5dc 100644 --- a/cmd/common_konnect.go +++ b/cmd/common_konnect.go @@ -183,7 +183,7 @@ func syncKonnect(ctx context.Context, httpClient := utils.HTTPClient() // read target file - targetContent, err := file.GetContentFromFiles(filenames) + targetContent, err := file.GetContentFromFiles(filenames, false) if err != nil { return err } diff --git a/cmd/convert.go b/cmd/convert.go index 4e0fac656..2dbfe0c6f 100644 --- a/cmd/convert.go +++ b/cmd/convert.go @@ -11,11 +11,12 @@ import ( ) var ( - convertCmdSourceFormat string - convertCmdDestinationFormat string - convertCmdInputFile string - convertCmdOutputFile string - convertCmdAssumeYes bool + convertCmdSourceFormat string + convertCmdDestinationFormat string + convertCmdInputFile []string + convertCmdOutputFile string + convertCmdAssumeYes bool + convertCmdDisableMockEnvVars bool ) // newConvertCmd represents the convert command @@ -37,7 +38,7 @@ can be converted into a 'konnect' configuration file.`, return err } - if convertCmdInputFile != "" { + if len(convertCmdInputFile) != 0 { if yes, err := utils.ConfirmFileOverwrite( convertCmdOutputFile, "", convertCmdAssumeYes, ); err != nil { @@ -46,7 +47,11 @@ can be converted into a 'konnect' configuration file.`, return nil } - err = convert.Convert(convertCmdInputFile, convertCmdOutputFile, sourceFormat, destinationFormat) + err = convert.Convert(convertCmdInputFile, + convertCmdOutputFile, + sourceFormat, + destinationFormat, + !convertCmdDisableMockEnvVars) if err != nil { return fmt.Errorf("converting file: %v", err) } @@ -60,7 +65,7 @@ can be converted into a 'konnect' configuration file.`, return fmt.Errorf("getting files from directory: %w", err) } for _, filename := range files { - err = convert.Convert(filename, filename, sourceFormat, destinationFormat) + err = convert.Convert([]string{filename}, filename, sourceFormat, destinationFormat, !convertCmdDisableMockEnvVars) if err != nil { return fmt.Errorf("converting '%s' file: %v", filename, err) } @@ -75,18 +80,20 @@ can be converted into a 'konnect' configuration file.`, }, } - sourceFormats := []convert.Format{convert.FormatKongGateway, convert.FormatKongGateway2x} + sourceFormats := []convert.Format{convert.FormatKongGateway, convert.FormatKongGateway2x, convert.FormatDistributed} destinationFormats := []convert.Format{convert.FormatKonnect, convert.FormatKongGateway3x} convertCmd.Flags().StringVar(&convertCmdSourceFormat, "from", "", fmt.Sprintf("format of the source file, allowed formats: %v", sourceFormats)) convertCmd.Flags().StringVar(&convertCmdDestinationFormat, "to", "", fmt.Sprintf("desired format of the output, allowed formats: %v", destinationFormats)) - convertCmd.Flags().StringVar(&convertCmdInputFile, "input-file", "", - "configuration file to be converted. Use `-` to read from stdin.") + convertCmd.Flags().StringSliceVar(&convertCmdInputFile, "input-file", []string{}, + "configuration files to be converted. Use `-` to read from stdin.") convertCmd.Flags().StringVar(&convertCmdOutputFile, "output-file", "kong.yaml", "file to write configuration to after conversion. Use `-` to write to stdout.") convertCmd.Flags().BoolVar(&convertCmdAssumeYes, "yes", false, "assume `yes` to prompts and run non-interactively.") + convertCmd.Flags().BoolVar(&convertCmdDisableMockEnvVars, "disable-mock-env", + false, "disables the mocking of environment variables.") return convertCmd } diff --git a/cmd/validate.go b/cmd/validate.go index 2035d9933..66ce4d758 100644 --- a/cmd/validate.go +++ b/cmd/validate.go @@ -43,7 +43,7 @@ this command unless --online flag is used. _ = sendAnalytics("validate", "", mode) // read target file // this does json schema validation as well - targetContent, err := file.GetContentFromFiles(validateCmdKongStateFile) + targetContent, err := file.GetContentFromFiles(validateCmdKongStateFile, false) if err != nil { return err } diff --git a/convert/testdata/2/.gitignore b/convert/.gitignore similarity index 100% rename from convert/testdata/2/.gitignore rename to convert/.gitignore diff --git a/convert/convert.go b/convert/convert.go index 49843efd9..774b0cbc6 100644 --- a/convert/convert.go +++ b/convert/convert.go @@ -1,11 +1,15 @@ package convert import ( + "context" "fmt" "strings" + "github.com/blang/semver/v4" "github.com/kong/deck/cprint" + "github.com/kong/deck/dump" "github.com/kong/deck/file" + "github.com/kong/deck/state" "github.com/kong/deck/utils" "github.com/kong/go-kong/kong" ) @@ -13,6 +17,8 @@ import ( type Format string const ( + // FormatDistributed represents the Deck configuration format. + FormatDistributed Format = "distributed" // FormatKongGateway represents the Kong gateway format. FormatKongGateway Format = "kong-gateway" // FormatKonnect represents the Konnect format. @@ -37,38 +43,49 @@ func ParseFormat(key string) (Format, error) { return FormatKongGateway2x, nil case FormatKongGateway3x: return FormatKongGateway3x, nil + case FormatDistributed: + return FormatDistributed, nil default: return "", fmt.Errorf("invalid format: '%v'", key) } } -func Convert(inputFilename, outputFilename string, from, to Format) error { +func Convert(inputFilenames []string, outputFilename string, from, to Format, mockEnvVars bool) error { + const outputFormat = file.YAML var ( outputContent *file.Content err error ) - inputContent, err := file.GetContentFromFiles([]string{inputFilename}) + inputContent, err := file.GetContentFromFiles(inputFilenames, mockEnvVars) if err != nil { return err } switch { case from == FormatKongGateway && to == FormatKonnect: - outputContent, err = convertKongGatewayToKonnect(inputContent) - if err != nil { - return err + if len(inputFilenames) > 1 { + return fmt.Errorf("only one input file can be provided when converting from Kong to Konnect format") } + outputContent, err = convertKongGatewayToKonnect(inputContent) case from == FormatKongGateway2x && to == FormatKongGateway3x: - outputContent, err = convertKongGateway2xTo3x(inputContent, inputFilename) - if err != nil { - return err + if len(inputFilenames) > 1 { + return fmt.Errorf("only one input file can be provided when converting from Kong 2.x to Kong 3.x format") } + outputContent, err = convertKongGateway2xTo3x(inputContent, inputFilenames[0]) + case from == FormatDistributed && to == FormatKongGateway, + from == FormatDistributed && to == FormatKongGateway2x, + from == FormatDistributed && to == FormatKongGateway3x: + outputContent, err = convertDistributedToKong(inputContent, outputFilename, outputFormat, to) default: return fmt.Errorf("cannot convert from '%s' to '%s' format", from, to) } - err = file.WriteContentToFile(outputContent, outputFilename, file.YAML) + if err != nil { + return err + } + + err = file.WriteContentToFile(outputContent, outputFilename, outputFormat) if err != nil { return err } @@ -195,3 +212,43 @@ func removeServiceName(service *file.FService) *file.FService { serviceCopy.ID = kong.String(utils.UUID()) return serviceCopy } + +// convertDistributedToKong is used to convert one or many distributed format +// files to create one Kong Gateway declarative config. It also leverages some +// deck features like the defaults/centralized plugin configurations. +func convertDistributedToKong( + targetContent *file.Content, + outputFilename string, + format file.Format, + kongFormat Format, +) (*file.Content, error) { + var version semver.Version + + switch kongFormat { //nolint:exhaustive + case FormatKongGateway, + FormatKongGateway3x: + version = semver.Version{Major: 3, Minor: 0} + case FormatKongGateway2x: + version = semver.Version{Major: 2, Minor: 8} + } + + s, _ := state.NewKongState() + rawState, err := file.Get(context.Background(), targetContent, file.RenderConfig{ + CurrentState: s, + KongVersion: version, + }, dump.Config{}, nil) + if err != nil { + return nil, err + } + targetState, err := state.Get(rawState) + if err != nil { + return nil, err + } + + // file.KongStateToContent calls file.WriteContentToFile + return file.KongStateToContent(targetState, file.WriteConfig{ + Filename: outputFilename, + FileFormat: format, + KongVersion: version.String(), + }) +} diff --git a/convert/convert_test.go b/convert/convert_test.go index ab341dfc4..dfc8f7575 100644 --- a/convert/convert_test.go +++ b/convert/convert_test.go @@ -158,9 +158,12 @@ func zeroOutID(sp file.FServicePackage) file.FServicePackage { func Test_Convert(t *testing.T) { type args struct { inputFilename string + inputFilenames []string outputFilename string fromFormat Format toFormat Format + disableMocks bool + envVars map[string]string expectedOutputFilename string } tests := []struct { @@ -237,21 +240,101 @@ func Test_Convert(t *testing.T) { }, wantErr: false, }, + { + name: "converts from distributed to kong gateway (no deck specific fields)", + args: args{ + inputFilename: "testdata/5/input.yaml", + outputFilename: "testdata/5/output.yaml", + expectedOutputFilename: "testdata/5/output-expected.yaml", + fromFormat: FormatDistributed, + toFormat: FormatKongGateway, + }, + wantErr: false, + }, + { + name: "converts from distributed to kong gateway with defaults", + args: args{ + inputFilename: "testdata/6/input.yaml", + outputFilename: "testdata/6/output.yaml", + expectedOutputFilename: "testdata/6/output-expected.yaml", + fromFormat: FormatDistributed, + toFormat: FormatKongGateway, + }, + wantErr: false, + }, + { + name: "converts from distributed to kong gateway with multiple files", + args: args{ + inputFilenames: []string{"testdata/7/input-1.yaml", "testdata/7/input-2.yaml"}, + outputFilename: "testdata/7/output.yaml", + expectedOutputFilename: "testdata/7/output-expected.yaml", + fromFormat: FormatDistributed, + toFormat: FormatKongGateway, + }, + wantErr: false, + }, + { + name: "converts from distributed to kong gateway with env variables", + args: args{ + inputFilenames: []string{"testdata/8/input.yaml"}, + outputFilename: "testdata/8/output.yaml", + expectedOutputFilename: "testdata/8/output-expected.yaml", + fromFormat: FormatDistributed, + toFormat: FormatKongGateway, + disableMocks: true, + envVars: map[string]string{ + "DECK_MOCKBIN_HOST": "mockbin.org", + "DECK_MOCKBIN_ENABLED": "true", + "DECK_WRITE_TIMEOUT": "777", + "DECK_FOO_FLOAT": "666", + }, + }, + wantErr: false, + }, + { + name: "converts from distributed to kong gateway with env variables (mocked)", + args: args{ + inputFilenames: []string{"testdata/9/input.yaml"}, + outputFilename: "testdata/9/output.yaml", + expectedOutputFilename: "testdata/9/output-expected.yaml", + fromFormat: FormatDistributed, + toFormat: FormatKongGateway, + disableMocks: false, + }, + wantErr: false, + }, + { + name: "errors from distributed to kong gateway with env variables not set", + args: args{ + inputFilenames: []string{"testdata/9/input.yaml"}, + fromFormat: FormatDistributed, + toFormat: FormatKongGateway, + disableMocks: true, + }, + wantErr: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := Convert(tt.args.inputFilename, tt.args.outputFilename, tt.args.fromFormat, - tt.args.toFormat) + inputFiles := tt.args.inputFilenames + if tt.args.inputFilename != "" { + inputFiles = []string{tt.args.inputFilename} + } + for k, v := range tt.args.envVars { + t.Setenv(k, v) + } + err := Convert(inputFiles, tt.args.outputFilename, tt.args.fromFormat, + tt.args.toFormat, !tt.args.disableMocks) if (err != nil) != tt.wantErr { t.Errorf("Convert() error = %v, wantErr %v", err, tt.wantErr) } if err == nil { - got, err := file.GetContentFromFiles([]string{tt.args.outputFilename}) + got, err := file.GetContentFromFiles([]string{tt.args.outputFilename}, !tt.args.disableMocks) if err != nil { t.Errorf("failed to read output file: %v", err) } - want, err := file.GetContentFromFiles([]string{tt.args.expectedOutputFilename}) + want, err := file.GetContentFromFiles([]string{tt.args.expectedOutputFilename}, !tt.args.disableMocks) if err != nil { t.Errorf("failed to read output file: %v", err) } diff --git a/convert/testdata/3/output.yaml b/convert/testdata/3/output.yaml deleted file mode 100644 index eafd39703..000000000 --- a/convert/testdata/3/output.yaml +++ /dev/null @@ -1,10 +0,0 @@ -_format_version: "3.0" -services: -- host: mockbin.org - name: svc1 - path: /status/200 - routes: - - name: r1 - paths: - - ~/status/\d+ - - ~/code/\d+ diff --git a/convert/testdata/4/output.yaml b/convert/testdata/4/output.yaml deleted file mode 100644 index eafd39703..000000000 --- a/convert/testdata/4/output.yaml +++ /dev/null @@ -1,10 +0,0 @@ -_format_version: "3.0" -services: -- host: mockbin.org - name: svc1 - path: /status/200 - routes: - - name: r1 - paths: - - ~/status/\d+ - - ~/code/\d+ diff --git a/convert/testdata/5/input.yaml b/convert/testdata/5/input.yaml new file mode 100644 index 000000000..fafdb57d5 --- /dev/null +++ b/convert/testdata/5/input.yaml @@ -0,0 +1,35 @@ +_format_version: "3.0" +services: +- name: svc1 + host: mockbin.org + tags: + - team-svc1 + routes: + - name: r1 + https_redirect_status_code: 301 + paths: + - /r1 +- name: svc2 + host: mockbin.org + routes: + - name: r2 + https_redirect_status_code: 301 + paths: + - /r2 +- name: svc3 + host: mockbin.org + port: 80 + routes: + - name: r3 + https_redirect_status_code: 301 + paths: + - /r3 + methods: + - GET +plugins: +- name: prometheus + enabled: true + run_on: first + protocols: + - http + - https diff --git a/convert/testdata/5/output-expected.yaml b/convert/testdata/5/output-expected.yaml new file mode 100644 index 000000000..82b2c44bc --- /dev/null +++ b/convert/testdata/5/output-expected.yaml @@ -0,0 +1,65 @@ +_format_version: "3.0" +plugins: +- enabled: true + name: prometheus + protocols: + - http + - https + run_on: first +services: +- connect_timeout: 60000 + host: mockbin.org + name: svc1 + protocol: http + read_timeout: 60000 + routes: + - https_redirect_status_code: 301 + name: r1 + paths: + - /r1 + preserve_host: false + protocols: + - http + - https + regex_priority: 0 + strip_path: false + tags: + - team-svc1 + write_timeout: 60000 +- connect_timeout: 60000 + host: mockbin.org + name: svc2 + protocol: http + read_timeout: 60000 + routes: + - https_redirect_status_code: 301 + name: r2 + paths: + - /r2 + preserve_host: false + protocols: + - http + - https + regex_priority: 0 + strip_path: false + write_timeout: 60000 +- connect_timeout: 60000 + host: mockbin.org + name: svc3 + port: 80 + protocol: http + read_timeout: 60000 + routes: + - https_redirect_status_code: 301 + methods: + - GET + name: r3 + paths: + - /r3 + preserve_host: false + protocols: + - http + - https + regex_priority: 0 + strip_path: false + write_timeout: 60000 diff --git a/convert/testdata/6/input.yaml b/convert/testdata/6/input.yaml new file mode 100644 index 000000000..fed85d987 --- /dev/null +++ b/convert/testdata/6/input.yaml @@ -0,0 +1,41 @@ +_format_version: "3.0" +_info: + defaults: + service: + connect_timeout: 30000 + write_timeout: 30000 + route: + protocols: + - https + https_redirect_status_code: 301 +services: +- name: svc1 + host: mockbin.org + tags: + - team-svc1 + routes: + - name: r1 + paths: + - /r1 +- name: svc2 + host: mockbin.org + routes: + - name: r2 + paths: + - /r2 +- name: svc3 + host: mockbin.org + port: 80 + routes: + - name: r3 + paths: + - /r3 + methods: + - GET +plugins: +- name: prometheus + enabled: true + run_on: first + protocols: + - http + - https diff --git a/convert/testdata/6/output-expected.yaml b/convert/testdata/6/output-expected.yaml new file mode 100644 index 000000000..0a69a1437 --- /dev/null +++ b/convert/testdata/6/output-expected.yaml @@ -0,0 +1,62 @@ +_format_version: "3.0" +plugins: +- enabled: true + name: prometheus + protocols: + - http + - https + run_on: first +services: +- connect_timeout: 30000 + host: mockbin.org + name: svc1 + protocol: http + read_timeout: 60000 + routes: + - https_redirect_status_code: 301 + name: r1 + paths: + - /r1 + preserve_host: false + protocols: + - https + regex_priority: 0 + strip_path: false + tags: + - team-svc1 + write_timeout: 30000 +- connect_timeout: 30000 + host: mockbin.org + name: svc2 + protocol: http + read_timeout: 60000 + routes: + - https_redirect_status_code: 301 + name: r2 + paths: + - /r2 + preserve_host: false + protocols: + - https + regex_priority: 0 + strip_path: false + write_timeout: 30000 +- connect_timeout: 30000 + host: mockbin.org + name: svc3 + port: 80 + protocol: http + read_timeout: 60000 + routes: + - https_redirect_status_code: 301 + methods: + - GET + name: r3 + paths: + - /r3 + preserve_host: false + protocols: + - https + regex_priority: 0 + strip_path: false + write_timeout: 30000 diff --git a/convert/testdata/7/input-1.yaml b/convert/testdata/7/input-1.yaml new file mode 100644 index 000000000..9b867b7ef --- /dev/null +++ b/convert/testdata/7/input-1.yaml @@ -0,0 +1,25 @@ +_format_version: "3.0" +services: +- name: svc1 + host: mockbin.org + tags: + - team-svc1 + routes: + - name: r1 + https_redirect_status_code: 301 + paths: + - /r1 +- name: svc2 + host: mockbin.org + routes: + - name: r2 + https_redirect_status_code: 301 + paths: + - /r2 +plugins: +- name: prometheus + enabled: true + run_on: first + protocols: + - http + - https diff --git a/convert/testdata/7/input-2.yaml b/convert/testdata/7/input-2.yaml new file mode 100644 index 000000000..609ab6d4f --- /dev/null +++ b/convert/testdata/7/input-2.yaml @@ -0,0 +1,12 @@ +_format_version: "3.0" +services: +- name: svc3 + host: mockbin.org + port: 80 + routes: + - name: r3 + https_redirect_status_code: 301 + paths: + - /r3 + methods: + - GET diff --git a/convert/testdata/7/output-expected.yaml b/convert/testdata/7/output-expected.yaml new file mode 100644 index 000000000..82b2c44bc --- /dev/null +++ b/convert/testdata/7/output-expected.yaml @@ -0,0 +1,65 @@ +_format_version: "3.0" +plugins: +- enabled: true + name: prometheus + protocols: + - http + - https + run_on: first +services: +- connect_timeout: 60000 + host: mockbin.org + name: svc1 + protocol: http + read_timeout: 60000 + routes: + - https_redirect_status_code: 301 + name: r1 + paths: + - /r1 + preserve_host: false + protocols: + - http + - https + regex_priority: 0 + strip_path: false + tags: + - team-svc1 + write_timeout: 60000 +- connect_timeout: 60000 + host: mockbin.org + name: svc2 + protocol: http + read_timeout: 60000 + routes: + - https_redirect_status_code: 301 + name: r2 + paths: + - /r2 + preserve_host: false + protocols: + - http + - https + regex_priority: 0 + strip_path: false + write_timeout: 60000 +- connect_timeout: 60000 + host: mockbin.org + name: svc3 + port: 80 + protocol: http + read_timeout: 60000 + routes: + - https_redirect_status_code: 301 + methods: + - GET + name: r3 + paths: + - /r3 + preserve_host: false + protocols: + - http + - https + regex_priority: 0 + strip_path: false + write_timeout: 60000 diff --git a/convert/testdata/8/input.yaml b/convert/testdata/8/input.yaml new file mode 100644 index 000000000..fbe43b53f --- /dev/null +++ b/convert/testdata/8/input.yaml @@ -0,0 +1,9 @@ +services: +- name: svc1 + host: ${{ env "DECK_MOCKBIN_HOST" }} + enabled: ${{ env "DECK_MOCKBIN_ENABLED" | toBool }} + write_timeout: ${{ env "DECK_WRITE_TIMEOUT" | toInt }} +plugins: +- config: + foo: ${{ env "DECK_FOO_FLOAT" | toFloat }} + name: foofloat diff --git a/convert/testdata/8/output-expected.yaml b/convert/testdata/8/output-expected.yaml new file mode 100644 index 000000000..ac8206034 --- /dev/null +++ b/convert/testdata/8/output-expected.yaml @@ -0,0 +1,13 @@ +_format_version: "3.0" +plugins: +- config: + foo: 666 + name: foofloat +services: +- connect_timeout: 60000 + enabled: true + host: mockbin.org + name: svc1 + protocol: http + read_timeout: 60000 + write_timeout: 777 diff --git a/convert/testdata/9/input.yaml b/convert/testdata/9/input.yaml new file mode 100644 index 000000000..fbe43b53f --- /dev/null +++ b/convert/testdata/9/input.yaml @@ -0,0 +1,9 @@ +services: +- name: svc1 + host: ${{ env "DECK_MOCKBIN_HOST" }} + enabled: ${{ env "DECK_MOCKBIN_ENABLED" | toBool }} + write_timeout: ${{ env "DECK_WRITE_TIMEOUT" | toInt }} +plugins: +- config: + foo: ${{ env "DECK_FOO_FLOAT" | toFloat }} + name: foofloat diff --git a/convert/testdata/9/output-expected.yaml b/convert/testdata/9/output-expected.yaml new file mode 100644 index 000000000..8bc7ffd48 --- /dev/null +++ b/convert/testdata/9/output-expected.yaml @@ -0,0 +1,13 @@ +_format_version: "3.0" +plugins: +- config: + foo: 42 + name: foofloat +services: +- connect_timeout: 60000 + enabled: false + host: DECK_MOCKBIN_HOST + name: svc1 + protocol: http + read_timeout: 60000 + write_timeout: 42 diff --git a/file/reader.go b/file/reader.go index 81ba985e7..1e53895ff 100644 --- a/file/reader.go +++ b/file/reader.go @@ -33,12 +33,12 @@ type RenderConfig struct { // // It will return an error if the file representation is invalid // or if there is any error during processing. -func GetContentFromFiles(filenames []string) (*Content, error) { +func GetContentFromFiles(filenames []string, mockEnvVars bool) (*Content, error) { if len(filenames) == 0 { return nil, ErrorFilenameEmpty } - return getContent(filenames) + return getContent(filenames, mockEnvVars) } // GetForKonnect processes the fileContent and renders a RawState and KonnectRawState diff --git a/file/reader_test.go b/file/reader_test.go index ebf5e7ea0..aae486134 100644 --- a/file/reader_test.go +++ b/file/reader_test.go @@ -68,7 +68,7 @@ func TestReadKongStateFromStdinFailsToParseText(t *testing.T) { os.Stdin = tmpfile - c, err := GetContentFromFiles(filenames) + c, err := GetContentFromFiles(filenames, false) assert.NotNil(err) assert.Nil(c) } @@ -97,7 +97,7 @@ func TestTransformNotFalse(t *testing.T) { os.Stdin = tmpfile - c, err := GetContentFromFiles(filenames) + c, err := GetContentFromFiles(filenames, false) if err != nil { panic(err) } @@ -139,7 +139,7 @@ func TestReadKongStateFromStdin(t *testing.T) { os.Stdin = tmpfile - c, err := GetContentFromFiles(filenames) + c, err := GetContentFromFiles(filenames, false) assert.NotNil(c) assert.Nil(err) @@ -155,7 +155,7 @@ func TestReadKongStateFromFile(t *testing.T) { assert := assert.New(t) assert.Equal("testdata/config.yaml", filenames[0]) - c, err := GetContentFromFiles(filenames) + c, err := GetContentFromFiles(filenames, false) assert.NotNil(c) assert.Nil(err) diff --git a/file/readfile.go b/file/readfile.go index 7fb8d3b82..03ca3c80b 100644 --- a/file/readfile.go +++ b/file/readfile.go @@ -19,7 +19,7 @@ import ( // getContent reads all the YAML and JSON files in the directory or the // file, depending on the type of each item in filenames, merges the content of // these files and renders a Content. -func getContent(filenames []string) (*Content, error) { +func getContent(filenames []string, mockEnvVars bool) (*Content, error) { var allReaders []io.Reader var workspaces []string for _, fileOrDir := range filenames { @@ -31,7 +31,7 @@ func getContent(filenames []string) (*Content, error) { } var res Content for _, r := range allReaders { - content, err := readContent(r) + content, err := readContent(r, mockEnvVars) if err != nil { return nil, fmt.Errorf("reading file: %w", err) } @@ -95,13 +95,13 @@ func hasLeadingSpace(fileContent string) bool { // readContent reads all the byes until io.EOF and unmarshals the read // bytes into Content. -func readContent(reader io.Reader) (*Content, error) { +func readContent(reader io.Reader, mockEnvVars bool) (*Content, error) { var err error contentBytes, err := ioutil.ReadAll(reader) if err != nil { return nil, err } - renderedContent, err := renderTemplate(string(contentBytes)) + renderedContent, err := renderTemplate(string(contentBytes), mockEnvVars) if err != nil { return nil, fmt.Errorf("parsing file: %w", err) } @@ -146,25 +146,65 @@ func getPrefixedEnvVar(key string) (string, error) { return value, nil } +// getPrefixedEnvVarMocked is used when we mock the env variables while rendering a template. +// It will always return the name of the environment variable in this case. +func getPrefixedEnvVarMocked(key string) (string, error) { + const envVarPrefix = "DECK_" + if !strings.HasPrefix(key, envVarPrefix) { + return "", fmt.Errorf("environment variables in the state file must "+ + "be prefixed with 'DECK_', found: '%s'", key) + } + return key, nil +} + func toBool(key string) (bool, error) { return strconv.ParseBool(key) } +// toBoolMocked is used when we mock the env variables while rendering a template. +// It will always return false in this case. +func toBoolMocked(_ string) (bool, error) { + return false, nil +} + func toInt(key string) (int, error) { return strconv.Atoi(key) } +// toIntMocked is used when we mock the env variables while rendering a template. +// It will always return 42 in this case. +func toIntMocked(_ string) (int, error) { + return 42, nil +} + func toFloat(key string) (float64, error) { return strconv.ParseFloat(key, 64) } -func renderTemplate(content string) (string, error) { - t := template.New("state").Funcs(template.FuncMap{ - "env": getPrefixedEnvVar, - "toBool": toBool, - "toInt": toInt, - "toFloat": toFloat, - }).Delims("${{", "}}") +// toFloatMocked is used when we mock the env variables while rendering a template. +// It will always return 42 in this case. +func toFloatMocked(_ string) (float64, error) { + return 42, nil +} + +func renderTemplate(content string, mockEnvVars bool) (string, error) { + var templateFuncs template.FuncMap + if mockEnvVars { + templateFuncs = template.FuncMap{ + "env": getPrefixedEnvVarMocked, + "toBool": toBoolMocked, + "toInt": toIntMocked, + "toFloat": toFloatMocked, + } + } else { + templateFuncs = template.FuncMap{ + "env": getPrefixedEnvVar, + "toBool": toBool, + "toInt": toInt, + "toFloat": toFloat, + } + } + t := template.New("state").Funcs(templateFuncs).Delims("${{", "}}") t, err := t.Parse(content) if err != nil { return "", err diff --git a/file/readfile_test.go b/file/readfile_test.go index 385132763..2ee703764 100644 --- a/file/readfile_test.go +++ b/file/readfile_test.go @@ -512,7 +512,7 @@ func Test_getContent(t *testing.T) { for k, v := range tt.envVars { t.Setenv(k, v) } - got, err := getContent(tt.args.filenames) + got, err := getContent(tt.args.filenames, false) if (err != nil) != tt.wantErr { t.Errorf("getContent() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/file/writer.go b/file/writer.go index fba59da24..55c281840 100644 --- a/file/writer.go +++ b/file/writer.go @@ -42,16 +42,16 @@ func getFormatVersion(kongVersion string) (string, error) { return formatVersion, nil } -// KongStateToFile writes a state object to file with filename. +// KongStateToFile generates a state object to file.Content. // It will omit timestamps and IDs while writing. -func KongStateToFile(kongState *state.KongState, config WriteConfig) error { +func KongStateToContent(kongState *state.KongState, config WriteConfig) (*Content, error) { file := &Content{} var err error file.Workspace = config.Workspace formatVersion, err := getFormatVersion(config.KongVersion) if err != nil { - return fmt.Errorf("get format version: %w", err) + return nil, fmt.Errorf("get format version: %w", err) } file.FormatVersion = formatVersion if config.RuntimeGroupName != "" { @@ -69,49 +69,58 @@ func KongStateToFile(kongState *state.KongState, config WriteConfig) error { err = populateServices(kongState, file, config) if err != nil { - return err + return nil, err } err = populateServicelessRoutes(kongState, file, config) if err != nil { - return err + return nil, err } err = populatePlugins(kongState, file, config) if err != nil { - return err + return nil, err } err = populateUpstreams(kongState, file, config) if err != nil { - return err + return nil, err } err = populateCertificates(kongState, file, config) if err != nil { - return err + return nil, err } err = populateCACertificates(kongState, file, config) if err != nil { - return err + return nil, err } err = populateConsumers(kongState, file, config) if err != nil { - return err + return nil, err } err = populateVaults(kongState, file, config) if err != nil { - return err + return nil, err } err = populateConsumerGroups(kongState, file, config) if err != nil { - return err + return nil, err } + return file, nil +} +// KongStateToFile writes a state object to file with filename. +// See KongStateToContent for the State generation +func KongStateToFile(kongState *state.KongState, config WriteConfig) error { + file, err := KongStateToContent(kongState, config) + if err != nil { + return err + } return WriteContentToFile(file, config.Filename, config.FileFormat) }