From 6b2af78d48ed632e9f64b8e265741f6329f998ae Mon Sep 17 00:00:00 2001 From: Martin Perez Date: Thu, 21 Sep 2023 10:30:41 +0200 Subject: [PATCH 1/3] Adds experimental support for generating Carvel bundles --- README.md | 17 +- chartutils/carvel.go | 87 ++++++++ cmd/dt/carvelize.go | 191 ++++++++++++++++++ cmd/dt/carvelize_test.go | 91 +++++++++ cmd/dt/chart.go | 2 +- cmd/dt/chart_test.go | 1 + cmd/dt/relocate.go | 1 + cmd/dt/wrap.go | 23 +++ cmd/dt/wrap_test.go | 32 ++- go.mod | 10 +- go.sum | 15 ++ relocator/chart.go | 60 ++++++ relocator/imagelock.go | 2 +- .../custom-chart/.imgpkg/bundle.yml.tmpl | 12 ++ .../custom-chart/.imgpkg/images.yml.tmpl | 15 ++ .../scenarios/custom-chart/Chart.yaml.tmpl | 17 ++ 16 files changed, 561 insertions(+), 15 deletions(-) create mode 100644 chartutils/carvel.go create mode 100644 cmd/dt/carvelize.go create mode 100644 cmd/dt/carvelize_test.go create mode 100644 testdata/scenarios/custom-chart/.imgpkg/bundle.yml.tmpl create mode 100644 testdata/scenarios/custom-chart/.imgpkg/images.yml.tmpl diff --git a/README.md b/README.md index a409a49..35d3056 100644 --- a/README.md +++ b/README.md @@ -405,7 +405,7 @@ images: ... ``` -### Annotating a chart (EXPERIMENTAL) +### Annotating a Helm chart (EXPERIMENTAL) `Images.lock` creation relies on the existence of the special images annotation inside `Chart.yaml`. If you have a Helm chart that does not contain any annotations, this command can be used to guess and generate an annotation with a tentative list of images. It's important to note that this list is a **best-effort** as the list of images is obtained from the `values.yaml` file and this is always an unreliable, often incomplete, and error-prone source as the configuration in `values.yaml` is very variable. @@ -414,6 +414,21 @@ $ helm dt chart annotate examples/mariadb INFO[0000] Helm chart annotated successfully ``` +### Converting a Helm chart into a Carvel bundle (EXPERIMENTAL) + +From `dt` 0.1.1 we have introduced a new command to create a [Carvel bundle](https://carvel.dev/imgpkg/docs/v0.37.x/resources/#bundle) from any Helm chart. + + +```console +$ helm dt chart carvelize examples/postgresql + ✔ Helm chart "examples/postgresql" lock is valid + » Generating Carvel bundle for Helm chart "examples/postgresql" + ✔ Validating Carvel images lock + ✔ Carvel images lock written to "examples/postgresql/.imgpkg/images.yml" + ✔ Carvel metadata written to "examples/postgresql/.imgpkg/bundle.yml" + 🎉 Carvel bundle created successfully +``` + ## Frequently Asked Questions ### I cannot install the plugin due to `Error: Unable to update repository: exit status 1` diff --git a/chartutils/carvel.go b/chartutils/carvel.go new file mode 100644 index 0000000..9d08d81 --- /dev/null +++ b/chartutils/carvel.go @@ -0,0 +1,87 @@ +// Package chartutils implements helper functions to manipulate helm Charts +package chartutils + +import ( + "fmt" + "io" + "os" + + "gopkg.in/yaml.v3" +) + +// Somehow there is no data structure for a bundle in Carvel. Copying some basics from the describe command. +// Author information from a Bundle +type Author struct { + Name string `json:"name,omitempty"` + Email string `json:"email,omitempty"` +} + +// Website URL where more information of the Bundle can be found +type Website struct { + URL string `json:"url,omitempty"` +} + +// Bundle Metadata +const ( + BundleAPIVersion = "imgpkg.carvel.dev/v1alpha1" + BundleKind = "Bundle" +) + +type BundleVersion struct { + APIVersion string `json:"apiVersion"` // This generated yaml, but due to lib we need to use `json` + Kind string `json:"kind"` // This generated yaml, but due to lib we need to use `json` +} +type Metadata struct { + Version BundleVersion + Metadata map[string]string `json:"metadata,omitempty"` + Authors []Author `json:"authors,omitempty"` + Websites []Website `json:"websites,omitempty"` +} + +func (il *Metadata) ToYAML(w io.Writer) error { + enc := yaml.NewEncoder(w) + enc.SetIndent(2) + + return enc.Encode(il) +} + +func CarvelBundleFromYAMLFile(file string) (*Metadata, error) { + fh, err := os.Open(file) + if err != nil { + return nil, fmt.Errorf("failed to open Images.lock file: %v", err) + } + defer fh.Close() + return CarvelBundleFromYAML(fh) +} + +// reads a Carvel metadata bundled from the YAML read from r +func CarvelBundleFromYAML(r io.Reader) (*Metadata, error) { + metadata := &Metadata{ + Version: BundleVersion{ + APIVersion: BundleAPIVersion, + Kind: BundleKind, + }, + Metadata: map[string]string{}, + Authors: []Author{}, + Websites: []Website{}, + } + dec := yaml.NewDecoder(r) + if err := dec.Decode(metadata); err != nil { + return nil, fmt.Errorf("failed to load Carvel bundle: %v", err) + } + + return metadata, nil +} + +func NewCarvelBundle() *Metadata { + + return &Metadata{ + Version: BundleVersion{ + APIVersion: BundleAPIVersion, + Kind: BundleKind, + }, + Metadata: map[string]string{}, + Authors: []Author{}, + Websites: []Website{}, + } +} diff --git a/cmd/dt/carvelize.go b/cmd/dt/carvelize.go new file mode 100644 index 0000000..d154524 --- /dev/null +++ b/cmd/dt/carvelize.go @@ -0,0 +1,191 @@ +package main + +import ( + "bytes" + "fmt" + "os" + "strings" + + "github.com/spf13/cobra" + "github.com/vmware-labs/distribution-tooling-for-helm/chartutils" + "github.com/vmware-labs/distribution-tooling-for-helm/internal/log" + "github.com/vmware-labs/distribution-tooling-for-helm/utils" + "github.com/vmware-tanzu/carvel-imgpkg/pkg/imgpkg/lockconfig" +) + +var carvelizeCmd = newCarvelizeCmd() + +func newCarvelizeCmd() *cobra.Command { + var yamlFormat bool + var showDetails bool + + cmd := &cobra.Command{ + Use: "carvelize FILE", + Short: "Adds a Carvel bundle to the Helm chart (Experimental)", + Long: `Experimental. Adds a Carvel bundle to an existing Helm chart`, + Example: ` # Adds a Carvel bundle to a Helm chart + $ dt charts carvelize examples/mariadb`, + SilenceUsage: true, + SilenceErrors: true, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + chartPath := args[0] + l := getLogger() + // Allows silencing called methods + silentLog := log.SilentLog + + lockFile, err := getImageLockFilePath(chartPath) + if err != nil { + return fmt.Errorf("failed to determine Images.lock file location: %w", err) + } + + if utils.FileExists(lockFile) { + if err := l.ExecuteStep("Verifying Images.lock", func() error { + return verifyLock(chartPath, lockFile) + }); err != nil { + return l.Failf("Failed to verify lock: %w", err) + } + l.Infof("Helm chart %q lock is valid", chartPath) + + } else { + err := l.ExecuteStep( + "Images.lock file does not exist. Generating it from annotations...", + func() error { + return createImagesLock(chartPath, + lockFile, silentLog, + ) + }, + ) + if err != nil { + return l.Failf("Failed to generate lock: %w", err) + } + l.Infof("Images.lock file written to %q", lockFile) + } + if err := l.Section(fmt.Sprintf("Generating Carvel bundle for Helm chart %q", chartPath), func(childLog log.SectionLogger) error { + if err := generateCarvelBundle( + chartPath, + chartutils.WithLog(childLog), + ); err != nil { + return childLog.Failf("%v", err) + } + return nil + }); err != nil { + return l.Failf("%w", err) + } + l.Successf("Carvel bundle created successfully") + return nil + }, + } + cmd.PersistentFlags().BoolVar(&yamlFormat, "yaml", yamlFormat, "Show report in YAML format") + cmd.PersistentFlags().BoolVar(&showDetails, "detailed", showDetails, "When using the printable report, add more details about the bundled images") + + return cmd +} + +func generateCarvelBundle(chartPath string, opts ...chartutils.Option) error { + + cfg := chartutils.NewConfiguration(opts...) + l := cfg.Log + + lock, err := readLockFromWrap(chartPath) + if err != nil { + return fmt.Errorf("failed to load Images.lock: %v", err) + } + + imgPkgPath := chartPath + "/.imgpkg" + if !utils.FileExists(imgPkgPath) { + err := os.Mkdir(imgPkgPath, os.FileMode(0755)) + if err != nil { + return fmt.Errorf("failed to create .imgpkg directory: %w", err) + } + } + + _, _ = chartPath, l + if !utils.FileExists(chartPath) { + return fmt.Errorf("wrap file %q does not exist", chartPath) + } + + bundleMetadata := chartutils.NewCarvelBundle() + + chart, err := chartutils.LoadChart(chartPath) + if err != nil { + return fmt.Errorf("failed to load chart: %w", err) + } + + for _, maintainer := range chart.Metadata.Maintainers { + author := chartutils.Author{ + Name: maintainer.Name, + } + if maintainer.Email != "" { + author.Email = maintainer.Email + } + bundleMetadata.Authors = append(bundleMetadata.Authors, author) + } + for _, source := range chart.Metadata.Sources { + website := chartutils.Website{ + URL: source, + } + bundleMetadata.Websites = append(bundleMetadata.Websites, website) + } + + bundleMetadata.Metadata["name"] = lock.Chart.Name + for key, value := range chart.Metadata.Annotations { + if key != "images" { + bundleMetadata.Metadata[key] = value + } + } + + imagesLock := lockconfig.ImagesLock{ + LockVersion: lockconfig.LockVersion{ + APIVersion: lockconfig.ImagesLockAPIVersion, + Kind: lockconfig.ImagesLockKind, + }, + } + for _, img := range lock.Images { + // Carvel does not seem to support multi-arch. Grab amd64 digest + name := img.Image + i := strings.LastIndex(img.Image, ":") + if i > -1 { + name = img.Image[0:i] + + } + for _, digest := range img.Digests { + if digest.Arch == "linux/amd64" { + name = name + "@" + digest.Digest.String() + break + } + } + imageRef := lockconfig.ImageRef{ + Image: name, + Annotations: map[string]string{ + "kbld.carvel.dev/id": img.Image, + }, + } + imagesLock.AddImageRef(imageRef) + } + l.Infof("Validating Carvel images lock") + imagesLock.Validate() + + path := imgPkgPath + "/images.yml" + err = imagesLock.WriteToPath(path) + if err != nil { + return fmt.Errorf("Could not write image lock: %v", err) + } + l.Infof("Carvel images lock written to %q", path) + + buff := &bytes.Buffer{} + if err = bundleMetadata.ToYAML(buff); err != nil { + return fmt.Errorf("failed to write bundle metadata file: %v", err) + } + + path = imgPkgPath + "/bundle.yml" + if err := os.WriteFile(path, buff.Bytes(), 0666); err != nil { + return fmt.Errorf("failed to write Carvel bundle metadata to %q: %w", path, err) + } + l.Infof("Carvel metadata written to %q", path) + return nil +} + +func init() { + rootCmd.AddCommand(infoCmd) +} diff --git a/cmd/dt/carvelize_test.go b/cmd/dt/carvelize_test.go new file mode 100644 index 0000000..56e2821 --- /dev/null +++ b/cmd/dt/carvelize_test.go @@ -0,0 +1,91 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + tu "github.com/vmware-labs/distribution-tooling-for-helm/internal/testutil" + "gopkg.in/yaml.v3" +) + +func (suite *CmdSuite) TestCarvelizeCommand() { + require := suite.Require() + + s, err := tu.NewTestServer() + require.NoError(err) + + defer s.Close() + + images, err := s.LoadImagesFromFile("../../testdata/images.json") + require.NoError(err) + + sb := suite.sb + serverURL := s.ServerURL + scenarioName := "custom-chart" + chartName := "test" + + scenarioDir := fmt.Sprintf("../../testdata/scenarios/%s", scenarioName) + t := suite.T() + + type author struct { + Name string + Email string + } + type website struct { + Url string + } + + dest := sb.TempFile() + require.NoError(tu.RenderScenario(scenarioDir, dest, + map[string]interface{}{"ServerURL": serverURL, "Images": images, "Name": chartName, "RepositoryURL": serverURL, + "Authors": []author{{ + Name: "VMware, Inc.", + Email: "dt@vmware.com", + }}, + "Websites": []website{{ + Url: "https://github.com/bitnami/charts/tree/main/bitnami/wordpress", + }}, + }, + )) + chartDir := filepath.Join(dest, scenarioName) + + bundleData, err := tu.RenderTemplateFile(filepath.Join(scenarioDir, ".imgpkg/bundle.yml.tmpl"), + map[string]interface{}{"ServerURL": serverURL, "Images": images, "Name": chartName}, + ) + + require.NoError(err) + var expectedBundle map[string]interface{} + require.NoError(yaml.Unmarshal([]byte(bundleData), &expectedBundle)) + + carvelImagesData, err := tu.RenderTemplateFile(filepath.Join(scenarioDir, ".imgpkg/images.yml.tmpl"), + map[string]interface{}{"ServerURL": serverURL, "Images": images, "Name": chartName}, + ) + require.NoError(err) + var expectedCarvelImagesLock map[string]interface{} + require.NoError(yaml.Unmarshal([]byte(carvelImagesData), &expectedCarvelImagesLock)) + + // We need to provide the --insecure flag or our test server won't validate + args := []string{"charts", "carvelize", "--insecure", chartDir} + res := dt(args...) + res.AssertSuccess(t) + + t.Run("Generates Carvel bundle", func(t *testing.T) { + newBundleData, err := os.ReadFile(filepath.Join(chartDir, ".imgpkg/bundle.yml")) + require.NoError(err) + var newBundle map[string]interface{} + require.NoError(yaml.Unmarshal(newBundleData, &newBundle)) + + require.Equal(expectedBundle, newBundle) + }) + + t.Run("Generates Carvel images", func(t *testing.T) { + newImagesData, err := os.ReadFile(filepath.Join(chartDir, ".imgpkg/images.yml")) + require.NoError(err) + var newImagesLock map[string]interface{} + require.NoError(yaml.Unmarshal(newImagesData, &newImagesLock)) + + require.Equal(expectedCarvelImagesLock, newImagesLock) + }) +} diff --git a/cmd/dt/chart.go b/cmd/dt/chart.go index 9a62d2a..c944078 100644 --- a/cmd/dt/chart.go +++ b/cmd/dt/chart.go @@ -15,5 +15,5 @@ var chartCmd = &cobra.Command{ } func init() { - chartCmd.AddCommand(relocateCmd, annotateCmd) + chartCmd.AddCommand(relocateCmd, annotateCmd, carvelizeCmd) } diff --git a/cmd/dt/chart_test.go b/cmd/dt/chart_test.go index 821ceaa..61b0a13 100644 --- a/cmd/dt/chart_test.go +++ b/cmd/dt/chart_test.go @@ -12,6 +12,7 @@ func (suite *CmdSuite) TestChartsHelp() { res.AssertSuccess(t) for _, reStr := range []string{ `annotate\s+Annotates a Helm chart`, + `carvelize\s+Adds a Carvel bundle to the Helm chart`, `relocate\s+Relocates a Helm chart`, } { res.AssertSuccessMatch(t, fmt.Sprintf(`(?s).*Available Commands:.*\n\s*%s.*`, reStr)) diff --git a/cmd/dt/relocate.go b/cmd/dt/relocate.go index 580675a..19d2e9f 100644 --- a/cmd/dt/relocate.go +++ b/cmd/dt/relocate.go @@ -39,6 +39,7 @@ func newRelocateCmd() *cobra.Command { return fmt.Errorf("repository cannot be empty") } l := getLogger() + if err := l.ExecuteStep(fmt.Sprintf("Relocating %q with prefix %q", chartPath, repository), func() error { return relocateChart(chartPath, repository, relocator.WithLog(l)) }); err != nil { diff --git a/cmd/dt/wrap.go b/cmd/dt/wrap.go index c2d9a54..737b624 100644 --- a/cmd/dt/wrap.go +++ b/cmd/dt/wrap.go @@ -62,6 +62,7 @@ func wrapChart(ctx context.Context, inputPath string, outputFile string, platfor } l.Infof("Images.lock file written to %q", lockFile) } + if outputFile == "" { outputBaseName := fmt.Sprintf("%s-%s.wrap.tgz", chart.Name(), chart.Metadata.Version) if outputFile, err = filepath.Abs(outputBaseName); err != nil { @@ -84,6 +85,26 @@ func wrapChart(ctx context.Context, inputPath string, outputFile string, platfor return err } + carvelize, err := flags.GetBool("add-carvel-bundle") + if err != nil { + return fmt.Errorf("failed to retrieve add-carvel-bundle flag: %w", err) + } + + if carvelize { + if err := l.Section(fmt.Sprintf("Generating Carvel bundle for Helm chart %q", chartPath), func(childLog log.SectionLogger) error { + if err := generateCarvelBundle( + chartPath, + chartutils.WithLog(childLog), + ); err != nil { + return childLog.Failf("%v", err) + } + return nil + }); err != nil { + return l.Failf("%w", err) + } + l.Infof("Carvel bundle created successfully") + } + if err := l.ExecuteStep( "Compressing Helm chart...", func() error { @@ -104,6 +125,7 @@ func newWrapCommand() *cobra.Command { var outputFile string var version string var platforms []string + var carvelize bool var examples = ` # Wrap a Helm chart from a local folder $ dt wrap examples/mariadb @@ -140,6 +162,7 @@ This command will pull all the container images and wrap it into a single tarbal cmd.PersistentFlags().StringVar(&version, "version", version, "when wrapping remote Helm charts from OCI, version to request") cmd.PersistentFlags().StringVar(&outputFile, "output-file", outputFile, "generate a tar.gz with the output of the pull operation") cmd.PersistentFlags().StringSliceVar(&platforms, "platforms", platforms, "platforms to include in the Images.lock file") + cmd.PersistentFlags().BoolVar(&carvelize, "add-carvel-bundle", carvelize, "whether the wrap should include a Carvel bundle or not") return cmd } diff --git a/cmd/dt/wrap_test.go b/cmd/dt/wrap_test.go index 9f379e8..b1d4dca 100644 --- a/cmd/dt/wrap_test.go +++ b/cmd/dt/wrap_test.go @@ -59,7 +59,8 @@ func (suite *CmdSuite) TestWrapCommand() { } return chartDir } - testWrap := func(t *testing.T, inputChart string, outputFile string, expectedLock map[string]interface{}) { + testWrap := func(t *testing.T, inputChart string, outputFile string, expectedLock map[string]interface{}, + generateCarvelBundle bool) { // Setup a working directory to look for the wrap when not providing a output-filename currentDir, err := os.Getwd() require.NoError(err) @@ -78,6 +79,9 @@ func (suite *CmdSuite) TestWrapCommand() { } else { expectedWrapFile = filepath.Join(workingDir, fmt.Sprintf("%s-%v.wrap.tgz", chartName, version)) } + if generateCarvelBundle { + args = append(args, "--add-carvel-bundle") + } res := dt(args...) res.AssertSuccess(t) @@ -97,6 +101,13 @@ func (suite *CmdSuite) TestWrapCommand() { lockFile := filepath.Join(tmpDir, "Images.lock") assert.FileExists(lockFile) + if generateCarvelBundle { + carvelBundleFile := filepath.Join(tmpDir, ".imgpkg/bundle.yml") + assert.FileExists(carvelBundleFile) + carvelImagesLockFile := filepath.Join(tmpDir, ".imgpkg/images.yml") + assert.FileExists(carvelImagesLockFile) + } + newData, err := os.ReadFile(lockFile) require.NoError(err) var newLock map[string]interface{} @@ -107,7 +118,7 @@ func (suite *CmdSuite) TestWrapCommand() { assert.Equal(expectedLock, newLock) } - testSampleWrap := func(t *testing.T, withLock bool, outputFile string) { + testSampleWrap := func(t *testing.T, withLock bool, outputFile string, generateCarvelBundle bool) { dest := sb.TempFile() chartDir := createSampleChart(dest, withLock) @@ -121,14 +132,14 @@ func (suite *CmdSuite) TestWrapCommand() { // Clear the timestamp expectedLock["metadata"] = nil - testWrap(t, chartDir, outputFile, expectedLock) + testWrap(t, chartDir, outputFile, expectedLock, generateCarvelBundle) } t.Run("Wrap Chart without exiting lock", func(t *testing.T) { - testSampleWrap(t, withoutLock, "") + testSampleWrap(t, withoutLock, "", false) }) t.Run("Wrap Chart with exiting lock", func(t *testing.T) { - testSampleWrap(t, withLock, "") + testSampleWrap(t, withLock, "", false) }) t.Run("Wrap Chart From compressed tgz", func(t *testing.T) { dest := sb.TempFile() @@ -149,7 +160,7 @@ func (suite *CmdSuite) TestWrapCommand() { require.NoError(utils.Tar(chartDir, tarFilename, utils.TarConfig{})) require.FileExists(tarFilename) - testWrap(t, tarFilename, "", expectedLock) + testWrap(t, tarFilename, "", expectedLock, false) }) t.Run("Wrap Chart From oci", func(t *testing.T) { @@ -185,13 +196,18 @@ func (suite *CmdSuite) TestWrapCommand() { require.NoError(utils.PushChart(tarFilename, pushChartURL)) - testWrap(t, fullChartURL, "", expectedLock) + testWrap(t, fullChartURL, "", expectedLock, false) }) t.Run("Wrap Chart with custom output filename", func(t *testing.T) { tempFilename := fmt.Sprintf("%s/chart.wrap.tar.gz", sb.TempFile()) - testSampleWrap(t, withLock, tempFilename) + testSampleWrap(t, withLock, tempFilename, false) // This should already be handled by testWrap, but make sure it is there suite.Assert().FileExists(tempFilename) }) + + t.Run("Wrap Chart and generate carvel bundle", func(t *testing.T) { + tempFilename := fmt.Sprintf("%s/chart.wrap.tar.gz", sb.TempFile()) + testSampleWrap(t, withLock, tempFilename, true) // triggers the Carvel checks + }) } diff --git a/go.mod b/go.mod index 4f77ca3..697cec1 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect - github.com/fatih/color v1.13.0 // indirect + github.com/fatih/color v1.14.1 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-gorp/gorp/v3 v3.0.5 // indirect github.com/go-logr/logr v1.2.3 // indirect @@ -84,7 +84,7 @@ require ( github.com/lithammer/fuzzysearch v1.1.8 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect @@ -114,6 +114,8 @@ require ( github.com/spf13/cast v1.5.0 // indirect github.com/spf13/pflag v1.0.5 github.com/vbatts/tar-split v0.11.3 // indirect + github.com/vmware-tanzu/carvel-imgpkg v0.37.3 + github.com/vmware-tanzu/carvel-kapp-controller v0.47.0 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect @@ -126,11 +128,11 @@ require ( golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.7.0 // indirect - golang.org/x/sync v0.1.0 // indirect + golang.org/x/sync v0.2.0 // indirect golang.org/x/sys v0.10.0 // indirect golang.org/x/term v0.10.0 // indirect golang.org/x/text v0.11.0 // indirect - golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect + golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect google.golang.org/grpc v1.53.0 // indirect diff --git a/go.sum b/go.sum index daf22ee..9a99543 100644 --- a/go.sum +++ b/go.sum @@ -183,6 +183,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= +github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= @@ -455,6 +457,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-oci8 v0.1.1/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= @@ -662,6 +666,12 @@ github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RV github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= github.com/vmware-labs/yaml-jsonpath v0.3.2 h1:/5QKeCBGdsInyDCyVNLbXyilb61MXGi9NP674f9Hobk= github.com/vmware-labs/yaml-jsonpath v0.3.2/go.mod h1:U6whw1z03QyqgWdgXxvVnQ90zN1BWz5V+51Ewf8k+rQ= +github.com/vmware-tanzu/carvel-imgpkg v0.37.3 h1:e83Ugr6Lmuw1Q8nD9U0gdPXN1hP5iLW9JMxJs5gV2SQ= +github.com/vmware-tanzu/carvel-imgpkg v0.37.3/go.mod h1:YFKwj9DebWdI/yOrHNDGtS7WEUYmYUrXjZl/zAlzX08= +github.com/vmware-tanzu/carvel-kapp-controller v0.47.0 h1:5HouKdHw3dfaDWsN0cQQGahnEtKjErgXQhCz4JuJPZg= +github.com/vmware-tanzu/carvel-kapp-controller v0.47.0/go.mod h1:NEEM+AklzRenvpeFMMM9EweGW1GqrtUZbnSZYIb0HE8= +github.com/vmware-tanzu/carvel-vendir v0.33.1 h1:5wzx0aRyEiorkWwrpGvACJOpFcgvmCeVqhIY9TPuLvk= +github.com/vmware-tanzu/carvel-vendir v0.33.1/go.mod h1:cZEa46rwzPt/ROdAuIgrCXLDJ+LqxNZaNxbz8MLtwWc= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -847,6 +857,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -921,6 +933,7 @@ golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -952,6 +965,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= +golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/relocator/chart.go b/relocator/chart.go index b836601..84edda4 100644 --- a/relocator/chart.go +++ b/relocator/chart.go @@ -11,6 +11,7 @@ import ( cu "github.com/vmware-labs/distribution-tooling-for-helm/chartutils" "github.com/vmware-labs/distribution-tooling-for-helm/imagelock" "github.com/vmware-labs/distribution-tooling-for-helm/utils" + "github.com/vmware-tanzu/carvel-imgpkg/pkg/imgpkg/lockconfig" ) // RelocationResult describes the result of performing a relocation @@ -78,6 +79,10 @@ func RelocateChartDir(chartPath string, prefix string, opts ...RelocateOption) e if err != nil { return err } + err = relocateCarvelBundle(chartPath, prefix) + if err != nil { + return err + } var allErrors error @@ -91,6 +96,61 @@ func RelocateChartDir(chartPath string, prefix string, opts ...RelocateOption) e return allErrors } +func relocateCarvelBundle(chartRoot string, prefix string) error { + + //TODO: Do better detection here, imgpkg probably has something + lockFile := chartRoot + "/.imgpkg/images.yml" + if !utils.FileExists(lockFile) { + fmt.Printf("Did not find Carvel images bundle at %s. Ignoring", lockFile) + return nil + } + + lock, err := lockconfig.NewImagesLockFromPath(lockFile) + if err != nil { + return fmt.Errorf("failed to load Carvel images lock: %v", err) + } + result, err := RelocateCarvelImagesLock(&lock, prefix) + if err != nil { + return err + } + if result.Count == 0 { + return nil + } + if err := utils.SafeWriteFile(lockFile, result.Data, 0600); err != nil { + return fmt.Errorf("failed to overwrite Carvel images lock file: %v", err) + } + return nil +} + +// RelocateCarvelImagesLock rewrites the images urls in the provided lock using prefix +func RelocateCarvelImagesLock(lock *lockconfig.ImagesLock, prefix string) (*RelocationResult, error) { + + count, err := relocateCarvelImages(lock.Images, prefix) + if err != nil { + return nil, fmt.Errorf("failed to relocate Carvel images lock file: %v", err) + } + + if buff, err := lock.AsBytes(); err != nil { + return nil, fmt.Errorf("failed to write Images.lock file: %v", err) + } else { + return &RelocationResult{Data: buff, Count: count}, nil + } +} + +func relocateCarvelImages(images []lockconfig.ImageRef, prefix string) (count int, err error) { + var allErrors error + for i, img := range images { + norm, err := utils.RelocateImageURL(img.Image, prefix, true) + if err != nil { + allErrors = errors.Join(allErrors, err) + continue + } + images[i].Image = norm + count++ + } + return count, allErrors +} + func normalizeRelocateURL(url string) string { ociPrefix := "oci://" // crane gets confused with the oci schema, so we diff --git a/relocator/imagelock.go b/relocator/imagelock.go index b6b53c9..3be83bc 100644 --- a/relocator/imagelock.go +++ b/relocator/imagelock.go @@ -36,7 +36,7 @@ func RelocateLock(lock *imagelock.ImagesLock, prefix string) (*RelocationResult, return &RelocationResult{Data: buff.Bytes(), Count: count}, nil } -// RelocateLockFile reloactes images urls in the provided Images.lock using prefix +// RelocateLockFile relocates images urls in the provided Images.lock using prefix func RelocateLockFile(file string, prefix string) error { lock, err := imagelock.FromYAMLFile(file) if err != nil { diff --git a/testdata/scenarios/custom-chart/.imgpkg/bundle.yml.tmpl b/testdata/scenarios/custom-chart/.imgpkg/bundle.yml.tmpl new file mode 100644 index 0000000..3211444 --- /dev/null +++ b/testdata/scenarios/custom-chart/.imgpkg/bundle.yml.tmpl @@ -0,0 +1,12 @@ +version: + apiversion: imgpkg.carvel.dev/v1alpha1 + kind: Bundle +metadata: + category: CMS + licenses: Apache-2.0 + name: {{or .Name "WordPress"}} +authors: + - name: VMware, Inc. + email: "dt@vmware.com" +websites: + - url: https://github.com/bitnami/charts/tree/main/bitnami/wordpress \ No newline at end of file diff --git a/testdata/scenarios/custom-chart/.imgpkg/images.yml.tmpl b/testdata/scenarios/custom-chart/.imgpkg/images.yml.tmpl new file mode 100644 index 0000000..315f6cb --- /dev/null +++ b/testdata/scenarios/custom-chart/.imgpkg/images.yml.tmpl @@ -0,0 +1,15 @@ +apiVersion: imgpkg.carvel.dev/v1alpha1 +images: +{{- $p := . -}} +{{- range $idx, $elem := .Images}} +{{ $imageParts := split ":" $elem.Image }} +{{ $img := $imageParts._0 }} +{{- range .Digests}} +{{- if eq .Arch "linux/amd64"}} +- annotations: + kbld.carvel.dev/id: {{$p.ServerURL}}/{{$elem.Image}} + image: {{$p.ServerURL}}/{{$img}}@{{.Digest}} +{{- end }} +{{- end}} +{{- end}} +kind: ImagesLock \ No newline at end of file diff --git a/testdata/scenarios/custom-chart/Chart.yaml.tmpl b/testdata/scenarios/custom-chart/Chart.yaml.tmpl index 3aee831..6ecb959 100644 --- a/testdata/scenarios/custom-chart/Chart.yaml.tmpl +++ b/testdata/scenarios/custom-chart/Chart.yaml.tmpl @@ -16,3 +16,20 @@ dependencies: {{end -}} {{end}} {{end}} +{{if .Authors }} +{{if gt (len .Authors) 0 }} +maintainers: +{{- range .Authors}} + - name: {{.Name}} + email: {{.Email}} +{{end -}} +{{end}} +{{end}} +{{if .Websites }} +{{if gt (len .Websites) 0 }} +sources: +{{- range .Websites}} + - {{.Url}} +{{end -}} +{{end}} +{{end}} From 47d064bfda02bb74e0569b92f160225f85f8034e Mon Sep 17 00:00:00 2001 From: Martin Perez Date: Thu, 21 Sep 2023 14:29:20 +0200 Subject: [PATCH 2/3] Code refactoring and cleaning up --- chartutils/carvel.go | 87 --------- chartutils/carvel/carvel.go | 168 ++++++++++++++++++ cmd/dt/carvelize.go | 66 ++----- cmd/dt/carvelize_test.go | 28 ++- relocator/chart.go | 8 +- .../custom-chart/.imgpkg/bundle.yml.tmpl | 10 +- .../scenarios/custom-chart/Chart.yaml.tmpl | 2 +- 7 files changed, 203 insertions(+), 166 deletions(-) delete mode 100644 chartutils/carvel.go create mode 100644 chartutils/carvel/carvel.go diff --git a/chartutils/carvel.go b/chartutils/carvel.go deleted file mode 100644 index 9d08d81..0000000 --- a/chartutils/carvel.go +++ /dev/null @@ -1,87 +0,0 @@ -// Package chartutils implements helper functions to manipulate helm Charts -package chartutils - -import ( - "fmt" - "io" - "os" - - "gopkg.in/yaml.v3" -) - -// Somehow there is no data structure for a bundle in Carvel. Copying some basics from the describe command. -// Author information from a Bundle -type Author struct { - Name string `json:"name,omitempty"` - Email string `json:"email,omitempty"` -} - -// Website URL where more information of the Bundle can be found -type Website struct { - URL string `json:"url,omitempty"` -} - -// Bundle Metadata -const ( - BundleAPIVersion = "imgpkg.carvel.dev/v1alpha1" - BundleKind = "Bundle" -) - -type BundleVersion struct { - APIVersion string `json:"apiVersion"` // This generated yaml, but due to lib we need to use `json` - Kind string `json:"kind"` // This generated yaml, but due to lib we need to use `json` -} -type Metadata struct { - Version BundleVersion - Metadata map[string]string `json:"metadata,omitempty"` - Authors []Author `json:"authors,omitempty"` - Websites []Website `json:"websites,omitempty"` -} - -func (il *Metadata) ToYAML(w io.Writer) error { - enc := yaml.NewEncoder(w) - enc.SetIndent(2) - - return enc.Encode(il) -} - -func CarvelBundleFromYAMLFile(file string) (*Metadata, error) { - fh, err := os.Open(file) - if err != nil { - return nil, fmt.Errorf("failed to open Images.lock file: %v", err) - } - defer fh.Close() - return CarvelBundleFromYAML(fh) -} - -// reads a Carvel metadata bundled from the YAML read from r -func CarvelBundleFromYAML(r io.Reader) (*Metadata, error) { - metadata := &Metadata{ - Version: BundleVersion{ - APIVersion: BundleAPIVersion, - Kind: BundleKind, - }, - Metadata: map[string]string{}, - Authors: []Author{}, - Websites: []Website{}, - } - dec := yaml.NewDecoder(r) - if err := dec.Decode(metadata); err != nil { - return nil, fmt.Errorf("failed to load Carvel bundle: %v", err) - } - - return metadata, nil -} - -func NewCarvelBundle() *Metadata { - - return &Metadata{ - Version: BundleVersion{ - APIVersion: BundleAPIVersion, - Kind: BundleKind, - }, - Metadata: map[string]string{}, - Authors: []Author{}, - Websites: []Website{}, - } -} diff --git a/chartutils/carvel/carvel.go b/chartutils/carvel/carvel.go new file mode 100644 index 0000000..b3918a0 --- /dev/null +++ b/chartutils/carvel/carvel.go @@ -0,0 +1,168 @@ +// Package chartutils implements helper functions to manipulate helm Charts +package chartutils + +import ( + "fmt" + "io" + "os" + "strings" + + "github.com/vmware-labs/distribution-tooling-for-helm/chartutils" + "github.com/vmware-labs/distribution-tooling-for-helm/imagelock" + "github.com/vmware-tanzu/carvel-imgpkg/pkg/imgpkg/lockconfig" + + "gopkg.in/yaml.v3" +) + +// Somehow there is no data structure for a bundle in Carvel. Copying some basics from the describe command. + +// Author information from a Bundle +type Author struct { + Name string `json:"name,omitempty"` + Email string `json:"email,omitempty"` +} + +// Website URL where more information of the Bundle can be found +type Website struct { + URL string `json:"url,omitempty"` +} + +// Bundle Metadata +const ( + BundleAPIVersion = "imgpkg.carvel.dev/v1alpha1" + BundleKind = "Bundle" +) + +// BundleVersion with detailsa bout the Carvel bundle version +type BundleVersion struct { + APIVersion string `json:"apiVersion"` // This generated yaml, but due to lib we need to use `json` + Kind string `json:"kind"` // This generated yaml, but due to lib we need to use `json` +} + +// Metadata for a Carvel bundle +type Metadata struct { + Version BundleVersion + Metadata map[string]string `json:"metadata,omitempty"` + Authors []Author `json:"authors,omitempty"` + Websites []Website `json:"websites,omitempty"` +} + +// ToYAML serializes the Carvel bundle into YAML +func (il *Metadata) ToYAML(w io.Writer) error { + enc := yaml.NewEncoder(w) + enc.SetIndent(2) + + return enc.Encode(il) +} + +// CarvelBundleFromYAMLFile Deserializes a string into a Metadata struct +func CarvelBundleFromYAMLFile(file string) (*Metadata, error) { + fh, err := os.Open(file) + if err != nil { + return nil, fmt.Errorf("failed to open Images.lock file: %v", err) + } + defer fh.Close() + return CarvelBundleFromYAML(fh) +} + +// CarvelBundleFromYAML reads a Carvel metadata bundled from the YAML read from r +func CarvelBundleFromYAML(r io.Reader) (*Metadata, error) { + metadata := &Metadata{ + Version: BundleVersion{ + APIVersion: BundleAPIVersion, + Kind: BundleKind, + }, + Metadata: map[string]string{}, + Authors: []Author{}, + Websites: []Website{}, + } + dec := yaml.NewDecoder(r) + if err := dec.Decode(metadata); err != nil { + return nil, fmt.Errorf("failed to load Carvel bundle: %v", err) + } + + return metadata, nil +} + +// NewCarvelBundle returns a new carvel bundle Metadata instance +func NewCarvelBundle() *Metadata { + + return &Metadata{ + Version: BundleVersion{ + APIVersion: BundleAPIVersion, + Kind: BundleKind, + }, + Metadata: map[string]string{}, + Authors: []Author{}, + Websites: []Website{}, + } +} + +// PrepareBundleMetadata builds and sets a new Carvel bundle struct +func PrepareBundleMetadata(chartPath string, lock *imagelock.ImagesLock) (*Metadata, error) { + + bundleMetadata := NewCarvelBundle() + + chart, err := chartutils.LoadChart(chartPath) + if err != nil { + return nil, fmt.Errorf("failed to load chart: %w", err) + } + + for _, maintainer := range chart.Metadata.Maintainers { + author := Author{ + Name: maintainer.Name, + } + if maintainer.Email != "" { + author.Email = maintainer.Email + } + bundleMetadata.Authors = append(bundleMetadata.Authors, author) + } + for _, source := range chart.Metadata.Sources { + website := Website{ + URL: source, + } + bundleMetadata.Websites = append(bundleMetadata.Websites, website) + } + + bundleMetadata.Metadata["name"] = lock.Chart.Name + for key, value := range chart.Metadata.Annotations { + if key != "images" { + bundleMetadata.Metadata[key] = value + } + } + return bundleMetadata, nil +} + +// PrepareImagesLock builds and set a new Carvel images lock struct +func PrepareImagesLock(lock *imagelock.ImagesLock) (lockconfig.ImagesLock, error) { + + imagesLock := lockconfig.ImagesLock{ + LockVersion: lockconfig.LockVersion{ + APIVersion: lockconfig.ImagesLockAPIVersion, + Kind: lockconfig.ImagesLockKind, + }, + } + for _, img := range lock.Images { + // Carvel does not seem to support multi-arch. Grab amd64 digest + name := img.Image + i := strings.LastIndex(img.Image, ":") + if i > -1 { + name = img.Image[0:i] + + } + for _, digest := range img.Digests { + if digest.Arch == "linux/amd64" { + name = name + "@" + digest.Digest.String() + break + } + } + imageRef := lockconfig.ImageRef{ + Image: name, + Annotations: map[string]string{ + "kbld.carvel.dev/id": img.Image, + }, + } + imagesLock.AddImageRef(imageRef) + } + return imagesLock, nil +} diff --git a/cmd/dt/carvelize.go b/cmd/dt/carvelize.go index d154524..4905f53 100644 --- a/cmd/dt/carvelize.go +++ b/cmd/dt/carvelize.go @@ -4,13 +4,12 @@ import ( "bytes" "fmt" "os" - "strings" "github.com/spf13/cobra" "github.com/vmware-labs/distribution-tooling-for-helm/chartutils" + carvel "github.com/vmware-labs/distribution-tooling-for-helm/chartutils/carvel" "github.com/vmware-labs/distribution-tooling-for-helm/internal/log" "github.com/vmware-labs/distribution-tooling-for-helm/utils" - "github.com/vmware-tanzu/carvel-imgpkg/pkg/imgpkg/lockconfig" ) var carvelizeCmd = newCarvelizeCmd() @@ -105,66 +104,21 @@ func generateCarvelBundle(chartPath string, opts ...chartutils.Option) error { return fmt.Errorf("wrap file %q does not exist", chartPath) } - bundleMetadata := chartutils.NewCarvelBundle() - - chart, err := chartutils.LoadChart(chartPath) + bundleMetadata, err := carvel.PrepareBundleMetadata(chartPath, lock) if err != nil { - return fmt.Errorf("failed to load chart: %w", err) - } - - for _, maintainer := range chart.Metadata.Maintainers { - author := chartutils.Author{ - Name: maintainer.Name, - } - if maintainer.Email != "" { - author.Email = maintainer.Email - } - bundleMetadata.Authors = append(bundleMetadata.Authors, author) - } - for _, source := range chart.Metadata.Sources { - website := chartutils.Website{ - URL: source, - } - bundleMetadata.Websites = append(bundleMetadata.Websites, website) - } - - bundleMetadata.Metadata["name"] = lock.Chart.Name - for key, value := range chart.Metadata.Annotations { - if key != "images" { - bundleMetadata.Metadata[key] = value - } + return fmt.Errorf("failed to prepare Carvel bundle: %w", err) } - imagesLock := lockconfig.ImagesLock{ - LockVersion: lockconfig.LockVersion{ - APIVersion: lockconfig.ImagesLockAPIVersion, - Kind: lockconfig.ImagesLockKind, - }, + imagesLock, err := carvel.PrepareImagesLock(lock) + if err != nil { + return fmt.Errorf("failed to prepare Carvel images lock: %w", err) } - for _, img := range lock.Images { - // Carvel does not seem to support multi-arch. Grab amd64 digest - name := img.Image - i := strings.LastIndex(img.Image, ":") - if i > -1 { - name = img.Image[0:i] + l.Infof("Validating Carvel images lock") - } - for _, digest := range img.Digests { - if digest.Arch == "linux/amd64" { - name = name + "@" + digest.Digest.String() - break - } - } - imageRef := lockconfig.ImageRef{ - Image: name, - Annotations: map[string]string{ - "kbld.carvel.dev/id": img.Image, - }, - } - imagesLock.AddImageRef(imageRef) + err = imagesLock.Validate() + if err != nil { + return fmt.Errorf("failed to validate Carvel images lock: %w", err) } - l.Infof("Validating Carvel images lock") - imagesLock.Validate() path := imgPkgPath + "/images.yml" err = imagesLock.WriteToPath(path) diff --git a/cmd/dt/carvelize_test.go b/cmd/dt/carvelize_test.go index 56e2821..be1c16b 100644 --- a/cmd/dt/carvelize_test.go +++ b/cmd/dt/carvelize_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "testing" + carvel "github.com/vmware-labs/distribution-tooling-for-helm/chartutils/carvel" tu "github.com/vmware-labs/distribution-tooling-for-helm/internal/testutil" "gopkg.in/yaml.v3" ) @@ -25,34 +26,29 @@ func (suite *CmdSuite) TestCarvelizeCommand() { serverURL := s.ServerURL scenarioName := "custom-chart" chartName := "test" + authors := []carvel.Author{{ + Name: "VMware, Inc.", + Email: "dt@vmware.com", + }} + websites := []carvel.Website{{ + URL: "https://github.com/bitnami/charts/tree/main/bitnami/wordpress", + }} scenarioDir := fmt.Sprintf("../../testdata/scenarios/%s", scenarioName) t := suite.T() - type author struct { - Name string - Email string - } - type website struct { - Url string - } - dest := sb.TempFile() require.NoError(tu.RenderScenario(scenarioDir, dest, map[string]interface{}{"ServerURL": serverURL, "Images": images, "Name": chartName, "RepositoryURL": serverURL, - "Authors": []author{{ - Name: "VMware, Inc.", - Email: "dt@vmware.com", - }}, - "Websites": []website{{ - Url: "https://github.com/bitnami/charts/tree/main/bitnami/wordpress", - }}, + "Authors": authors, "Websites": websites, }, )) chartDir := filepath.Join(dest, scenarioName) bundleData, err := tu.RenderTemplateFile(filepath.Join(scenarioDir, ".imgpkg/bundle.yml.tmpl"), - map[string]interface{}{"ServerURL": serverURL, "Images": images, "Name": chartName}, + map[string]interface{}{"ServerURL": serverURL, "Images": images, "Name": chartName, + "Authors": authors, "Websites": websites, + }, ) require.NoError(err) diff --git a/relocator/chart.go b/relocator/chart.go index 84edda4..738692c 100644 --- a/relocator/chart.go +++ b/relocator/chart.go @@ -130,11 +130,13 @@ func RelocateCarvelImagesLock(lock *lockconfig.ImagesLock, prefix string) (*Relo return nil, fmt.Errorf("failed to relocate Carvel images lock file: %v", err) } - if buff, err := lock.AsBytes(); err != nil { + buff, err := lock.AsBytes() + if err != nil { return nil, fmt.Errorf("failed to write Images.lock file: %v", err) - } else { - return &RelocationResult{Data: buff, Count: count}, nil } + + return &RelocationResult{Data: buff, Count: count}, nil + } func relocateCarvelImages(images []lockconfig.ImageRef, prefix string) (count int, err error) { diff --git a/testdata/scenarios/custom-chart/.imgpkg/bundle.yml.tmpl b/testdata/scenarios/custom-chart/.imgpkg/bundle.yml.tmpl index 3211444..e91eecb 100644 --- a/testdata/scenarios/custom-chart/.imgpkg/bundle.yml.tmpl +++ b/testdata/scenarios/custom-chart/.imgpkg/bundle.yml.tmpl @@ -6,7 +6,11 @@ metadata: licenses: Apache-2.0 name: {{or .Name "WordPress"}} authors: - - name: VMware, Inc. - email: "dt@vmware.com" +{{- range .Authors}} + - name: {{.Name}} + email: {{.Email}} +{{end -}} websites: - - url: https://github.com/bitnami/charts/tree/main/bitnami/wordpress \ No newline at end of file +{{- range .Websites}} + - url: {{.URL}} +{{end -}} diff --git a/testdata/scenarios/custom-chart/Chart.yaml.tmpl b/testdata/scenarios/custom-chart/Chart.yaml.tmpl index 6ecb959..455b080 100644 --- a/testdata/scenarios/custom-chart/Chart.yaml.tmpl +++ b/testdata/scenarios/custom-chart/Chart.yaml.tmpl @@ -29,7 +29,7 @@ maintainers: {{if gt (len .Websites) 0 }} sources: {{- range .Websites}} - - {{.Url}} + - {{.URL}} {{end -}} {{end}} {{end}} From 9f72b3e8b5cfee837b909362e1e49fed536948ad Mon Sep 17 00:00:00 2001 From: Martin Perez Date: Sat, 23 Sep 2023 07:30:57 +0200 Subject: [PATCH 3/3] Refactoring and lint fixing --- {chartutils/carvel => carvel}/carvel.go | 90 +++++++++++-------------- cmd/dt/carvelize.go | 22 +++--- cmd/dt/carvelize_test.go | 6 +- cmd/dt/root.go | 2 +- cmd/dt/wrap.go | 1 + cmd/dt/wrap_test.go | 5 +- relocator/chart.go | 22 +++--- 7 files changed, 67 insertions(+), 81 deletions(-) rename {chartutils/carvel => carvel}/carvel.go (66%) diff --git a/chartutils/carvel/carvel.go b/carvel/carvel.go similarity index 66% rename from chartutils/carvel/carvel.go rename to carvel/carvel.go index b3918a0..59a33fa 100644 --- a/chartutils/carvel/carvel.go +++ b/carvel/carvel.go @@ -1,10 +1,9 @@ -// Package chartutils implements helper functions to manipulate helm Charts -package chartutils +// Package carvel implements experimental Carvel support +package carvel import ( "fmt" "io" - "os" "strings" "github.com/vmware-labs/distribution-tooling-for-helm/chartutils" @@ -14,6 +13,14 @@ import ( "gopkg.in/yaml.v3" ) +// CarvelBundleFilePath represents the usual bundle file for Carvel packaging +const CarvelBundleFilePath = ".imgpkg/bundle.yml" + +// CarvelImagesFilePath represents the usual images file for Carvel packaging +const CarvelImagesFilePath = ".imgpkg/images.yml" + +const carvelID = "kbld.carvel.dev/id" + // Somehow there is no data structure for a bundle in Carvel. Copying some basics from the describe command. // Author information from a Bundle @@ -55,38 +62,8 @@ func (il *Metadata) ToYAML(w io.Writer) error { return enc.Encode(il) } -// CarvelBundleFromYAMLFile Deserializes a string into a Metadata struct -func CarvelBundleFromYAMLFile(file string) (*Metadata, error) { - fh, err := os.Open(file) - if err != nil { - return nil, fmt.Errorf("failed to open Images.lock file: %v", err) - } - defer fh.Close() - return CarvelBundleFromYAML(fh) -} - -// CarvelBundleFromYAML reads a Carvel metadata bundled from the YAML read from r -func CarvelBundleFromYAML(r io.Reader) (*Metadata, error) { - metadata := &Metadata{ - Version: BundleVersion{ - APIVersion: BundleAPIVersion, - Kind: BundleKind, - }, - Metadata: map[string]string{}, - Authors: []Author{}, - Websites: []Website{}, - } - dec := yaml.NewDecoder(r) - if err := dec.Decode(metadata); err != nil { - return nil, fmt.Errorf("failed to load Carvel bundle: %v", err) - } - - return metadata, nil -} - // NewCarvelBundle returns a new carvel bundle Metadata instance func NewCarvelBundle() *Metadata { - return &Metadata{ Version: BundleVersion{ APIVersion: BundleAPIVersion, @@ -98,9 +75,8 @@ func NewCarvelBundle() *Metadata { } } -// PrepareBundleMetadata builds and sets a new Carvel bundle struct -func PrepareBundleMetadata(chartPath string, lock *imagelock.ImagesLock) (*Metadata, error) { - +// CreateBundleMetadata builds and sets a new Carvel bundle struct +func CreateBundleMetadata(chartPath string, lock *imagelock.ImagesLock, cfg *chartutils.Configuration) (*Metadata, error) { bundleMetadata := NewCarvelBundle() chart, err := chartutils.LoadChart(chartPath) @@ -112,9 +88,7 @@ func PrepareBundleMetadata(chartPath string, lock *imagelock.ImagesLock) (*Metad author := Author{ Name: maintainer.Name, } - if maintainer.Email != "" { - author.Email = maintainer.Email - } + author.Email = maintainer.Email bundleMetadata.Authors = append(bundleMetadata.Authors, author) } for _, source := range chart.Metadata.Sources { @@ -126,16 +100,19 @@ func PrepareBundleMetadata(chartPath string, lock *imagelock.ImagesLock) (*Metad bundleMetadata.Metadata["name"] = lock.Chart.Name for key, value := range chart.Metadata.Annotations { - if key != "images" { + annotationsKey := cfg.AnnotationsKey + if annotationsKey == "" { + annotationsKey = imagelock.DefaultAnnotationsKey + } + if key != annotationsKey { bundleMetadata.Metadata[key] = value } } return bundleMetadata, nil } -// PrepareImagesLock builds and set a new Carvel images lock struct -func PrepareImagesLock(lock *imagelock.ImagesLock) (lockconfig.ImagesLock, error) { - +// CreateImagesLock builds and set a new Carvel images lock struct +func CreateImagesLock(lock *imagelock.ImagesLock) (lockconfig.ImagesLock, error) { imagesLock := lockconfig.ImagesLock{ LockVersion: lockconfig.LockVersion{ APIVersion: lockconfig.ImagesLockAPIVersion, @@ -144,25 +121,36 @@ func PrepareImagesLock(lock *imagelock.ImagesLock) (lockconfig.ImagesLock, error } for _, img := range lock.Images { // Carvel does not seem to support multi-arch. Grab amd64 digest + name := img.Image i := strings.LastIndex(img.Image, ":") if i > -1 { name = img.Image[0:i] - } - for _, digest := range img.Digests { - if digest.Arch == "linux/amd64" { - name = name + "@" + digest.Digest.String() - break - } + //TODO: Clarify with Carvel community their multi-arch support + //for the time being we stick to amd64 + imageWithDigest := getIntelImageWithDigest(name, img) + if imageWithDigest == "" { + // See above. Skip + break } imageRef := lockconfig.ImageRef{ - Image: name, + Image: imageWithDigest, Annotations: map[string]string{ - "kbld.carvel.dev/id": img.Image, + carvelID: img.Image, }, } imagesLock.AddImageRef(imageRef) } return imagesLock, nil } + +func getIntelImageWithDigest(name string, img *imagelock.ChartImage) string { + + for _, digest := range img.Digests { + if digest.Arch == "linux/amd64" { + return fmt.Sprintf("%s@%s", name, digest.Digest.String()) + } + } + return "" +} diff --git a/cmd/dt/carvelize.go b/cmd/dt/carvelize.go index 4905f53..1935e81 100644 --- a/cmd/dt/carvelize.go +++ b/cmd/dt/carvelize.go @@ -4,10 +4,11 @@ import ( "bytes" "fmt" "os" + "path/filepath" "github.com/spf13/cobra" + "github.com/vmware-labs/distribution-tooling-for-helm/carvel" "github.com/vmware-labs/distribution-tooling-for-helm/chartutils" - carvel "github.com/vmware-labs/distribution-tooling-for-helm/chartutils/carvel" "github.com/vmware-labs/distribution-tooling-for-helm/internal/log" "github.com/vmware-labs/distribution-tooling-for-helm/utils" ) @@ -63,6 +64,7 @@ func newCarvelizeCmd() *cobra.Command { if err := l.Section(fmt.Sprintf("Generating Carvel bundle for Helm chart %q", chartPath), func(childLog log.SectionLogger) error { if err := generateCarvelBundle( chartPath, + chartutils.WithAnnotationsKey(getAnnotationsKey()), chartutils.WithLog(childLog), ); err != nil { return childLog.Failf("%v", err) @@ -82,7 +84,6 @@ func newCarvelizeCmd() *cobra.Command { } func generateCarvelBundle(chartPath string, opts ...chartutils.Option) error { - cfg := chartutils.NewConfiguration(opts...) l := cfg.Log @@ -91,7 +92,7 @@ func generateCarvelBundle(chartPath string, opts ...chartutils.Option) error { return fmt.Errorf("failed to load Images.lock: %v", err) } - imgPkgPath := chartPath + "/.imgpkg" + imgPkgPath := filepath.Join(chartPath, ".imgpkg") if !utils.FileExists(imgPkgPath) { err := os.Mkdir(imgPkgPath, os.FileMode(0755)) if err != nil { @@ -99,29 +100,24 @@ func generateCarvelBundle(chartPath string, opts ...chartutils.Option) error { } } - _, _ = chartPath, l - if !utils.FileExists(chartPath) { - return fmt.Errorf("wrap file %q does not exist", chartPath) - } - - bundleMetadata, err := carvel.PrepareBundleMetadata(chartPath, lock) + bundleMetadata, err := carvel.CreateBundleMetadata(chartPath, lock, cfg) if err != nil { return fmt.Errorf("failed to prepare Carvel bundle: %w", err) } - imagesLock, err := carvel.PrepareImagesLock(lock) + carvelImagesLock, err := carvel.CreateImagesLock(lock) if err != nil { return fmt.Errorf("failed to prepare Carvel images lock: %w", err) } l.Infof("Validating Carvel images lock") - err = imagesLock.Validate() + err = carvelImagesLock.Validate() if err != nil { return fmt.Errorf("failed to validate Carvel images lock: %w", err) } - path := imgPkgPath + "/images.yml" - err = imagesLock.WriteToPath(path) + path := filepath.Join(imgPkgPath, "images.yml") + err = carvelImagesLock.WriteToPath(path) if err != nil { return fmt.Errorf("Could not write image lock: %v", err) } diff --git a/cmd/dt/carvelize_test.go b/cmd/dt/carvelize_test.go index be1c16b..f799458 100644 --- a/cmd/dt/carvelize_test.go +++ b/cmd/dt/carvelize_test.go @@ -6,7 +6,7 @@ import ( "path/filepath" "testing" - carvel "github.com/vmware-labs/distribution-tooling-for-helm/chartutils/carvel" + carvel "github.com/vmware-labs/distribution-tooling-for-helm/carvel" tu "github.com/vmware-labs/distribution-tooling-for-helm/internal/testutil" "gopkg.in/yaml.v3" ) @@ -68,7 +68,7 @@ func (suite *CmdSuite) TestCarvelizeCommand() { res.AssertSuccess(t) t.Run("Generates Carvel bundle", func(t *testing.T) { - newBundleData, err := os.ReadFile(filepath.Join(chartDir, ".imgpkg/bundle.yml")) + newBundleData, err := os.ReadFile(filepath.Join(chartDir, carvel.CarvelBundleFilePath)) require.NoError(err) var newBundle map[string]interface{} require.NoError(yaml.Unmarshal(newBundleData, &newBundle)) @@ -77,7 +77,7 @@ func (suite *CmdSuite) TestCarvelizeCommand() { }) t.Run("Generates Carvel images", func(t *testing.T) { - newImagesData, err := os.ReadFile(filepath.Join(chartDir, ".imgpkg/images.yml")) + newImagesData, err := os.ReadFile(filepath.Join(chartDir, carvel.CarvelImagesFilePath)) require.NoError(err) var newImagesLock map[string]interface{} require.NoError(yaml.Unmarshal(newImagesData, &newImagesLock)) diff --git a/cmd/dt/root.go b/cmd/dt/root.go index e3ee199..72ba31d 100644 --- a/cmd/dt/root.go +++ b/cmd/dt/root.go @@ -21,7 +21,7 @@ const ( terminalSpacer = "" ) -// Global falgs +// Global flags var ( insecure bool annotationsKey string = imagelock.DefaultAnnotationsKey diff --git a/cmd/dt/wrap.go b/cmd/dt/wrap.go index 737b624..54ee69b 100644 --- a/cmd/dt/wrap.go +++ b/cmd/dt/wrap.go @@ -94,6 +94,7 @@ func wrapChart(ctx context.Context, inputPath string, outputFile string, platfor if err := l.Section(fmt.Sprintf("Generating Carvel bundle for Helm chart %q", chartPath), func(childLog log.SectionLogger) error { if err := generateCarvelBundle( chartPath, + chartutils.WithAnnotationsKey(annotationsKey), chartutils.WithLog(childLog), ); err != nil { return childLog.Failf("%v", err) diff --git a/cmd/dt/wrap_test.go b/cmd/dt/wrap_test.go index b1d4dca..9ee4cf3 100644 --- a/cmd/dt/wrap_test.go +++ b/cmd/dt/wrap_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/google/go-containerregistry/pkg/registry" + "github.com/vmware-labs/distribution-tooling-for-helm/carvel" tu "github.com/vmware-labs/distribution-tooling-for-helm/internal/testutil" "github.com/vmware-labs/distribution-tooling-for-helm/utils" "gopkg.in/yaml.v3" @@ -102,9 +103,9 @@ func (suite *CmdSuite) TestWrapCommand() { assert.FileExists(lockFile) if generateCarvelBundle { - carvelBundleFile := filepath.Join(tmpDir, ".imgpkg/bundle.yml") + carvelBundleFile := filepath.Join(tmpDir, carvel.CarvelBundleFilePath) assert.FileExists(carvelBundleFile) - carvelImagesLockFile := filepath.Join(tmpDir, ".imgpkg/images.yml") + carvelImagesLockFile := filepath.Join(tmpDir, carvel.CarvelImagesFilePath) assert.FileExists(carvelImagesLockFile) } diff --git a/relocator/chart.go b/relocator/chart.go index 738692c..4ef8f87 100644 --- a/relocator/chart.go +++ b/relocator/chart.go @@ -6,8 +6,10 @@ import ( "errors" "fmt" "os" + "path/filepath" "strings" + "github.com/vmware-labs/distribution-tooling-for-helm/carvel" cu "github.com/vmware-labs/distribution-tooling-for-helm/chartutils" "github.com/vmware-labs/distribution-tooling-for-helm/imagelock" "github.com/vmware-labs/distribution-tooling-for-helm/utils" @@ -79,9 +81,12 @@ func RelocateChartDir(chartPath string, prefix string, opts ...RelocateOption) e if err != nil { return err } - err = relocateCarvelBundle(chartPath, prefix) - if err != nil { - return err + if utils.FileExists(filepath.Join(chartPath, carvel.CarvelImagesFilePath)) { + err = relocateCarvelBundle(chartPath, prefix) + + if err != nil { + return err + } } var allErrors error @@ -99,13 +104,8 @@ func RelocateChartDir(chartPath string, prefix string, opts ...RelocateOption) e func relocateCarvelBundle(chartRoot string, prefix string) error { //TODO: Do better detection here, imgpkg probably has something - lockFile := chartRoot + "/.imgpkg/images.yml" - if !utils.FileExists(lockFile) { - fmt.Printf("Did not find Carvel images bundle at %s. Ignoring", lockFile) - return nil - } - - lock, err := lockconfig.NewImagesLockFromPath(lockFile) + carvelImagesFile := filepath.Join(chartRoot, carvel.CarvelImagesFilePath) + lock, err := lockconfig.NewImagesLockFromPath(carvelImagesFile) if err != nil { return fmt.Errorf("failed to load Carvel images lock: %v", err) } @@ -116,7 +116,7 @@ func relocateCarvelBundle(chartRoot string, prefix string) error { if result.Count == 0 { return nil } - if err := utils.SafeWriteFile(lockFile, result.Data, 0600); err != nil { + if err := utils.SafeWriteFile(carvelImagesFile, result.Data, 0600); err != nil { return fmt.Errorf("failed to overwrite Carvel images lock file: %v", err) } return nil