diff --git a/README.md b/README.md index a409a49..35d3056 100644 --- a/README.md +++ b/README.md @@ -405,7 +405,7 @@ images: ... ``` -### Annotating a chart (EXPERIMENTAL) +### Annotating a Helm chart (EXPERIMENTAL) `Images.lock` creation relies on the existence of the special images annotation inside `Chart.yaml`. If you have a Helm chart that does not contain any annotations, this command can be used to guess and generate an annotation with a tentative list of images. It's important to note that this list is a **best-effort** as the list of images is obtained from the `values.yaml` file and this is always an unreliable, often incomplete, and error-prone source as the configuration in `values.yaml` is very variable. @@ -414,6 +414,21 @@ $ helm dt chart annotate examples/mariadb INFO[0000] Helm chart annotated successfully ``` +### Converting a Helm chart into a Carvel bundle (EXPERIMENTAL) + +From `dt` 0.1.1 we have introduced a new command to create a [Carvel bundle](https://carvel.dev/imgpkg/docs/v0.37.x/resources/#bundle) from any Helm chart. + + +```console +$ helm dt chart carvelize examples/postgresql + ✔ Helm chart "examples/postgresql" lock is valid + » Generating Carvel bundle for Helm chart "examples/postgresql" + ✔ Validating Carvel images lock + ✔ Carvel images lock written to "examples/postgresql/.imgpkg/images.yml" + ✔ Carvel metadata written to "examples/postgresql/.imgpkg/bundle.yml" + 🎉 Carvel bundle created successfully +``` + ## Frequently Asked Questions ### I cannot install the plugin due to `Error: Unable to update repository: exit status 1` diff --git a/carvel/carvel.go b/carvel/carvel.go new file mode 100644 index 0000000..59a33fa --- /dev/null +++ b/carvel/carvel.go @@ -0,0 +1,156 @@ +// Package carvel implements experimental Carvel support +package carvel + +import ( + "fmt" + "io" + "strings" + + "github.com/vmware-labs/distribution-tooling-for-helm/chartutils" + "github.com/vmware-labs/distribution-tooling-for-helm/imagelock" + "github.com/vmware-tanzu/carvel-imgpkg/pkg/imgpkg/lockconfig" + + "gopkg.in/yaml.v3" +) + +// CarvelBundleFilePath represents the usual bundle file for Carvel packaging +const CarvelBundleFilePath = ".imgpkg/bundle.yml" + +// CarvelImagesFilePath represents the usual images file for Carvel packaging +const CarvelImagesFilePath = ".imgpkg/images.yml" + +const carvelID = "kbld.carvel.dev/id" + +// Somehow there is no data structure for a bundle in Carvel. Copying some basics from the describe command. + +// Author information from a Bundle +type Author struct { + Name string `json:"name,omitempty"` + Email string `json:"email,omitempty"` +} + +// Website URL where more information of the Bundle can be found +type Website struct { + URL string `json:"url,omitempty"` +} + +// Bundle Metadata +const ( + BundleAPIVersion = "imgpkg.carvel.dev/v1alpha1" + BundleKind = "Bundle" +) + +// BundleVersion with detailsa bout the Carvel bundle version +type BundleVersion struct { + APIVersion string `json:"apiVersion"` // This generated yaml, but due to lib we need to use `json` + Kind string `json:"kind"` // This generated yaml, but due to lib we need to use `json` +} + +// Metadata for a Carvel bundle +type Metadata struct { + Version BundleVersion + Metadata map[string]string `json:"metadata,omitempty"` + Authors []Author `json:"authors,omitempty"` + Websites []Website `json:"websites,omitempty"` +} + +// ToYAML serializes the Carvel bundle into YAML +func (il *Metadata) ToYAML(w io.Writer) error { + enc := yaml.NewEncoder(w) + enc.SetIndent(2) + + return enc.Encode(il) +} + +// NewCarvelBundle returns a new carvel bundle Metadata instance +func NewCarvelBundle() *Metadata { + return &Metadata{ + Version: BundleVersion{ + APIVersion: BundleAPIVersion, + Kind: BundleKind, + }, + Metadata: map[string]string{}, + Authors: []Author{}, + Websites: []Website{}, + } +} + +// CreateBundleMetadata builds and sets a new Carvel bundle struct +func CreateBundleMetadata(chartPath string, lock *imagelock.ImagesLock, cfg *chartutils.Configuration) (*Metadata, error) { + bundleMetadata := NewCarvelBundle() + + chart, err := chartutils.LoadChart(chartPath) + if err != nil { + return nil, fmt.Errorf("failed to load chart: %w", err) + } + + for _, maintainer := range chart.Metadata.Maintainers { + author := Author{ + Name: maintainer.Name, + } + author.Email = maintainer.Email + bundleMetadata.Authors = append(bundleMetadata.Authors, author) + } + for _, source := range chart.Metadata.Sources { + website := Website{ + URL: source, + } + bundleMetadata.Websites = append(bundleMetadata.Websites, website) + } + + bundleMetadata.Metadata["name"] = lock.Chart.Name + for key, value := range chart.Metadata.Annotations { + annotationsKey := cfg.AnnotationsKey + if annotationsKey == "" { + annotationsKey = imagelock.DefaultAnnotationsKey + } + if key != annotationsKey { + bundleMetadata.Metadata[key] = value + } + } + return bundleMetadata, nil +} + +// CreateImagesLock builds and set a new Carvel images lock struct +func CreateImagesLock(lock *imagelock.ImagesLock) (lockconfig.ImagesLock, error) { + imagesLock := lockconfig.ImagesLock{ + LockVersion: lockconfig.LockVersion{ + APIVersion: lockconfig.ImagesLockAPIVersion, + Kind: lockconfig.ImagesLockKind, + }, + } + for _, img := range lock.Images { + // Carvel does not seem to support multi-arch. Grab amd64 digest + + name := img.Image + i := strings.LastIndex(img.Image, ":") + if i > -1 { + name = img.Image[0:i] + } + //TODO: Clarify with Carvel community their multi-arch support + //for the time being we stick to amd64 + imageWithDigest := getIntelImageWithDigest(name, img) + if imageWithDigest == "" { + // See above. Skip + break + } + imageRef := lockconfig.ImageRef{ + Image: imageWithDigest, + Annotations: map[string]string{ + carvelID: img.Image, + }, + } + imagesLock.AddImageRef(imageRef) + } + return imagesLock, nil +} + +func getIntelImageWithDigest(name string, img *imagelock.ChartImage) string { + + for _, digest := range img.Digests { + if digest.Arch == "linux/amd64" { + return fmt.Sprintf("%s@%s", name, digest.Digest.String()) + } + } + return "" +} diff --git a/cmd/dt/carvelize.go b/cmd/dt/carvelize.go new file mode 100644 index 0000000..1935e81 --- /dev/null +++ b/cmd/dt/carvelize.go @@ -0,0 +1,141 @@ +package main + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + + "github.com/spf13/cobra" + "github.com/vmware-labs/distribution-tooling-for-helm/carvel" + "github.com/vmware-labs/distribution-tooling-for-helm/chartutils" + "github.com/vmware-labs/distribution-tooling-for-helm/internal/log" + "github.com/vmware-labs/distribution-tooling-for-helm/utils" +) + +var carvelizeCmd = newCarvelizeCmd() + +func newCarvelizeCmd() *cobra.Command { + var yamlFormat bool + var showDetails bool + + cmd := &cobra.Command{ + Use: "carvelize FILE", + Short: "Adds a Carvel bundle to the Helm chart (Experimental)", + Long: `Experimental. Adds a Carvel bundle to an existing Helm chart`, + Example: ` # Adds a Carvel bundle to a Helm chart + $ dt charts carvelize examples/mariadb`, + SilenceUsage: true, + SilenceErrors: true, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + chartPath := args[0] + l := getLogger() + // Allows silencing called methods + silentLog := log.SilentLog + + lockFile, err := getImageLockFilePath(chartPath) + if err != nil { + return fmt.Errorf("failed to determine Images.lock file location: %w", err) + } + + if utils.FileExists(lockFile) { + if err := l.ExecuteStep("Verifying Images.lock", func() error { + return verifyLock(chartPath, lockFile) + }); err != nil { + return l.Failf("Failed to verify lock: %w", err) + } + l.Infof("Helm chart %q lock is valid", chartPath) + + } else { + err := l.ExecuteStep( + "Images.lock file does not exist. Generating it from annotations...", + func() error { + return createImagesLock(chartPath, + lockFile, silentLog, + ) + }, + ) + if err != nil { + return l.Failf("Failed to generate lock: %w", err) + } + l.Infof("Images.lock file written to %q", lockFile) + } + if err := l.Section(fmt.Sprintf("Generating Carvel bundle for Helm chart %q", chartPath), func(childLog log.SectionLogger) error { + if err := generateCarvelBundle( + chartPath, + chartutils.WithAnnotationsKey(getAnnotationsKey()), + chartutils.WithLog(childLog), + ); err != nil { + return childLog.Failf("%v", err) + } + return nil + }); err != nil { + return l.Failf("%w", err) + } + l.Successf("Carvel bundle created successfully") + return nil + }, + } + cmd.PersistentFlags().BoolVar(&yamlFormat, "yaml", yamlFormat, "Show report in YAML format") + cmd.PersistentFlags().BoolVar(&showDetails, "detailed", showDetails, "When using the printable report, add more details about the bundled images") + + return cmd +} + +func generateCarvelBundle(chartPath string, opts ...chartutils.Option) error { + cfg := chartutils.NewConfiguration(opts...) + l := cfg.Log + + lock, err := readLockFromWrap(chartPath) + if err != nil { + return fmt.Errorf("failed to load Images.lock: %v", err) + } + + imgPkgPath := filepath.Join(chartPath, ".imgpkg") + if !utils.FileExists(imgPkgPath) { + err := os.Mkdir(imgPkgPath, os.FileMode(0755)) + if err != nil { + return fmt.Errorf("failed to create .imgpkg directory: %w", err) + } + } + + bundleMetadata, err := carvel.CreateBundleMetadata(chartPath, lock, cfg) + if err != nil { + return fmt.Errorf("failed to prepare Carvel bundle: %w", err) + } + + carvelImagesLock, err := carvel.CreateImagesLock(lock) + if err != nil { + return fmt.Errorf("failed to prepare Carvel images lock: %w", err) + } + l.Infof("Validating Carvel images lock") + + err = carvelImagesLock.Validate() + if err != nil { + return fmt.Errorf("failed to validate Carvel images lock: %w", err) + } + + path := filepath.Join(imgPkgPath, "images.yml") + err = carvelImagesLock.WriteToPath(path) + if err != nil { + return fmt.Errorf("Could not write image lock: %v", err) + } + l.Infof("Carvel images lock written to %q", path) + + buff := &bytes.Buffer{} + if err = bundleMetadata.ToYAML(buff); err != nil { + return fmt.Errorf("failed to write bundle metadata file: %v", err) + } + + path = imgPkgPath + "/bundle.yml" + if err := os.WriteFile(path, buff.Bytes(), 0666); err != nil { + return fmt.Errorf("failed to write Carvel bundle metadata to %q: %w", path, err) + } + l.Infof("Carvel metadata written to %q", path) + return nil +} + +func init() { + rootCmd.AddCommand(infoCmd) +} diff --git a/cmd/dt/carvelize_test.go b/cmd/dt/carvelize_test.go new file mode 100644 index 0000000..f799458 --- /dev/null +++ b/cmd/dt/carvelize_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + carvel "github.com/vmware-labs/distribution-tooling-for-helm/carvel" + tu "github.com/vmware-labs/distribution-tooling-for-helm/internal/testutil" + "gopkg.in/yaml.v3" +) + +func (suite *CmdSuite) TestCarvelizeCommand() { + require := suite.Require() + + s, err := tu.NewTestServer() + require.NoError(err) + + defer s.Close() + + images, err := s.LoadImagesFromFile("../../testdata/images.json") + require.NoError(err) + + sb := suite.sb + serverURL := s.ServerURL + scenarioName := "custom-chart" + chartName := "test" + authors := []carvel.Author{{ + Name: "VMware, Inc.", + Email: "dt@vmware.com", + }} + websites := []carvel.Website{{ + URL: "https://github.com/bitnami/charts/tree/main/bitnami/wordpress", + }} + + scenarioDir := fmt.Sprintf("../../testdata/scenarios/%s", scenarioName) + t := suite.T() + + dest := sb.TempFile() + require.NoError(tu.RenderScenario(scenarioDir, dest, + map[string]interface{}{"ServerURL": serverURL, "Images": images, "Name": chartName, "RepositoryURL": serverURL, + "Authors": authors, "Websites": websites, + }, + )) + chartDir := filepath.Join(dest, scenarioName) + + bundleData, err := tu.RenderTemplateFile(filepath.Join(scenarioDir, ".imgpkg/bundle.yml.tmpl"), + map[string]interface{}{"ServerURL": serverURL, "Images": images, "Name": chartName, + "Authors": authors, "Websites": websites, + }, + ) + + require.NoError(err) + var expectedBundle map[string]interface{} + require.NoError(yaml.Unmarshal([]byte(bundleData), &expectedBundle)) + + carvelImagesData, err := tu.RenderTemplateFile(filepath.Join(scenarioDir, ".imgpkg/images.yml.tmpl"), + map[string]interface{}{"ServerURL": serverURL, "Images": images, "Name": chartName}, + ) + require.NoError(err) + var expectedCarvelImagesLock map[string]interface{} + require.NoError(yaml.Unmarshal([]byte(carvelImagesData), &expectedCarvelImagesLock)) + + // We need to provide the --insecure flag or our test server won't validate + args := []string{"charts", "carvelize", "--insecure", chartDir} + res := dt(args...) + res.AssertSuccess(t) + + t.Run("Generates Carvel bundle", func(t *testing.T) { + newBundleData, err := os.ReadFile(filepath.Join(chartDir, carvel.CarvelBundleFilePath)) + require.NoError(err) + var newBundle map[string]interface{} + require.NoError(yaml.Unmarshal(newBundleData, &newBundle)) + + require.Equal(expectedBundle, newBundle) + }) + + t.Run("Generates Carvel images", func(t *testing.T) { + newImagesData, err := os.ReadFile(filepath.Join(chartDir, carvel.CarvelImagesFilePath)) + require.NoError(err) + var newImagesLock map[string]interface{} + require.NoError(yaml.Unmarshal(newImagesData, &newImagesLock)) + + require.Equal(expectedCarvelImagesLock, newImagesLock) + }) +} diff --git a/cmd/dt/chart.go b/cmd/dt/chart.go index 9a62d2a..c944078 100644 --- a/cmd/dt/chart.go +++ b/cmd/dt/chart.go @@ -15,5 +15,5 @@ var chartCmd = &cobra.Command{ } func init() { - chartCmd.AddCommand(relocateCmd, annotateCmd) + chartCmd.AddCommand(relocateCmd, annotateCmd, carvelizeCmd) } diff --git a/cmd/dt/chart_test.go b/cmd/dt/chart_test.go index 821ceaa..61b0a13 100644 --- a/cmd/dt/chart_test.go +++ b/cmd/dt/chart_test.go @@ -12,6 +12,7 @@ func (suite *CmdSuite) TestChartsHelp() { res.AssertSuccess(t) for _, reStr := range []string{ `annotate\s+Annotates a Helm chart`, + `carvelize\s+Adds a Carvel bundle to the Helm chart`, `relocate\s+Relocates a Helm chart`, } { res.AssertSuccessMatch(t, fmt.Sprintf(`(?s).*Available Commands:.*\n\s*%s.*`, reStr)) diff --git a/cmd/dt/relocate.go b/cmd/dt/relocate.go index 580675a..19d2e9f 100644 --- a/cmd/dt/relocate.go +++ b/cmd/dt/relocate.go @@ -39,6 +39,7 @@ func newRelocateCmd() *cobra.Command { return fmt.Errorf("repository cannot be empty") } l := getLogger() + if err := l.ExecuteStep(fmt.Sprintf("Relocating %q with prefix %q", chartPath, repository), func() error { return relocateChart(chartPath, repository, relocator.WithLog(l)) }); err != nil { diff --git a/cmd/dt/root.go b/cmd/dt/root.go index e3ee199..72ba31d 100644 --- a/cmd/dt/root.go +++ b/cmd/dt/root.go @@ -21,7 +21,7 @@ const ( terminalSpacer = "" ) -// Global falgs +// Global flags var ( insecure bool annotationsKey string = imagelock.DefaultAnnotationsKey diff --git a/cmd/dt/wrap.go b/cmd/dt/wrap.go index c2d9a54..54ee69b 100644 --- a/cmd/dt/wrap.go +++ b/cmd/dt/wrap.go @@ -62,6 +62,7 @@ func wrapChart(ctx context.Context, inputPath string, outputFile string, platfor } l.Infof("Images.lock file written to %q", lockFile) } + if outputFile == "" { outputBaseName := fmt.Sprintf("%s-%s.wrap.tgz", chart.Name(), chart.Metadata.Version) if outputFile, err = filepath.Abs(outputBaseName); err != nil { @@ -84,6 +85,27 @@ func wrapChart(ctx context.Context, inputPath string, outputFile string, platfor return err } + carvelize, err := flags.GetBool("add-carvel-bundle") + if err != nil { + return fmt.Errorf("failed to retrieve add-carvel-bundle flag: %w", err) + } + + if carvelize { + if err := l.Section(fmt.Sprintf("Generating Carvel bundle for Helm chart %q", chartPath), func(childLog log.SectionLogger) error { + if err := generateCarvelBundle( + chartPath, + chartutils.WithAnnotationsKey(annotationsKey), + chartutils.WithLog(childLog), + ); err != nil { + return childLog.Failf("%v", err) + } + return nil + }); err != nil { + return l.Failf("%w", err) + } + l.Infof("Carvel bundle created successfully") + } + if err := l.ExecuteStep( "Compressing Helm chart...", func() error { @@ -104,6 +126,7 @@ func newWrapCommand() *cobra.Command { var outputFile string var version string var platforms []string + var carvelize bool var examples = ` # Wrap a Helm chart from a local folder $ dt wrap examples/mariadb @@ -140,6 +163,7 @@ This command will pull all the container images and wrap it into a single tarbal cmd.PersistentFlags().StringVar(&version, "version", version, "when wrapping remote Helm charts from OCI, version to request") cmd.PersistentFlags().StringVar(&outputFile, "output-file", outputFile, "generate a tar.gz with the output of the pull operation") cmd.PersistentFlags().StringSliceVar(&platforms, "platforms", platforms, "platforms to include in the Images.lock file") + cmd.PersistentFlags().BoolVar(&carvelize, "add-carvel-bundle", carvelize, "whether the wrap should include a Carvel bundle or not") return cmd } diff --git a/cmd/dt/wrap_test.go b/cmd/dt/wrap_test.go index 9f379e8..9ee4cf3 100644 --- a/cmd/dt/wrap_test.go +++ b/cmd/dt/wrap_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/google/go-containerregistry/pkg/registry" + "github.com/vmware-labs/distribution-tooling-for-helm/carvel" tu "github.com/vmware-labs/distribution-tooling-for-helm/internal/testutil" "github.com/vmware-labs/distribution-tooling-for-helm/utils" "gopkg.in/yaml.v3" @@ -59,7 +60,8 @@ func (suite *CmdSuite) TestWrapCommand() { } return chartDir } - testWrap := func(t *testing.T, inputChart string, outputFile string, expectedLock map[string]interface{}) { + testWrap := func(t *testing.T, inputChart string, outputFile string, expectedLock map[string]interface{}, + generateCarvelBundle bool) { // Setup a working directory to look for the wrap when not providing a output-filename currentDir, err := os.Getwd() require.NoError(err) @@ -78,6 +80,9 @@ func (suite *CmdSuite) TestWrapCommand() { } else { expectedWrapFile = filepath.Join(workingDir, fmt.Sprintf("%s-%v.wrap.tgz", chartName, version)) } + if generateCarvelBundle { + args = append(args, "--add-carvel-bundle") + } res := dt(args...) res.AssertSuccess(t) @@ -97,6 +102,13 @@ func (suite *CmdSuite) TestWrapCommand() { lockFile := filepath.Join(tmpDir, "Images.lock") assert.FileExists(lockFile) + if generateCarvelBundle { + carvelBundleFile := filepath.Join(tmpDir, carvel.CarvelBundleFilePath) + assert.FileExists(carvelBundleFile) + carvelImagesLockFile := filepath.Join(tmpDir, carvel.CarvelImagesFilePath) + assert.FileExists(carvelImagesLockFile) + } + newData, err := os.ReadFile(lockFile) require.NoError(err) var newLock map[string]interface{} @@ -107,7 +119,7 @@ func (suite *CmdSuite) TestWrapCommand() { assert.Equal(expectedLock, newLock) } - testSampleWrap := func(t *testing.T, withLock bool, outputFile string) { + testSampleWrap := func(t *testing.T, withLock bool, outputFile string, generateCarvelBundle bool) { dest := sb.TempFile() chartDir := createSampleChart(dest, withLock) @@ -121,14 +133,14 @@ func (suite *CmdSuite) TestWrapCommand() { // Clear the timestamp expectedLock["metadata"] = nil - testWrap(t, chartDir, outputFile, expectedLock) + testWrap(t, chartDir, outputFile, expectedLock, generateCarvelBundle) } t.Run("Wrap Chart without exiting lock", func(t *testing.T) { - testSampleWrap(t, withoutLock, "") + testSampleWrap(t, withoutLock, "", false) }) t.Run("Wrap Chart with exiting lock", func(t *testing.T) { - testSampleWrap(t, withLock, "") + testSampleWrap(t, withLock, "", false) }) t.Run("Wrap Chart From compressed tgz", func(t *testing.T) { dest := sb.TempFile() @@ -149,7 +161,7 @@ func (suite *CmdSuite) TestWrapCommand() { require.NoError(utils.Tar(chartDir, tarFilename, utils.TarConfig{})) require.FileExists(tarFilename) - testWrap(t, tarFilename, "", expectedLock) + testWrap(t, tarFilename, "", expectedLock, false) }) t.Run("Wrap Chart From oci", func(t *testing.T) { @@ -185,13 +197,18 @@ func (suite *CmdSuite) TestWrapCommand() { require.NoError(utils.PushChart(tarFilename, pushChartURL)) - testWrap(t, fullChartURL, "", expectedLock) + testWrap(t, fullChartURL, "", expectedLock, false) }) t.Run("Wrap Chart with custom output filename", func(t *testing.T) { tempFilename := fmt.Sprintf("%s/chart.wrap.tar.gz", sb.TempFile()) - testSampleWrap(t, withLock, tempFilename) + testSampleWrap(t, withLock, tempFilename, false) // This should already be handled by testWrap, but make sure it is there suite.Assert().FileExists(tempFilename) }) + + t.Run("Wrap Chart and generate carvel bundle", func(t *testing.T) { + tempFilename := fmt.Sprintf("%s/chart.wrap.tar.gz", sb.TempFile()) + testSampleWrap(t, withLock, tempFilename, true) // triggers the Carvel checks + }) } diff --git a/go.mod b/go.mod index 4f77ca3..697cec1 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect - github.com/fatih/color v1.13.0 // indirect + github.com/fatih/color v1.14.1 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-gorp/gorp/v3 v3.0.5 // indirect github.com/go-logr/logr v1.2.3 // indirect @@ -84,7 +84,7 @@ require ( github.com/lithammer/fuzzysearch v1.1.8 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect @@ -114,6 +114,8 @@ require ( github.com/spf13/cast v1.5.0 // indirect github.com/spf13/pflag v1.0.5 github.com/vbatts/tar-split v0.11.3 // indirect + github.com/vmware-tanzu/carvel-imgpkg v0.37.3 + github.com/vmware-tanzu/carvel-kapp-controller v0.47.0 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect @@ -126,11 +128,11 @@ require ( golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.7.0 // indirect - golang.org/x/sync v0.1.0 // indirect + golang.org/x/sync v0.2.0 // indirect golang.org/x/sys v0.10.0 // indirect golang.org/x/term v0.10.0 // indirect golang.org/x/text v0.11.0 // indirect - golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect + golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect google.golang.org/grpc v1.53.0 // indirect diff --git a/go.sum b/go.sum index daf22ee..9a99543 100644 --- a/go.sum +++ b/go.sum @@ -183,6 +183,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= +github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= @@ -455,6 +457,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-oci8 v0.1.1/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= @@ -662,6 +666,12 @@ github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RV github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= github.com/vmware-labs/yaml-jsonpath v0.3.2 h1:/5QKeCBGdsInyDCyVNLbXyilb61MXGi9NP674f9Hobk= github.com/vmware-labs/yaml-jsonpath v0.3.2/go.mod h1:U6whw1z03QyqgWdgXxvVnQ90zN1BWz5V+51Ewf8k+rQ= +github.com/vmware-tanzu/carvel-imgpkg v0.37.3 h1:e83Ugr6Lmuw1Q8nD9U0gdPXN1hP5iLW9JMxJs5gV2SQ= +github.com/vmware-tanzu/carvel-imgpkg v0.37.3/go.mod h1:YFKwj9DebWdI/yOrHNDGtS7WEUYmYUrXjZl/zAlzX08= +github.com/vmware-tanzu/carvel-kapp-controller v0.47.0 h1:5HouKdHw3dfaDWsN0cQQGahnEtKjErgXQhCz4JuJPZg= +github.com/vmware-tanzu/carvel-kapp-controller v0.47.0/go.mod h1:NEEM+AklzRenvpeFMMM9EweGW1GqrtUZbnSZYIb0HE8= +github.com/vmware-tanzu/carvel-vendir v0.33.1 h1:5wzx0aRyEiorkWwrpGvACJOpFcgvmCeVqhIY9TPuLvk= +github.com/vmware-tanzu/carvel-vendir v0.33.1/go.mod h1:cZEa46rwzPt/ROdAuIgrCXLDJ+LqxNZaNxbz8MLtwWc= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -847,6 +857,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -921,6 +933,7 @@ golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -952,6 +965,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= +golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/relocator/chart.go b/relocator/chart.go index b836601..4ef8f87 100644 --- a/relocator/chart.go +++ b/relocator/chart.go @@ -6,11 +6,14 @@ import ( "errors" "fmt" "os" + "path/filepath" "strings" + "github.com/vmware-labs/distribution-tooling-for-helm/carvel" cu "github.com/vmware-labs/distribution-tooling-for-helm/chartutils" "github.com/vmware-labs/distribution-tooling-for-helm/imagelock" "github.com/vmware-labs/distribution-tooling-for-helm/utils" + "github.com/vmware-tanzu/carvel-imgpkg/pkg/imgpkg/lockconfig" ) // RelocationResult describes the result of performing a relocation @@ -78,6 +81,13 @@ func RelocateChartDir(chartPath string, prefix string, opts ...RelocateOption) e if err != nil { return err } + if utils.FileExists(filepath.Join(chartPath, carvel.CarvelImagesFilePath)) { + err = relocateCarvelBundle(chartPath, prefix) + + if err != nil { + return err + } + } var allErrors error @@ -91,6 +101,58 @@ func RelocateChartDir(chartPath string, prefix string, opts ...RelocateOption) e return allErrors } +func relocateCarvelBundle(chartRoot string, prefix string) error { + + //TODO: Do better detection here, imgpkg probably has something + carvelImagesFile := filepath.Join(chartRoot, carvel.CarvelImagesFilePath) + lock, err := lockconfig.NewImagesLockFromPath(carvelImagesFile) + if err != nil { + return fmt.Errorf("failed to load Carvel images lock: %v", err) + } + result, err := RelocateCarvelImagesLock(&lock, prefix) + if err != nil { + return err + } + if result.Count == 0 { + return nil + } + if err := utils.SafeWriteFile(carvelImagesFile, result.Data, 0600); err != nil { + return fmt.Errorf("failed to overwrite Carvel images lock file: %v", err) + } + return nil +} + +// RelocateCarvelImagesLock rewrites the images urls in the provided lock using prefix +func RelocateCarvelImagesLock(lock *lockconfig.ImagesLock, prefix string) (*RelocationResult, error) { + + count, err := relocateCarvelImages(lock.Images, prefix) + if err != nil { + return nil, fmt.Errorf("failed to relocate Carvel images lock file: %v", err) + } + + buff, err := lock.AsBytes() + if err != nil { + return nil, fmt.Errorf("failed to write Images.lock file: %v", err) + } + + return &RelocationResult{Data: buff, Count: count}, nil + +} + +func relocateCarvelImages(images []lockconfig.ImageRef, prefix string) (count int, err error) { + var allErrors error + for i, img := range images { + norm, err := utils.RelocateImageURL(img.Image, prefix, true) + if err != nil { + allErrors = errors.Join(allErrors, err) + continue + } + images[i].Image = norm + count++ + } + return count, allErrors +} + func normalizeRelocateURL(url string) string { ociPrefix := "oci://" // crane gets confused with the oci schema, so we diff --git a/relocator/imagelock.go b/relocator/imagelock.go index b6b53c9..3be83bc 100644 --- a/relocator/imagelock.go +++ b/relocator/imagelock.go @@ -36,7 +36,7 @@ func RelocateLock(lock *imagelock.ImagesLock, prefix string) (*RelocationResult, return &RelocationResult{Data: buff.Bytes(), Count: count}, nil } -// RelocateLockFile reloactes images urls in the provided Images.lock using prefix +// RelocateLockFile relocates images urls in the provided Images.lock using prefix func RelocateLockFile(file string, prefix string) error { lock, err := imagelock.FromYAMLFile(file) if err != nil { diff --git a/testdata/scenarios/custom-chart/.imgpkg/bundle.yml.tmpl b/testdata/scenarios/custom-chart/.imgpkg/bundle.yml.tmpl new file mode 100644 index 0000000..e91eecb --- /dev/null +++ b/testdata/scenarios/custom-chart/.imgpkg/bundle.yml.tmpl @@ -0,0 +1,16 @@ +version: + apiversion: imgpkg.carvel.dev/v1alpha1 + kind: Bundle +metadata: + category: CMS + licenses: Apache-2.0 + name: {{or .Name "WordPress"}} +authors: +{{- range .Authors}} + - name: {{.Name}} + email: {{.Email}} +{{end -}} +websites: +{{- range .Websites}} + - url: {{.URL}} +{{end -}} diff --git a/testdata/scenarios/custom-chart/.imgpkg/images.yml.tmpl b/testdata/scenarios/custom-chart/.imgpkg/images.yml.tmpl new file mode 100644 index 0000000..315f6cb --- /dev/null +++ b/testdata/scenarios/custom-chart/.imgpkg/images.yml.tmpl @@ -0,0 +1,15 @@ +apiVersion: imgpkg.carvel.dev/v1alpha1 +images: +{{- $p := . -}} +{{- range $idx, $elem := .Images}} +{{ $imageParts := split ":" $elem.Image }} +{{ $img := $imageParts._0 }} +{{- range .Digests}} +{{- if eq .Arch "linux/amd64"}} +- annotations: + kbld.carvel.dev/id: {{$p.ServerURL}}/{{$elem.Image}} + image: {{$p.ServerURL}}/{{$img}}@{{.Digest}} +{{- end }} +{{- end}} +{{- end}} +kind: ImagesLock \ No newline at end of file diff --git a/testdata/scenarios/custom-chart/Chart.yaml.tmpl b/testdata/scenarios/custom-chart/Chart.yaml.tmpl index 3aee831..455b080 100644 --- a/testdata/scenarios/custom-chart/Chart.yaml.tmpl +++ b/testdata/scenarios/custom-chart/Chart.yaml.tmpl @@ -16,3 +16,20 @@ dependencies: {{end -}} {{end}} {{end}} +{{if .Authors }} +{{if gt (len .Authors) 0 }} +maintainers: +{{- range .Authors}} + - name: {{.Name}} + email: {{.Email}} +{{end -}} +{{end}} +{{end}} +{{if .Websites }} +{{if gt (len .Websites) 0 }} +sources: +{{- range .Websites}} + - {{.URL}} +{{end -}} +{{end}} +{{end}}