From a12c0b878f188c89804b13b0971d53af5bd9598f Mon Sep 17 00:00:00 2001 From: Guilherme Caulada Date: Fri, 25 Aug 2023 15:26:00 -0300 Subject: [PATCH] Several improvements to package version handling --- cmd/flags.go | 7 +++--- cmd/npm.go | 7 +++--- containers/npm_publish.go | 38 ++++++++++++++++++++++------ containers/opts_grafana.go | 2 +- containers/version.go | 49 +++++++++++++++++++++++++------------ pipelines/docker_publish.go | 3 ++- pipelines/npm_publish.go | 26 ++++++-------------- 7 files changed, 81 insertions(+), 51 deletions(-) diff --git a/cmd/flags.go b/cmd/flags.go index 5ab2667b..ac89d40f 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -34,10 +34,9 @@ var GCPFlags = []cli.Flag{ // NPMFlags are used in commands that need to authenticate with package registries to publish NPM packages var NPMFlags = []cli.Flag{ &cli.StringFlag{ - Name: "registry", - Usage: "The package registry to publish packages", - Required: true, - Value: "registry.npmjs.org", + Name: "registry", + Usage: "The package registry to publish packages", + Value: "registry.npmjs.org", }, &cli.StringFlag{ Name: "token", diff --git a/cmd/npm.go b/cmd/npm.go index 00e329b3..dcfb7843 100644 --- a/cmd/npm.go +++ b/cmd/npm.go @@ -6,9 +6,10 @@ import ( ) var NPMCommand = &cli.Command{ - Name: "npm", - Action: PipelineActionWithPackageInput(pipelines.NPM), - Usage: "Using a grafana.tar.gz as input (ideally one built using the 'package' command), take the npm artifacts and upload them to the destination. This can be used to put Grafana's npm artifacts into a bucket for external use.", + Name: "npm", + Action: PipelineActionWithPackageInput(pipelines.NPM), + Usage: "Using a grafana.tar.gz as input (ideally one built using the 'package' command), take the npm artifacts and upload them to the destination. This can be used to put Grafana's npm artifacts into a bucket for external use.", + Subcommands: []*cli.Command{PublishNPMCommand}, Flags: JoinFlagsWithDefault( PackageInputFlags, PublishFlags, diff --git a/containers/npm_publish.go b/containers/npm_publish.go index 47e699fd..fc71de31 100644 --- a/containers/npm_publish.go +++ b/containers/npm_publish.go @@ -3,24 +3,39 @@ package containers import ( "context" "fmt" + "strings" "dagger.io/dagger" "github.com/grafana/grafana-build/executil" ) // PublishNPM publishes a npm package to the given destination. -func PublishNPM(ctx context.Context, d *dagger.Client, pkg *dagger.File, name string, version string, opts *NPMOpts) (string, error) { - isLatestStable, err := IsLatest(ctx, d, executil.Stable, version) +func PublishNPM(ctx context.Context, d *dagger.Client, pkg *dagger.File, opts *NPMOpts) (string, error) { + src := ExtractedArchive(d, pkg, "pkg.tgz") + version, err := GetJSONValue(ctx, d, src, "package.json", "version") + name, err := GetJSONValue(ctx, d, src, "package.json", "name") + + isLatestStable, err := IsLatestGrafana(ctx, d, executil.Stable, version) + if err != nil { + return "", err + } + + isLatestPreview, err := IsLatestGrafana(ctx, d, executil.Preview, version) if err != nil { return "", err } - isLatestPreview, err := IsLatest(ctx, d, executil.Preview, version) + latestStable, err := GetLatestGrafanaVersion(ctx, d, executil.Stable) if err != nil { return "", err } - latest, err := GetLatestVersion(ctx, d, executil.Stable) + latestPreview, err := GetLatestGrafanaVersion(ctx, d, executil.Preview) + if err != nil { + return "", err + } + + isLaterThanPreview, err := IsLaterThan(ctx, d, version, latestPreview) if err != nil { return "", err } @@ -36,14 +51,21 @@ func PublishNPM(ctx context.Context, d *dagger.Client, pkg *dagger.File, name st c := d.Container().From(NodeImage("lts")). WithFile("/pkg.tgz", pkg). - WithExec([]string{"npm", "set", fmt.Sprintf("//%s/:_authToken", opts.Registry), opts.Token}). - WithExec([]string{"npm", "publish", "/pkg.tgz", "--registry", opts.Registry, "--tag", tag}) + // Workaround for now (maybe unnecessary?): set a NAME environment variable so that we don't accidentally cache + WithEnvVariable("NAME", name). + WithExec([]string{"npm", "set", fmt.Sprintf("//%s/:_authToken", opts.Registry), opts.Token}) + + out, err := c.WithExec([]string{"npm", "view", name, "versions"}).Stdout(ctx) + if !strings.Contains(out, fmt.Sprintf("'%s'", version)) { + // Publish only if this version is not published already + c = c.WithExec([]string{"npm", "publish", "/pkg.tgz", fmt.Sprintf("--registry https://%s", opts.Registry), "--tag", tag}) + } if !isLatestStable { - c = c.WithExec([]string{"npm", "dist-tag", "add", fmt.Sprintf("%s@%s", name, latest), "latest"}) + c = c.WithExec([]string{"npm", "dist-tag", "add", fmt.Sprintf("%s@%s", name, latestStable), "latest"}) } - if isLatestPreview { + if isLatestPreview || (isLatestStable && isLaterThanPreview) { c = c.WithExec([]string{"npm", "dist-tag", "add", fmt.Sprintf("%s@%s", name, version), "next"}) } diff --git a/containers/opts_grafana.go b/containers/opts_grafana.go index 18ba05ec..a75a651d 100644 --- a/containers/opts_grafana.go +++ b/containers/opts_grafana.go @@ -113,7 +113,7 @@ func (g *GrafanaOpts) DetectVersion(ctx context.Context, client *dagger.Client, } log.Println("Version not provided; getting version from package.json...") - v, err := GetPackageJSONVersion(ctx, client, grafanaDir) + v, err := GetJSONValue(ctx, client, grafanaDir, "package.json", "version") if err != nil { return "", err } diff --git a/containers/version.go b/containers/version.go index 4c8d4c24..164f302e 100644 --- a/containers/version.go +++ b/containers/version.go @@ -9,13 +9,13 @@ import ( "github.com/grafana/grafana-build/executil" ) -// GetPackageJSONVersion gets the "version" field from package.json in the 'src' directory. -func GetPackageJSONVersion(ctx context.Context, d *dagger.Client, src *dagger.Directory) (string, error) { +// GetJSONValue gets the value of a JSON field from a JSON file in the 'src' directory. +func GetJSONValue(ctx context.Context, d *dagger.Client, src *dagger.Directory, file string, field string) (string, error) { c := d.Container().From("alpine"). WithExec([]string{"apk", "--update", "add", "jq"}). WithMountedDirectory("/src", src). WithWorkdir("/src"). - WithExec([]string{"/bin/sh", "-c", "cat package.json | jq -r .version"}) + WithExec([]string{"/bin/sh", "-c", fmt.Sprintf("cat %s | jq -r .%s", file, field)}) if stdout, err := c.Stdout(ctx); err == nil { return strings.TrimSpace(stdout), nil @@ -24,36 +24,55 @@ func GetPackageJSONVersion(ctx context.Context, d *dagger.Client, src *dagger.Di return c.Stderr(ctx) } -// GetLatestVersion gets the "version" field from https://grafana.com/api/grafana/versions/. -func GetLatestVersion(ctx context.Context, d *dagger.Client, channel executil.VersionChannel) (string, error) { +// GetLatestGrafanaVersion gets the "version" field from https://grafana.com/api/grafana/versions/. +func GetLatestGrafanaVersion(ctx context.Context, d *dagger.Client, channel executil.VersionChannel) (string, error) { c := d.Container().From("alpine"). - WithExec([]string{"apk", "--update", "add", "jq"}). - WithExec([]string{"/bin/sh", "-c", fmt.Sprintf("curl https://grafana.com/api/grafana/versions/%s | jq -r .version", channel)}) + WithExec([]string{"apk", "--update", "add", "jq", "curl"}). + WithExec([]string{"/bin/sh", "-c", fmt.Sprintf("curl -s https://grafana.com/api/grafana/versions/%s | jq -r .version", channel)}) if stdout, err := c.Stdout(ctx); err == nil { - return strings.TrimSpace(stdout), nil + out := strings.TrimSpace(stdout) + if out == "" { + return out, fmt.Errorf("failed to retrieve grafana version from grafana.com") + } + return out, nil } return c.Stderr(ctx) } -// IsLatest compares versions and returns true if the version provided is grater or equal the latest version on the channel. -func IsLatest(ctx context.Context, d *dagger.Client, channel executil.VersionChannel, version string) (bool, error) { +// IsLatestGrafana compares versions and returns true if the version provided is grater or equal the latest version of Grafana on the channel. +func IsLatestGrafana(ctx context.Context, d *dagger.Client, channel executil.VersionChannel, version string) (bool, error) { if versionChannel := executil.GetVersionChannel(version); versionChannel != channel { return false, nil } - latest, err := GetLatestVersion(ctx, d, channel) + latestGrafana, err := GetLatestGrafanaVersion(ctx, d, channel) if err != nil { return false, err } + return IsLaterThan(ctx, d, version, latestGrafana) +} + +// GetLatestVersion compares versions and returns the latest version provided in the slice. +func GetLatestVersion(ctx context.Context, d *dagger.Client, versions []string) (string, error) { c := d.Container().From("alpine"). - WithExec([]string{"/bin/sh", "-c", fmt.Sprintf("echo -e '%s\n%s' | sort -V | tail -1", latest, version)}) + WithExec([]string{"/bin/sh", "-c", fmt.Sprintf("echo -e '%s' | sort -V | tail -1", strings.Join(versions, "\\n"))}) - if stdout, err := c.Stdout(ctx); err == nil { - return strings.TrimSpace(stdout) == version, nil + stdout, err := c.Stdout(ctx) + if err != nil { + return "", err } - return false, nil + return strings.TrimSpace(stdout), nil +} + +// IsLaterThan compares versions and returns true if v1 is later than v2 +func IsLaterThan(ctx context.Context, d *dagger.Client, v1 string, v2 string) (bool, error) { + latest, err := GetLatestVersion(ctx, d, []string{v1, v2}) + if err != nil { + return false, err + } + return latest == v1, nil } diff --git a/pipelines/docker_publish.go b/pipelines/docker_publish.go index 7746bb30..1e81144c 100644 --- a/pipelines/docker_publish.go +++ b/pipelines/docker_publish.go @@ -8,6 +8,7 @@ import ( "dagger.io/dagger" "github.com/grafana/grafana-build/containers" + "github.com/grafana/grafana-build/executil" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" ) @@ -60,7 +61,7 @@ func PublishDocker(ctx context.Context, d *dagger.Client, args PipelineArgs) err base = BaseImageUbuntu } - isLatest, err := containers.IsLatest(ctx, d, "stable", tarOpts.Version) + isLatest, err := containers.IsLatestGrafana(ctx, d, executil.Stable, tarOpts.Version) if err != nil { return err } diff --git a/pipelines/npm_publish.go b/pipelines/npm_publish.go index 465036b1..f88dd983 100644 --- a/pipelines/npm_publish.go +++ b/pipelines/npm_publish.go @@ -4,8 +4,6 @@ import ( "context" "fmt" "log" - "path/filepath" - "strings" "dagger.io/dagger" "github.com/grafana/grafana-build/containers" @@ -13,15 +11,6 @@ import ( "golang.org/x/sync/semaphore" ) -func NPMPackageName(path string) (string, string) { - filename := filepath.Base(path) - name := WithoutExt(filename) - parts := strings.Split(name, "-") - packageName := strings.Join([]string{parts[0], parts[1]}, "/") - packageVersion := strings.Join(parts[2:], "-") - return packageName, packageVersion -} - func PublishNPM(ctx context.Context, d *dagger.Client, args PipelineArgs) error { var ( opts = args.NPMOpts @@ -57,21 +46,20 @@ func PublishNPM(ctx context.Context, d *dagger.Client, args PipelineArgs) error func PublishNPMFunc(ctx context.Context, sm *semaphore.Weighted, d *dagger.Client, pkg *dagger.File, path string, opts *containers.NPMOpts) func() error { return func() error { - name, version := NPMPackageName(path) - log.Printf("[%s@%s] Attempting to publish package", name, version) - log.Printf("[%s@%s] Acquiring semaphore", name, version) + log.Printf("[%s] Attempting to publish package", path) + log.Printf("[%s] Acquiring semaphore", path) if err := sm.Acquire(ctx, 1); err != nil { return fmt.Errorf("failed to acquire semaphore: %w", err) } defer sm.Release(1) - log.Printf("[%s@%s] Acquired semaphore", name, version) + log.Printf("[%s] Acquired semaphore", path) - log.Printf("[%s@%s] Publishing package", name, version) - out, err := containers.PublishNPM(ctx, d, pkg, name, version, opts) + log.Printf("[%s] Publishing package", path) + out, err := containers.PublishNPM(ctx, d, pkg, opts) if err != nil { - return fmt.Errorf("[%s] error: %w", name, err) + return fmt.Errorf("[%s] error: %w", path, err) } - log.Printf("[%s@%s] Done publishing package", name, version) + log.Printf("[%s] Done publishing package", path) fmt.Fprintln(Stdout, out) return nil