From 142d2dc7c7cff37f0d8573e2809b0c78cefe2967 Mon Sep 17 00:00:00 2001 From: Guilherme Caulada Date: Mon, 2 Oct 2023 18:19:31 -0300 Subject: [PATCH] Add gcom publish command (#191) --- cmd/flags.go | 18 ++++ cmd/gcom.go | 11 ++ cmd/gcom_publish.go | 18 ++++ cmd/main.go | 1 + containers/gcom_publish.go | 56 ++++++++++ containers/opts_gcom.go | 18 ++++ containers/publish.go | 7 +- containers/sha256.go | 16 +++ pipelines/gcom_publish.go | 128 +++++++++++++++++++++++ pipelines/pipeline_args.go | 4 + scripts/drone_publish_nightly_grafana.sh | 9 +- 11 files changed, 279 insertions(+), 7 deletions(-) create mode 100644 cmd/gcom.go create mode 100644 cmd/gcom_publish.go create mode 100644 containers/gcom_publish.go create mode 100644 containers/opts_gcom.go create mode 100644 containers/sha256.go create mode 100644 pipelines/gcom_publish.go diff --git a/cmd/flags.go b/cmd/flags.go index bdee9624..3dd19556 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -324,6 +324,24 @@ var DefaultFlags = []cli.Flag{ }, } +var GCOMFlags = []cli.Flag{ + &cli.StringFlag{ + Name: "url", + Usage: "URL used in requests to grafana.com", + Value: "https://grafana.com", + }, + &cli.StringFlag{ + Name: "api-key", + Usage: "API Key used in requests to grafana.com", + Required: true, + }, + &cli.StringFlag{ + Name: "download-url", + Usage: "URL used to download packages from grafana.com", + Required: true, + }, +} + // JoinFlags combines several slices of flags into one slice of flags. func JoinFlags(f ...[]cli.Flag) []cli.Flag { flags := []cli.Flag{} diff --git a/cmd/gcom.go b/cmd/gcom.go new file mode 100644 index 00000000..97684109 --- /dev/null +++ b/cmd/gcom.go @@ -0,0 +1,11 @@ +package main + +import ( + "github.com/urfave/cli/v2" +) + +var GCOMCommand = &cli.Command{ + Name: "gcom", + Description: "Executes requests to grafana.com", + Subcommands: []*cli.Command{GCOMPublishCommand}, +} diff --git a/cmd/gcom_publish.go b/cmd/gcom_publish.go new file mode 100644 index 00000000..36bd6257 --- /dev/null +++ b/cmd/gcom_publish.go @@ -0,0 +1,18 @@ +package main + +import ( + "github.com/grafana/grafana-build/pipelines" + "github.com/urfave/cli/v2" +) + +var GCOMPublishCommand = &cli.Command{ + Name: "publish", + Action: PipelineActionWithPackageInput(pipelines.PublishGCOM), + Description: "Publishes a grafana.tar.gz (ideally one built using the 'package' command) to grafana.com (--destination will be the download path)", + Flags: JoinFlagsWithDefault( + GCOMFlags, + PackageInputFlags, + PublishFlags, + ConcurrencyFlags, + ), +} diff --git a/cmd/main.go b/cmd/main.go index bec5bd3f..e649d63b 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -29,6 +29,7 @@ var app = &cli.App{ ProImageCommand, StorybookCommand, NPMCommand, + GCOMCommand, }, } diff --git a/containers/gcom_publish.go b/containers/gcom_publish.go new file mode 100644 index 00000000..38205866 --- /dev/null +++ b/containers/gcom_publish.go @@ -0,0 +1,56 @@ +package containers + +import ( + "context" + "encoding/json" + "fmt" + + "dagger.io/dagger" +) + +type GCOMVersionPayload struct { + Version string `json:"version"` // "10.0.3" + ReleaseDate string `json:"releaseDate"` // "2023-07-26T08:20:16.628278891Z" + Stable bool `json:"stable"` // true + Beta bool `json:"beta"` // false + Nightly bool `json:"nightly"` // false + WhatsNewURL string `json:"whatsNewUrl"` // "https://grafana.com/docs/grafana/next/whatsnew/whats-new-in-v10-0/" + ReleaseNotesURL string `json:"releaseNotesUrl"` // "https://grafana.com/docs/grafana/next/release-notes/" +} + +type GCOMPackagePayload struct { + OS string `json:"os"` // "deb" + URL string `json:"url"` // "https://dl.grafana.com/oss/release/grafana_10.0.3_arm64.deb" + Sha256 string `json:"sha256"` // "78a718816dd556198cfa3007dd594aaf1d80886decae8c4bd0f615bd3f118279\n" + Arch string `json:"arch"` // "arm64" +} + +// PublishGCOM publishes a package to grafana.com. +func PublishGCOM(ctx context.Context, d *dagger.Client, versionPayload *GCOMVersionPayload, packagePayload *GCOMPackagePayload, opts *GCOMOpts) error { + versionApiUrl := fmt.Sprintf("%s/api/grafana/versions", opts.URL) + packagesApiUrl := fmt.Sprintf("%s/api/grafana/versions/%s/packages", opts.URL, versionPayload.Version) + + jsonVersionPayload, err := json.Marshal(versionPayload) + if err != nil { + return err + } + + jsonPackagePayload, err := json.Marshal(packagePayload) + if err != nil { + return err + } + + apiKeySecret := d.SetSecret("gcom-api-key", opts.ApiKey) + + _, err = d.Container().From("alpine/curl"). + WithSecretVariable("GCOM_API_KEY", apiKeySecret). + WithExec([]string{"/bin/sh", "-c", fmt.Sprintf(`curl -H "Content-Type: application/json" -H "Authorization: Bearer $GCOM_API_KEY" -d '%s' %s`, string(jsonVersionPayload), versionApiUrl)}). + WithExec([]string{"/bin/sh", "-c", fmt.Sprintf(`curl -H "Content-Type: application/json" -H "Authorization: Bearer $GCOM_API_KEY" -d '%s' %s`, string(jsonPackagePayload), packagesApiUrl)}). + Sync(ctx) + + if err != nil { + return err + } + + return nil +} diff --git a/containers/opts_gcom.go b/containers/opts_gcom.go new file mode 100644 index 00000000..9536685d --- /dev/null +++ b/containers/opts_gcom.go @@ -0,0 +1,18 @@ +package containers + +import "github.com/grafana/grafana-build/cliutil" + +// GCOMOpts are options used when making requests to grafana.com. +type GCOMOpts struct { + URL string + ApiKey string + DownloadURL string +} + +func GCOMOptsFromFlags(c cliutil.CLIContext) *GCOMOpts { + return &GCOMOpts{ + URL: c.String("url"), + ApiKey: c.String("api-key"), + DownloadURL: c.String("download-url"), + } +} diff --git a/containers/publish.go b/containers/publish.go index cba02a4d..696b9d14 100644 --- a/containers/publish.go +++ b/containers/publish.go @@ -80,12 +80,7 @@ func PublishFile(ctx context.Context, d *dagger.Client, opts *PublishFileOpts) ( if publishOpts.Checksum { name := destination + ".sha256" log.Println("Checksum is enabled, creating checksum", name) - files[name] = d.Container(). - From("busybox"). - WithEnvVariable("DESTINATION", destination). - WithFile("/src/file", file). - WithExec([]string{"/bin/sh", "-c", "sha256sum /src/file | awk '{print $1}' > /src/file.sha256"}). - File("/src/file.sha256") + files[name] = Sha256(d, file) } for dst, f := range files { diff --git a/containers/sha256.go b/containers/sha256.go new file mode 100644 index 00000000..e5b5a7d4 --- /dev/null +++ b/containers/sha256.go @@ -0,0 +1,16 @@ +package containers + +import ( + "time" + + "dagger.io/dagger" +) + +// Sha256 returns a dagger.File which contains the sha256 for the provided file. +func Sha256(d *dagger.Client, file *dagger.File) *dagger.File { + return d.Container().From("busybox"). + WithEnvVariable("CACHE_DISABLE", time.Now().String()). + WithFile("/src/file", file). + WithExec([]string{"/bin/sh", "-c", "sha256sum /src/file | awk '{print $1}' > /src/file.sha256"}). + File("/src/file.sha256") +} diff --git a/pipelines/gcom_publish.go b/pipelines/gcom_publish.go new file mode 100644 index 00000000..2c1e9fd0 --- /dev/null +++ b/pipelines/gcom_publish.go @@ -0,0 +1,128 @@ +package pipelines + +import ( + "context" + "fmt" + "log" + "path/filepath" + "strings" + "time" + + "dagger.io/dagger" + "github.com/grafana/grafana-build/containers" + "github.com/grafana/grafana-build/executil" + "golang.org/x/sync/errgroup" + "golang.org/x/sync/semaphore" +) + +func VersionPayloadFromFileName(name string) *containers.GCOMVersionPayload { + var ( + opts = TarOptsFromFileName(name) + splitVersion = strings.Split(opts.Version, ".") + stable = true + nightly = false + beta = false + ) + + if strings.Contains(opts.Version, "-") { + stable = false + beta = true + } + if strings.Contains(opts.Version, "nightly") { + beta = false + nightly = true + } + + return &containers.GCOMVersionPayload{ + Version: opts.Version, + ReleaseDate: time.Now().Format(time.RFC3339Nano), + Stable: stable, + Beta: beta, + Nightly: nightly, + WhatsNewURL: fmt.Sprintf("https://grafana.com/docs/grafana/next/whatsnew/whats-new-in-v%s-%s/", splitVersion[0], splitVersion[1]), + ReleaseNotesURL: "https://grafana.com/docs/grafana/next/release-notes/", + } +} + +func PackagePayloadFromFile(ctx context.Context, d *dagger.Client, name string, file *dagger.File, opts *containers.GCOMOpts) (*containers.GCOMPackagePayload, error) { + tarOpts := TarOptsFromFileName(name) + ext := filepath.Ext(name) + os, _ := executil.OSAndArch(tarOpts.Distro) + arch := strings.ReplaceAll(executil.FullArch(tarOpts.Distro), "/", "") + + if os == "windows" { + os = "win" + } + + if ext == ".deb" { + os = "deb" + } + if ext == ".rpm" { + os = "rhel" + } + if ext == ".exe" { + os = "win-installer" + } + + sha256, err := containers.Sha256(d, file).Contents(ctx) + if err != nil { + return nil, err + } + + return &containers.GCOMPackagePayload{ + OS: os, + URL: fmt.Sprintf("%s/%s", opts.DownloadURL, name), + Sha256: sha256, + Arch: arch, + }, nil +} + +func PublishGCOM(ctx context.Context, d *dagger.Client, args PipelineArgs) error { + var ( + opts = args.GCOMOpts + wg = &errgroup.Group{} + sm = semaphore.NewWeighted(args.ConcurrencyOpts.Parallel) + ) + + packages, err := containers.GetPackages(ctx, d, args.PackageInputOpts, args.GCPOpts) + if err != nil { + return err + } + + // Extract the package(s) + for i, name := range args.PackageInputOpts.Packages { + wg.Go(PublishGCOMFunc(ctx, sm, d, opts, name, packages[i])) + } + return wg.Wait() +} + +func PublishGCOMFunc(ctx context.Context, sm *semaphore.Weighted, d *dagger.Client, opts *containers.GCOMOpts, path string, file *dagger.File) func() error { + return func() error { + name := filepath.Base(path) + log.Printf("[%s] Attempting to publish package", name) + log.Printf("[%s] Acquiring semaphore", name) + if err := sm.Acquire(ctx, 1); err != nil { + return fmt.Errorf("failed to acquire semaphore: %w", err) + } + defer sm.Release(1) + log.Printf("[%s] Acquired semaphore", name) + + log.Printf("[%s] Building version payload", name) + versionPayload := VersionPayloadFromFileName(name) + + log.Printf("[%s] Building package payload", name) + packagePayload, err := PackagePayloadFromFile(ctx, d, name, file, opts) + if err != nil { + return fmt.Errorf("[%s] error: %w", name, err) + } + + log.Printf("[%s] Publishing package", name) + err = containers.PublishGCOM(ctx, d, versionPayload, packagePayload, opts) + if err != nil { + return fmt.Errorf("[%s] error: %w", name, err) + } + + log.Printf("[%s] Done publishing package", name) + return nil + } +} diff --git a/pipelines/pipeline_args.go b/pipelines/pipeline_args.go index 76d68374..f8f82ade 100644 --- a/pipelines/pipeline_args.go +++ b/pipelines/pipeline_args.go @@ -61,6 +61,9 @@ type PipelineArgs struct { // NPMOpts will be populated if NPMFlags are enabled on the current sub-command. NPMOpts *containers.NPMOpts + + // GCOMOpts will be populated if GCOMFlags are enabled on the current sub-command. + GCOMOpts *containers.GCOMOpts } // PipelineArgsFromContext populates a pipelines.PipelineArgs from a CLI context. @@ -89,6 +92,7 @@ func PipelineArgsFromContext(ctx context.Context, c cliutil.CLIContext) (Pipelin ConcurrencyOpts: ConcurrencyOptsFromFlags(c), ProImageOpts: containers.ProImageOptsFromFlags(c), NPMOpts: containers.NPMOptsFromFlags(c), + GCOMOpts: containers.GCOMOptsFromFlags(c), }, nil } diff --git a/scripts/drone_publish_nightly_grafana.sh b/scripts/drone_publish_nightly_grafana.sh index e4576ad4..ea6bd543 100755 --- a/scripts/drone_publish_nightly_grafana.sh +++ b/scripts/drone_publish_nightly_grafana.sh @@ -41,4 +41,11 @@ dagger run --silent go run ./cmd cdn \ dagger run --silent go run ./cmd npm publish \ $(find $local_dir | grep tar.gz | grep linux | grep amd64 | grep -v sha256 | grep -v docker | awk '{print "--package=file://"$0}') \ --token=${NPM_TOKEN} \ - --tag="nightly" \ No newline at end of file + --tag="nightly" || true + +# Publish packages to grafana.com +dagger run --silent go run ./cmd gcom publish \ + $(find $local_dir | grep -e .rpm -e .tar.gz -e .exe -e .zip -e .deb | grep -v sha256 | awk '{print "--package=file://"$0}') \ + --api-key=${GCOM_API_KEY} \ + --url="https://grafana-dev.com" \ + --download-url="https://dl.grafana.com/oss/release" \ No newline at end of file