Add gcom publish command (#191)

cmd/flags.go

@@ -324,6 +324,24 @@ var DefaultFlags = []cli.Flag{
 	},
 }
 
+var GCOMFlags = []cli.Flag{
+	&cli.StringFlag{
+		Name:  "url",
+		Usage: "URL used in requests to grafana.com",
+		Value: "https://grafana.com",
+	},
+	&cli.StringFlag{
+		Name:     "api-key",
+		Usage:    "API Key used in requests to grafana.com",
+		Required: true,
+	},
+	&cli.StringFlag{
+		Name:     "download-url",
+		Usage:    "URL used to download packages from grafana.com",
+		Required: true,
+	},
+}
+
 // JoinFlags combines several slices of flags into one slice of flags.
 func JoinFlags(f ...[]cli.Flag) []cli.Flag {
 	flags := []cli.Flag{}

cmd/gcom.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+var GCOMCommand = &cli.Command{
+	Name:        "gcom",
+	Description: "Executes requests to grafana.com",
+	Subcommands: []*cli.Command{GCOMPublishCommand},
+}

cmd/gcom_publish.go

@@ -0,0 +1,18 @@
+package main
+
+import (
+	"github.com/grafana/grafana-build/pipelines"
+	"github.com/urfave/cli/v2"
+)
+
+var GCOMPublishCommand = &cli.Command{
+	Name:        "publish",
+	Action:      PipelineActionWithPackageInput(pipelines.PublishGCOM),
+	Description: "Publishes a grafana.tar.gz (ideally one built using the 'package' command) to grafana.com (--destination will be the download path)",
+	Flags: JoinFlagsWithDefault(
+		GCOMFlags,
+		PackageInputFlags,
+		PublishFlags,
+		ConcurrencyFlags,
+	),
+}

cmd/main.go

@@ -29,6 +29,7 @@ var app = &cli.App{
 		ProImageCommand,
 		StorybookCommand,
 		NPMCommand,
+		GCOMCommand,
 	},
 }
 

containers/gcom_publish.go

@@ -0,0 +1,56 @@
+package containers
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"dagger.io/dagger"
+)
+
+type GCOMVersionPayload struct {
+	Version         string `json:"version"`         // "10.0.3"
+	ReleaseDate     string `json:"releaseDate"`     // "2023-07-26T08:20:16.628278891Z"
+	Stable          bool   `json:"stable"`          // true
+	Beta            bool   `json:"beta"`            // false
+	Nightly         bool   `json:"nightly"`         // false
+	WhatsNewURL     string `json:"whatsNewUrl"`     // "https://grafana.com/docs/grafana/next/whatsnew/whats-new-in-v10-0/"
+	ReleaseNotesURL string `json:"releaseNotesUrl"` // "https://grafana.com/docs/grafana/next/release-notes/"
+}
+
+type GCOMPackagePayload struct {
+	OS     string `json:"os"`     // "deb"
+	URL    string `json:"url"`    // "https://dl.grafana.com/oss/release/grafana_10.0.3_arm64.deb"
+	Sha256 string `json:"sha256"` // "78a718816dd556198cfa3007dd594aaf1d80886decae8c4bd0f615bd3f118279\n"
+	Arch   string `json:"arch"`   // "arm64"
+}
+
+// PublishGCOM publishes a package to grafana.com.
+func PublishGCOM(ctx context.Context, d *dagger.Client, versionPayload *GCOMVersionPayload, packagePayload *GCOMPackagePayload, opts *GCOMOpts) error {
+	versionApiUrl := fmt.Sprintf("%s/api/grafana/versions", opts.URL)
+	packagesApiUrl := fmt.Sprintf("%s/api/grafana/versions/%s/packages", opts.URL, versionPayload.Version)
+
+	jsonVersionPayload, err := json.Marshal(versionPayload)
+	if err != nil {
+		return err
+	}
+
+	jsonPackagePayload, err := json.Marshal(packagePayload)
+	if err != nil {
+		return err
+	}
+
+	apiKeySecret := d.SetSecret("gcom-api-key", opts.ApiKey)
+
+	_, err = d.Container().From("alpine/curl").
+		WithSecretVariable("GCOM_API_KEY", apiKeySecret).
+		WithExec([]string{"/bin/sh", "-c", fmt.Sprintf(`curl -H "Content-Type: application/json" -H "Authorization: Bearer $GCOM_API_KEY" -d '%s' %s`, string(jsonVersionPayload), versionApiUrl)}).
+		WithExec([]string{"/bin/sh", "-c", fmt.Sprintf(`curl -H "Content-Type: application/json" -H "Authorization: Bearer $GCOM_API_KEY" -d '%s' %s`, string(jsonPackagePayload), packagesApiUrl)}).
+		Sync(ctx)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

containers/opts_gcom.go

@@ -0,0 +1,18 @@
+package containers
+
+import "github.com/grafana/grafana-build/cliutil"
+
+// GCOMOpts are options used when making requests to grafana.com.
+type GCOMOpts struct {
+	URL         string
+	ApiKey      string
+	DownloadURL string
+}
+
+func GCOMOptsFromFlags(c cliutil.CLIContext) *GCOMOpts {
+	return &GCOMOpts{
+		URL:         c.String("url"),
+		ApiKey:      c.String("api-key"),
+		DownloadURL: c.String("download-url"),
+	}
+}

containers/publish.go

@@ -80,12 +80,7 @@ func PublishFile(ctx context.Context, d *dagger.Client, opts *PublishFileOpts) (
 	if publishOpts.Checksum {
 		name := destination + ".sha256"
 		log.Println("Checksum is enabled, creating checksum", name)
-		files[name] = d.Container().
-			From("busybox").
-			WithEnvVariable("DESTINATION", destination).
-			WithFile("/src/file", file).
-			WithExec([]string{"/bin/sh", "-c", "sha256sum /src/file | awk '{print $1}' > /src/file.sha256"}).
-			File("/src/file.sha256")
+		files[name] = Sha256(d, file)
 	}
 
 	for dst, f := range files {

containers/sha256.go

@@ -0,0 +1,16 @@
+package containers
+
+import (
+	"time"
+
+	"dagger.io/dagger"
+)
+
+// Sha256 returns a dagger.File which contains the sha256 for the provided file.
+func Sha256(d *dagger.Client, file *dagger.File) *dagger.File {
+	return d.Container().From("busybox").
+		WithEnvVariable("CACHE_DISABLE", time.Now().String()).
+		WithFile("/src/file", file).
+		WithExec([]string{"/bin/sh", "-c", "sha256sum /src/file | awk '{print $1}' > /src/file.sha256"}).
+		File("/src/file.sha256")
+}

pipelines/gcom_publish.go

@@ -0,0 +1,128 @@
+package pipelines
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"dagger.io/dagger"
+	"github.com/grafana/grafana-build/containers"
+	"github.com/grafana/grafana-build/executil"
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/sync/semaphore"
+)
+
+func VersionPayloadFromFileName(name string) *containers.GCOMVersionPayload {
+	var (
+		opts         = TarOptsFromFileName(name)
+		splitVersion = strings.Split(opts.Version, ".")
+		stable       = true
+		nightly      = false
+		beta         = false
+	)
+
+	if strings.Contains(opts.Version, "-") {
+		stable = false
+		beta = true
+	}
+	if strings.Contains(opts.Version, "nightly") {
+		beta = false
+		nightly = true
+	}
+
+	return &containers.GCOMVersionPayload{
+		Version:         opts.Version,
+		ReleaseDate:     time.Now().Format(time.RFC3339Nano),
+		Stable:          stable,
+		Beta:            beta,
+		Nightly:         nightly,
+		WhatsNewURL:     fmt.Sprintf("https://grafana.com/docs/grafana/next/whatsnew/whats-new-in-v%s-%s/", splitVersion[0], splitVersion[1]),
+		ReleaseNotesURL: "https://grafana.com/docs/grafana/next/release-notes/",
+	}
+}
+
+func PackagePayloadFromFile(ctx context.Context, d *dagger.Client, name string, file *dagger.File, opts *containers.GCOMOpts) (*containers.GCOMPackagePayload, error) {
+	tarOpts := TarOptsFromFileName(name)
+	ext := filepath.Ext(name)
+	os, _ := executil.OSAndArch(tarOpts.Distro)
+	arch := strings.ReplaceAll(executil.FullArch(tarOpts.Distro), "/", "")
+
+	if os == "windows" {
+		os = "win"
+	}
+
+	if ext == ".deb" {
+		os = "deb"
+	}
+	if ext == ".rpm" {
+		os = "rhel"
+	}
+	if ext == ".exe" {
+		os = "win-installer"
+	}
+
+	sha256, err := containers.Sha256(d, file).Contents(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return &containers.GCOMPackagePayload{
+		OS:     os,
+		URL:    fmt.Sprintf("%s/%s", opts.DownloadURL, name),
+		Sha256: sha256,
+		Arch:   arch,
+	}, nil
+}
+
+func PublishGCOM(ctx context.Context, d *dagger.Client, args PipelineArgs) error {
+	var (
+		opts = args.GCOMOpts
+		wg   = &errgroup.Group{}
+		sm   = semaphore.NewWeighted(args.ConcurrencyOpts.Parallel)
+	)
+
+	packages, err := containers.GetPackages(ctx, d, args.PackageInputOpts, args.GCPOpts)
+	if err != nil {
+		return err
+	}
+
+	// Extract the package(s)
+	for i, name := range args.PackageInputOpts.Packages {
+		wg.Go(PublishGCOMFunc(ctx, sm, d, opts, name, packages[i]))
+	}
+	return wg.Wait()
+}
+
+func PublishGCOMFunc(ctx context.Context, sm *semaphore.Weighted, d *dagger.Client, opts *containers.GCOMOpts, path string, file *dagger.File) func() error {
+	return func() error {
+		name := filepath.Base(path)
+		log.Printf("[%s] Attempting to publish package", name)
+		log.Printf("[%s] Acquiring semaphore", name)
+		if err := sm.Acquire(ctx, 1); err != nil {
+			return fmt.Errorf("failed to acquire semaphore: %w", err)
+		}
+		defer sm.Release(1)
+		log.Printf("[%s] Acquired semaphore", name)
+
+		log.Printf("[%s] Building version payload", name)
+		versionPayload := VersionPayloadFromFileName(name)
+
+		log.Printf("[%s] Building package payload", name)
+		packagePayload, err := PackagePayloadFromFile(ctx, d, name, file, opts)
+		if err != nil {
+			return fmt.Errorf("[%s] error: %w", name, err)
+		}
+
+		log.Printf("[%s] Publishing package", name)
+		err = containers.PublishGCOM(ctx, d, versionPayload, packagePayload, opts)
+		if err != nil {
+			return fmt.Errorf("[%s] error: %w", name, err)
+		}
+
+		log.Printf("[%s] Done publishing package", name)
+		return nil
+	}
+}

pipelines/pipeline_args.go

@@ -61,6 +61,9 @@ type PipelineArgs struct {
 
 	// NPMOpts will be populated if NPMFlags are enabled on the current sub-command.
 	NPMOpts *containers.NPMOpts
+
+	// GCOMOpts will be populated if GCOMFlags are enabled on the current sub-command.
+	GCOMOpts *containers.GCOMOpts
 }
 
 // PipelineArgsFromContext populates a pipelines.PipelineArgs from a CLI context.
@@ -89,6 +92,7 @@ func PipelineArgsFromContext(ctx context.Context, c cliutil.CLIContext) (Pipelin
 		ConcurrencyOpts:  ConcurrencyOptsFromFlags(c),
 		ProImageOpts:     containers.ProImageOptsFromFlags(c),
 		NPMOpts:          containers.NPMOptsFromFlags(c),
+		GCOMOpts:         containers.GCOMOptsFromFlags(c),
 	}, nil
 }
 

scripts/drone_publish_nightly_grafana.sh

@@ -41,4 +41,11 @@ dagger run --silent go run ./cmd cdn \
 dagger run --silent go run ./cmd npm publish \
   $(find $local_dir | grep tar.gz | grep linux | grep amd64 | grep -v sha256 | grep -v docker | awk '{print "--package=file://"$0}') \
   --token=${NPM_TOKEN} \
-  --tag="nightly"
+  --tag="nightly" || true
+
+# Publish packages to grafana.com
+dagger run --silent go run ./cmd gcom publish \
+  $(find $local_dir | grep -e .rpm -e .tar.gz -e .exe -e .zip -e .deb | grep -v sha256 | awk '{print "--package=file://"$0}') \
+  --api-key=${GCOM_API_KEY} \
+  --url="https://grafana-dev.com" \
+  --download-url="https://dl.grafana.com/oss/release"