diff --git a/cmd/flags.go b/cmd/flags.go index b54119e2..b7190427 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -169,6 +169,15 @@ var DockerFlags = []cli.Flag{ Usage: "The Ubuntu image to use as the base image when building the Ubuntu version of the Grafana docker image", Value: "ubuntu:latest", }, + &cli.StringFlag{ + Name: "org", + Usage: "Overrides the organization of the images", + Value: "grafana", + }, + &cli.StringFlag{ + Name: "repo", + Usage: "Overrides the repository of the images", + }, } var DockerPublishFlags = []cli.Flag{ @@ -187,6 +196,15 @@ var DockerPublishFlags = []cli.Flag{ Usage: "Prefix the image name with the registry provided", Value: "docker.io", }, + &cli.StringFlag{ + Name: "org", + Usage: "Overrides the organization of the images", + Value: "grafana", + }, + &cli.StringFlag{ + Name: "repo", + Usage: "Overrides the repository of the images", + }, &cli.BoolFlag{ Name: "latest", Usage: "Tags the published images as latest", diff --git a/containers/opts_docker.go b/containers/opts_docker.go index be4e78d5..dd55cc4c 100644 --- a/containers/opts_docker.go +++ b/containers/opts_docker.go @@ -23,6 +23,12 @@ type DockerOpts struct { // Password is supplied to login to the docker registry when publishing images. Password string + // Org overrides the organization when when publishing images. + Org string + + // Repository overrides the repository when when publishing images. + Repository string + // Latest is supplied to also tag as latest when publishing images. Latest bool } @@ -34,6 +40,8 @@ func DockerOptsFromFlags(c cliutil.CLIContext) *DockerOpts { UbuntuBase: c.String("ubuntu-base"), Username: c.String("username"), Password: c.String("password"), + Org: c.String("org"), + Repository: c.String("repo"), Latest: c.Bool("latest"), } } diff --git a/pipelines/docker.go b/pipelines/docker.go index 902154b0..7ad039b7 100644 --- a/pipelines/docker.go +++ b/pipelines/docker.go @@ -26,19 +26,17 @@ func ImageTag(registry, org, repo, version string) string { // GrafanaImageTag returns the name of the grafana docker image based on the tar package name. // To maintain backwards compatibility, we must keep this the same as it was before. -func GrafanaImageTags(base BaseImage, registry string, opts TarFileOpts) []string { +func GrafanaImageTags(base BaseImage, dockerOpts *containers.DockerOpts, tarOpts TarFileOpts) []string { var ( - org = "grafana" - repos = []string{"grafana-image-tags", "grafana-oss-image-tags"} - version = opts.Version + repos = []string{"grafana-image-tags", "grafana-oss-image-tags"} - edition = opts.Edition - ) + version = tarOpts.Version + edition = tarOpts.Edition - if edition != "" { - // Non-grafana repositories only create images in 1 repository instead of 2. Reason unknown. - repos = []string{fmt.Sprintf("grafana-%s-image-tags", edition)} - } + org = dockerOpts.Org + registry = dockerOpts.Registry + repository = dockerOpts.Repository + ) // For some unknown reason, versions in docker hub do not have a 'v'. // I think this was something that was established a long time ago and just stuck. @@ -48,11 +46,20 @@ func GrafanaImageTags(base BaseImage, registry string, opts TarFileOpts) []strin version += "-ubuntu" } - if opts.Distro != "" { - arch := executil.FullArch(opts.Distro) + if tarOpts.Distro != "" { + arch := executil.FullArch(tarOpts.Distro) version += "-" + strings.ReplaceAll(arch, "/", "") } + if repository != "" { + return []string{ImageTag(registry, org, repository, version)} + } + + if edition != "" { + // Non-grafana repositories only create images in 1 repository instead of 2. Reason unknown. + repos = []string{fmt.Sprintf("grafana-%s-image-tags", edition)} + } + tags := make([]string, len(repos)) for i, repo := range repos { tags[i] = ImageTag(registry, org, repo, version) @@ -88,7 +95,7 @@ func Docker(ctx context.Context, d *dagger.Client, args PipelineArgs) error { for _, base := range bases { var ( platform = executil.Platform(tarOpts.Distro) - tags = GrafanaImageTags(base, opts.Registry, tarOpts) + tags = GrafanaImageTags(base, opts, tarOpts) baseImage = opts.AlpineBase socket = d.Host().UnixSocket("/var/run/docker.sock") ) diff --git a/pipelines/docker_publish.go b/pipelines/docker_publish.go index 6bdb81f1..11355fa6 100644 --- a/pipelines/docker_publish.go +++ b/pipelines/docker_publish.go @@ -52,7 +52,7 @@ func PublishDocker(ctx context.Context, d *dagger.Client, args PipelineArgs) err base = BaseImageUbuntu } - tags := GrafanaImageTags(base, opts.Registry, tarOpts) + tags := GrafanaImageTags(base, opts, tarOpts) for _, tag := range tags { // For each tag we publish an image and add the tag to the list of tags for a specific manifest // Since each package has a maximum of 2 tags, this for loop will only run twice on a worst case scenario diff --git a/pipelines/docker_test.go b/pipelines/docker_test.go index 3cf34a43..90744d10 100644 --- a/pipelines/docker_test.go +++ b/pipelines/docker_test.go @@ -5,6 +5,7 @@ import ( "sort" "testing" + "github.com/grafana/grafana-build/containers" "github.com/grafana/grafana-build/pipelines" ) @@ -17,6 +18,7 @@ func TestImageName(t *testing.T) { Description string Tags []string BaseImage pipelines.BaseImage + DockerOpts *containers.DockerOpts TarOpts pipelines.TarFileOpts } @@ -26,12 +28,16 @@ func TestImageName(t *testing.T) { cases := []tc{ { - Description: "Grafana docker images are created for both the 'docker.io/grafana/grafana-image-tags' and 'docker.io/grafana/grafana-oss-image-tags' repositories. AMD64 docker images have no suffix. Alpine images also have no suffix.", + Description: "Grafana docker images are created for both the 'docker.io/grafana/grafana-image-tags' and 'docker.io/grafana/grafana-oss-image-tags' repositories. Alpine images have no suffix.", TarOpts: pipelines.TarFileOpts{ Edition: "", Distro: "linux/amd64", Version: version, }, + DockerOpts: &containers.DockerOpts{ + Org: "grafana", + Registry: "docker.io", + }, BaseImage: pipelines.BaseImageAlpine, Tags: []string{ "docker.io/grafana/grafana-image-tags:1.2.3-test.1.2.3-amd64", @@ -39,12 +45,16 @@ func TestImageName(t *testing.T) { }, }, { - Description: "Grafana docker images are created for both the 'docker.io/grafana/grafana-image-tags' and 'docker.io/grafana/grafana-oss-image-tags' repositories. ARM64 images have a -arm64 suffix. Alpine images also have no suffix.", + Description: "Grafana docker images are created for both the 'docker.io/grafana/grafana-image-tags' and 'docker.io/grafana/grafana-oss-image-tags' repositories. ARM64 images have a -arm64 suffix. Alpine images have no suffix.", TarOpts: pipelines.TarFileOpts{ Edition: "", Distro: "linux/arm64", Version: version, }, + DockerOpts: &containers.DockerOpts{ + Org: "grafana", + Registry: "docker.io", + }, BaseImage: pipelines.BaseImageAlpine, Tags: []string{ "docker.io/grafana/grafana-image-tags:1.2.3-test.1.2.3-arm64", @@ -52,12 +62,16 @@ func TestImageName(t *testing.T) { }, }, { - Description: "Grafana docker images are created for both the 'docker.io/grafana/grafana-image-tags' and 'docker.io/grafana/grafana-oss-image-tags' repositories. AMD64 docker images have no suffix. Ubuntu images have a '-ubuntu' suffix.", + Description: "Grafana docker images are created for both the 'docker.io/grafana/grafana-image-tags' and 'docker.io/grafana/grafana-oss-image-tags' repositories. Ubuntu images have a '-ubuntu' suffix.", TarOpts: pipelines.TarFileOpts{ Edition: "", Distro: "linux/amd64", Version: version, }, + DockerOpts: &containers.DockerOpts{ + Org: "grafana", + Registry: "docker.io", + }, BaseImage: pipelines.BaseImageUbuntu, Tags: []string{ "docker.io/grafana/grafana-image-tags:1.2.3-test.1.2.3-ubuntu-amd64", @@ -71,6 +85,10 @@ func TestImageName(t *testing.T) { Distro: "linux/arm64", Version: version, }, + DockerOpts: &containers.DockerOpts{ + Org: "grafana", + Registry: "docker.io", + }, BaseImage: pipelines.BaseImageUbuntu, Tags: []string{ "docker.io/grafana/grafana-image-tags:1.2.3-test.1.2.3-ubuntu-arm64", @@ -78,36 +96,48 @@ func TestImageName(t *testing.T) { }, }, { - Description: "Enterprise docker images are created for only the docker.io/grafana/grafana-enterprise-image-tags repository. AMD64 docker images have no suffix. Alpine images also have no suffix.", + Description: "Enterprise docker images are created for only the docker.io/grafana/grafana-enterprise-image-tags repository. Alpine images have no suffix.", TarOpts: pipelines.TarFileOpts{ Edition: "enterprise", Distro: "linux/amd64", Version: version, }, + DockerOpts: &containers.DockerOpts{ + Org: "grafana", + Registry: "docker.io", + }, BaseImage: pipelines.BaseImageAlpine, Tags: []string{ "docker.io/grafana/grafana-enterprise-image-tags:1.2.3-test.1.2.3-amd64", }, }, { - Description: "Enterprise docker images are created for only the docker.io/grafana/grafana-enterprise-image-tags repository. ARM64 images have an -arm64 suffix. Alpine images also have no suffix.", + Description: "Enterprise docker images are created for only the docker.io/grafana/grafana-enterprise-image-tags repository. ARM64 images have an -arm64 suffix. Alpine images have no suffix.", TarOpts: pipelines.TarFileOpts{ Edition: "enterprise", Distro: "linux/arm64", Version: version, }, + DockerOpts: &containers.DockerOpts{ + Org: "grafana", + Registry: "docker.io", + }, BaseImage: pipelines.BaseImageAlpine, Tags: []string{ "docker.io/grafana/grafana-enterprise-image-tags:1.2.3-test.1.2.3-arm64", }, }, { - Description: "Enterprise docker images are created for only the docker.io/grafana/grafana-enterprise-image-tags repository. AMD64 docker images have no suffix. Ubuntu images have a '-ubuntu' suffix.", + Description: "Enterprise docker images are created for only the docker.io/grafana/grafana-enterprise-image-tags repository. Ubuntu images have a '-ubuntu' suffix.", TarOpts: pipelines.TarFileOpts{ Edition: "enterprise", Distro: "linux/amd64", Version: version, }, + DockerOpts: &containers.DockerOpts{ + Org: "grafana", + Registry: "docker.io", + }, BaseImage: pipelines.BaseImageUbuntu, Tags: []string{ "docker.io/grafana/grafana-enterprise-image-tags:1.2.3-test.1.2.3-ubuntu-amd64", @@ -120,17 +150,72 @@ func TestImageName(t *testing.T) { Distro: "linux/arm64", Version: version, }, + DockerOpts: &containers.DockerOpts{ + Org: "grafana", + Registry: "docker.io", + }, BaseImage: pipelines.BaseImageUbuntu, Tags: []string{ "docker.io/grafana/grafana-enterprise-image-tags:1.2.3-test.1.2.3-ubuntu-arm64", }, }, + { + Description: "Grafana docker images are created for both the 'registry.io/org/grafana-image-tags' and 'registry.io/org/grafana-oss-image-tags' repositories. Alpine images have no suffix.", + TarOpts: pipelines.TarFileOpts{ + Edition: "", + Distro: "linux/amd64", + Version: version, + }, + DockerOpts: &containers.DockerOpts{ + Org: "org", + Registry: "registry.io", + }, + BaseImage: pipelines.BaseImageAlpine, + Tags: []string{ + "registry.io/org/grafana-image-tags:1.2.3-test.1.2.3-amd64", + "registry.io/org/grafana-oss-image-tags:1.2.3-test.1.2.3-amd64", + }, + }, + { + Description: "Grafana docker images are created for only the 'registry.io/org/grafana-dev' repository. Alpine images have no suffix.", + TarOpts: pipelines.TarFileOpts{ + Edition: "", + Distro: "linux/amd64", + Version: version, + }, + DockerOpts: &containers.DockerOpts{ + Org: "org", + Registry: "registry.io", + Repository: "grafana-dev", + }, + BaseImage: pipelines.BaseImageAlpine, + Tags: []string{ + "registry.io/org/grafana-dev:1.2.3-test.1.2.3-amd64", + }, + }, + { + Description: "Grafana docker images are created for only the 'registry.io/org/grafana-dev' repository.", + TarOpts: pipelines.TarFileOpts{ + Edition: "", + Distro: "linux/amd64", + Version: version, + }, + DockerOpts: &containers.DockerOpts{ + Org: "org", + Registry: "registry.io", + Repository: "grafana-dev", + }, + BaseImage: pipelines.BaseImageUbuntu, + Tags: []string{ + "registry.io/org/grafana-dev:1.2.3-test.1.2.3-ubuntu-amd64", + }, + }, } for n, test := range cases { t.Run(fmt.Sprintf("[%d / %d] %s", n+1, len(cases), test.Description), func(t *testing.T) { expect := sort.StringSlice(test.Tags) - res := sort.StringSlice(pipelines.GrafanaImageTags(test.BaseImage, "docker.io", test.TarOpts)) + res := sort.StringSlice(pipelines.GrafanaImageTags(test.BaseImage, test.DockerOpts, test.TarOpts)) for i := range expect { e := expect[i] diff --git a/scripts/drone_build_nightly_grafana.sh b/scripts/drone_build_nightly_grafana.sh new file mode 100755 index 00000000..502d5c6b --- /dev/null +++ b/scripts/drone_build_nightly_grafana.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +ver="nightly-${DRONE_COMMIT_SHA:0:8}" +local_dst="file://${DRONE_WORKSPACE}/dist" +set -e + +# This command enables qemu emulators for building Docker images for arm64/armv6/armv7/etc on the host. +docker run --privileged --rm tonistiigi/binfmt --install all + +# Build all of the grafana.tar.gz packages. +echo "Building tar.gz packages..." +dagger run --silent go run ./cmd \ + package \ + --yarn-cache=${YARN_CACHE_FOLDER} \ + --distro=linux/amd64 \ + --distro=linux/arm64 \ + --distro=linux/arm/v6 \ + --distro=linux/arm/v7 \ + --distro=windows/amd64 \ + --distro=darwin/amd64 \ + --checksum \ + --build-id=${DRONE_BUILD_NUMBER} \ + --grafana-dir=${GRAFANA_DIR} \ + --github-token=${GITHUB_TOKEN} \ + --version=${ver} \ + --destination=${local_dst} \ + --gcp-service-account-key-base64=${GCP_KEY_BASE64} >> assets.txt + +# Use the non-windows, non-darwin, non-rpi packages and create deb packages from them. +dagger run --silent go run ./cmd deb \ + $(cat assets.txt | grep tar.gz | grep -v docker | grep -v sha256 | grep -v windows | grep -v darwin | grep -v arm-6 | awk '{print "--package=" $0}') \ + --checksum \ + --destination=${local_dst} \ + --gcp-service-account-key-base64=${GCP_KEY_BASE64} >> assets.txt + +# Use the armv7 package to build the `rpi` specific version. +dagger run --silent go run ./cmd deb \ + $(cat assets.txt | grep tar.gz | grep -v docker | grep -v sha256 | grep -v windows | grep -v darwin | grep arm-7 | awk '{print "--package=" $0}') \ + --name=grafana-rpi \ + --checksum \ + --destination=${local_dst} \ + --gcp-service-account-key-base64=${GCP_KEY_BASE64} >> assets.txt + +# Make rpm installers for all the same Linux distros, and sign them because RPM packages are signed. +dagger run --silent go run ./cmd rpm \ + $(cat assets.txt | grep tar.gz | grep -v docker | grep -v sha256 | grep -v windows | grep -v darwin | grep -v arm-6 | awk '{print "--package=" $0}') \ + --checksum \ + --destination=${local_dst} \ + --gcp-service-account-key-base64=${GCP_KEY_BASE64} \ + --sign=true \ + --gpg-private-key-base64="${GPG_PRIVATE_KEY}" \ + --gpg-public-key-base64="${GPG_PUBLIC_KEY}" \ + --gpg-passphrase="${GPG_PASSPHRASE}" >> assets.txt + +# For Windows we distribute zips and exes +dagger run --silent go run ./cmd zip \ + $(cat assets.txt | grep tar.gz | grep -v docker | grep -v sha256 | grep windows | awk '{print "--package=" $0}') \ + --destination=${local_dst} \ + --gcp-service-account-key-base64=${GCP_KEY_BASE64} \ + --checksum >> assets.txt + +dagger run --silent go run ./cmd windows-installer \ + $(cat assets.txt | grep tar.gz | grep -v docker | grep -v sha256 | grep windows | awk '{print "--package=" $0}') \ + --destination=${local_dst} \ + --gcp-service-account-key-base64=${GCP_KEY_BASE64} \ + --checksum >> assets.txt + +# Build a docker image for all Linux distros except armv6 +dagger run --silent go run ./cmd docker \ + $(cat assets.txt | grep tar.gz | grep -v docker | grep -v sha256 | grep -v windows | grep -v darwin | grep -v arm-6 | awk '{print "--package=" $0}') \ + --checksum \ + --repo="grafana-dev" \ + --ubuntu-base="ubuntu:22.10" \ + --alpine-base="alpine:3.18.0" \ + --destination=${local_dst} \ + --gcp-service-account-key-base64=${GCP_KEY_BASE64} >> assets.txt + +echo "Final list of artifacts:" +cat assets.txt diff --git a/scripts/drone_publish_nightly_grafana.sh b/scripts/drone_publish_nightly_grafana.sh new file mode 100755 index 00000000..4a3d1f5a --- /dev/null +++ b/scripts/drone_publish_nightly_grafana.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -e +local_dir="${DRONE_WORKSPACE}/dist" + +# Publish the docker images present in the bucket +dagger run --silent go run ./cmd docker publish \ + $(find $local_dir | grep docker.tar.gz | grep -v sha256 | grep -v enterprise | awk '{print "--package=file://"$0}') \ + --username=${DOCKER_USERNAME} \ + --password=${DOCKER_PASSWORD} \ + --repo="grafana-dev" \ No newline at end of file