[CLD-5883] Add support for Grafana annotations in Elrond releases

README.md

@@ -50,6 +50,10 @@ elrond server
 ```
 tip: if you want to use a remote running Mattermost Cloud server pass the `--provisioner-server` flag
 
+#### Grafana Integration
+
+In case you want to integrate Elrond with Grafana, Elrond supports creation of Grafana annotations covering start and finish of an ring release. You can enable the feature by passing `--grafana-token` and `--grafana-url` flags. You can pass multiple `--grafana-token` flags to add annotations in multiple Grafana orgs. 
+
 
 #### Ring
 The ring reflects a group of Installation Groups that have a similar release purpose. Therefore a ring can have many registered Installation Groups. A number of registered installation groups higher than 1 can help to achieve canary releases. 

cmd/elrond/server.go

@@ -41,6 +41,8 @@ func init() {
 	serverCmd.PersistentFlags().Bool("machine-readable-logs", false, "Output the logs in machine readable format.")
 	serverCmd.PersistentFlags().String("provisioner-server", "http://localhost:8075", "The provisioning server whose API will be queried.")
 	serverCmd.PersistentFlags().Int("provisioner-group-release-timeout", 3600, "The provisioner group release timeout")
+	serverCmd.PersistentFlags().String("grafana-url", "", "The Grafana url for the Grafana intergration.")
+	serverCmd.PersistentFlags().StringSlice("grafana-token", []string{""}, "The grafana token registered with Grafana Org. You can pass multiple entries.")
 
 	// Supervisors
 	serverCmd.PersistentFlags().Int("poll", 30, "The interval in seconds to poll for background work.")
@@ -87,6 +89,16 @@ var serverCmd = &cobra.Command{
 			return errors.Errorf("server requires at least schema %s, current is %s", serverVersion, currentVersion)
 		}
 
+		grafanaURL, _ := command.Flags().GetString("grafana-url")
+		if len(grafanaURL) == 0 {
+			logger.Warn("The grafana-url flag was empty; no Grafana integration configured")
+		}
+
+		grafanaTokens, _ := command.Flags().GetStringSlice("grafana-token")
+		if len(grafanaTokens) == 0 {
+			logger.Warn("The grafana-tokens flag was empty; no Grafana integration configured")
+		}
+
 		ringSupervisor, _ := command.Flags().GetBool("ring-supervisor")
 		installationGroupSupervisor, _ := command.Flags().GetBool("installationgroup-supervisor")
 		if !ringSupervisor && !installationGroupSupervisor {
@@ -111,6 +123,8 @@ var serverCmd = &cobra.Command{
 
 		provisioningParams := elrond.ProvisioningParams{
 			ProvisionerGroupReleaseTimeout: provisionerGroupReleaseTimeout,
+			GrafanaURL:                     grafanaURL,
+			GrafanaTokens:                  grafanaTokens,
 		}
 
 		// Setup the provisioner.

go.mod

@@ -40,6 +40,8 @@ require (
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
+	github.com/grafana/grafana-api-golang-client v0.23.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect

go.sum

@@ -163,6 +163,7 @@ github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b/go.mod h1:Xo4aNUOrJnVruqWQJBtW6+bTBDTniY8yZum5rF3b5jw=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
@@ -256,6 +257,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/grafana/grafana-api-golang-client v0.23.0 h1:Uta0dSkxWYf1D83/E7MRLCG69387FiUc+k9U/35nMhY=
+github.com/grafana/grafana-api-golang-client v0.23.0/go.mod h1:24W29gPe9yl0/3A9X624TPkAOR8DpHno490cPwnkv8E=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
@@ -264,6 +267,8 @@ github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBt
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=

internal/elrond/elrond_grafana.go

@@ -0,0 +1,34 @@
+package elrond
+
+import (
+	"fmt"
+	"time"
+
+	gapi "github.com/grafana/grafana-api-golang-client"
+	"github.com/mattermost/elrond/model"
+	"github.com/pkg/errors"
+)
+
+// AddGrafanaAnnotations adds annotations via the Grafana API
+func (provisioner *ElProvisioner) AddGrafanaAnnotations(text string, ring *model.Ring, installationGroup *model.InstallationGroup, release *model.RingRelease) error {
+	if provisioner.params.GrafanaURL != "" && len(provisioner.params.GrafanaTokens) > 0 {
+		logger := provisioner.logger.WithField("installationgroup", installationGroup.ID)
+		logger.Infof("Adding Grafana release annotations for %s release", installationGroup.ID)
+		for _, token := range provisioner.params.GrafanaTokens {
+			grafanaClient, err := gapi.New(provisioner.params.GrafanaURL, gapi.Config{APIKey: token})
+			if err != nil {
+				return errors.Wrap(err, "failed to initiate Grafana client")
+			}
+			id, err := grafanaClient.NewAnnotation(&gapi.Annotation{
+				Text: text,
+				Tags: []string{fmt.Sprintf("ring:%s", ring.Name), fmt.Sprintf("installation-group:%s", installationGroup.ProvisionerGroupID), fmt.Sprintf("image:%s", release.Image), fmt.Sprintf("version:%s", release.Version), "elrond"},
+				Time: time.Now().UnixNano() / int64(time.Millisecond),
+			})
+			if err != nil {
+				return errors.Wrap(err, "failed to create Grafana annotation")
+			}
+			logger.Infof("Annotation created successfully with ID %d", id)
+		}
+	}
+	return nil
+}

internal/elrond/elrond_provisioner.go

@@ -11,6 +11,8 @@ import (
 // ProvisioningParams represent configuration used during various provisioning operations.
 type ProvisioningParams struct {
 	ProvisionerGroupReleaseTimeout int
+	GrafanaURL                     string
+	GrafanaTokens                  []string
 }
 
 // ElProvisioner provisions release rings.

internal/supervisor/installationgroup.go

@@ -5,6 +5,7 @@
 package supervisor
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/mattermost/elrond/internal/webhook"
@@ -32,6 +33,7 @@ type installationGroupStore interface {
 type installationGroupProvisioner interface {
 	ReleaseInstallationGroup(installationGroup *model.InstallationGroup, release *model.RingRelease) error
 	SoakInstallationGroup(installationGroup *model.InstallationGroup) error
+	AddGrafanaAnnotations(text string, ring *model.Ring, installationGroup *model.InstallationGroup, release *model.RingRelease) error
 }
 
 // InstallationGroupSupervisor finds installation groups pending work and effects the required changes.
@@ -226,6 +228,12 @@ func (s *InstallationGroupSupervisor) releaseInstallationGroup(installationGroup
 		return model.InstallationGroupReleaseFailed
 	}
 
+	err = s.provisioner.AddGrafanaAnnotations(fmt.Sprintf("Initiating release for ring %s and installation group %s", ring.Name, installationGroup.ProvisionerGroupID), ring, installationGroup, release)
+	if err != nil {
+		logger.WithError(err).Error("Failed to add release Grafana Annotations")
+		return model.InstallationGroupReleaseFailed
+	}
+
 	err = s.provisioner.ReleaseInstallationGroup(installationGroup, release)
 	if err != nil {
 		logger.WithError(err).Error("Failed to release installation group")
@@ -253,5 +261,24 @@ func (s *InstallationGroupSupervisor) soakInstallationGroup(installationGroup *m
 	}
 
 	logger.Info("Finished soaking installation group")
+
+	ring, err := s.store.GetRingFromInstallationGroupID(installationGroup.ID)
+	if err != nil {
+		logger.WithError(err).Error("Failed to get the ring from the installation group pending work")
+		return model.InstallationGroupReleaseFailed
+	}
+
+	release, err := s.store.GetRingRelease(ring.DesiredReleaseID)
+	if err != nil {
+		logger.WithError(err).Error("Failed to get the ring release for the installation group pending work")
+		return model.InstallationGroupReleaseFailed
+	}
+
+	err = s.provisioner.AddGrafanaAnnotations(fmt.Sprintf("Release for ring %s and installation group %s is complete", ring.Name, installationGroup.ProvisionerGroupID), ring, installationGroup, release)
+	if err != nil {
+		logger.WithError(err).Error("Failed to add release Grafana Annotations")
+		return model.InstallationGroupReleaseFailed
+	}
+
 	return model.InstallationGroupStable
 }