Add periodic check for metric client health (#2355)

* Add periodic check for metric client health

* Improve logging

src/app/backend/dashboard.go

@@ -54,6 +54,7 @@ var (
 	argTokenTTL           = pflag.Int("token-ttl", int(authApi.DefaultTokenTTL), "Expiration time (in seconds) of JWE tokens generated by dashboard. Default: 15 min. 0 - never expires")
 	argAuthenticationMode = pflag.StringSlice("authentication-mode", []string{authApi.Token.String()}, "Enables authentication options that will be reflected on login screen. Supported values: token, basic. Default: token."+
 		"Note that basic option should only be used if apiserver has '--authorization-mode=ABAC' and '--basic-auth-file' flags set.")
+	argMetricClientCheckPeriod = pflag.Int("metric-client-check-period", 30, "Time in seconds that defines how often configured metric client health check should be run. Default: 30 seconds.")
 )
 
 func main() {
@@ -84,12 +85,8 @@ func main() {
 
 	// Init integrations
 	integrationManager := integration.NewIntegrationManager(clientManager)
-	err = integrationManager.Metric().
-		ConfigureHeapster(*argHeapsterHost).
-		Enable(integrationapi.HeapsterIntegrationID)
-	if err != nil {
-		log.Printf("Could not enable metric client: %s. Continuing.", err)
-	}
+	integrationManager.Metric().ConfigureHeapster(*argHeapsterHost).
+		EnableWithRetry(integrationapi.HeapsterIntegrationID, time.Duration(*argMetricClientCheckPeriod))
 
 	apiHandler, err := handler.CreateHTTPAPIHandler(
 		integrationManager,

src/app/backend/integration/metric/heapster/restclient.go

@@ -15,8 +15,6 @@
 package heapster
 
 import (
-	"log"
-
 	"k8s.io/client-go/rest"
 )
 
@@ -60,11 +58,6 @@ func (self inClusterHeapsterClient) HealthCheck() error {
 		Name("heapster").
 		Suffix("/healthz").
 		DoRaw()
-
-	if err == nil {
-		log.Print("Successful initial request to heapster")
-	}
-
 	return err
 }
 
@@ -83,9 +76,5 @@ func (c remoteHeapsterClient) Get(path string) RequestInterface {
 // Returns nil if connection to application can be established, error object otherwise.
 func (self remoteHeapsterClient) HealthCheck() error {
 	_, err := self.Get("healthz").AbsPath("/").DoRaw()
-	if err == nil {
-		log.Print("Successful initial request to heapster")
-	}
-
 	return err
 }

src/app/backend/integration/metric/manager.go

@@ -16,12 +16,14 @@ package metric
 
 import (
 	"fmt"
+	"log"
+	"time"
 
-	"github.com/emicklei/go-restful/log"
 	"github.com/kubernetes/dashboard/src/app/backend/client"
 	integrationapi "github.com/kubernetes/dashboard/src/app/backend/integration/api"
 	metricapi "github.com/kubernetes/dashboard/src/app/backend/integration/metric/api"
 	"github.com/kubernetes/dashboard/src/app/backend/integration/metric/heapster"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 // MetricManager is responsible for management of all integrated applications related to metrics.
@@ -33,6 +35,9 @@ type MetricManager interface {
 	// Enable is responsible for switching active client if given integration application id
 	// is found and related application is healthy (we can connect to it).
 	Enable(integrationapi.IntegrationID) error
+	// EnableWithRetry works similar to enable. It runs in a separate thread and tries to enable integration with given
+	// id every 'period' seconds.
+	EnableWithRetry(id integrationapi.IntegrationID, period time.Duration)
 	// List returns list of available metric related integrations.
 	List() []integrationapi.Integration
 	// ConfigureHeapster configures and adds heapster to clients list.
@@ -76,6 +81,29 @@ func (self *metricManager) Enable(id integrationapi.IntegrationID) error {
 	return nil
 }
 
+// EnableWithRetry implements metric manager interface. See MetricManager for more information.
+func (self *metricManager) EnableWithRetry(id integrationapi.IntegrationID, period time.Duration) {
+	go wait.Forever(func() {
+		metricClient, exists := self.clients[id]
+		if !exists {
+			log.Printf("Metric client with given id %s does not exist.", id)
+			return
+		}
+
+		err := metricClient.HealthCheck()
+		if err != nil {
+			self.active = nil
+			log.Printf("Metric client health check failed: %s. Retrying in %d seconds.", err, period)
+			return
+		}
+
+		if self.active == nil {
+			log.Printf("Successful request to %s", id)
+			self.active = metricClient
+		}
+	}, period*time.Second)
+}
+
 // List implements metric manager interface. See MetricManager for more information.
 func (self *metricManager) List() []integrationapi.Integration {
 	result := make([]integrationapi.Integration, 0)

src/app/backend/resource/dataselect/dataselect.go

@@ -15,11 +15,10 @@
 package dataselect
 
 import (
-	"sort"
-
 	"errors"
+	"log"
+	"sort"
 
-	"github.com/emicklei/go-restful/log"
 	metricapi "github.com/kubernetes/dashboard/src/app/backend/integration/metric/api"
 )