Prometheus metrics

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/cloud-bulldozer/go-commons v1.0.10
 	github.com/openshift/api v0.0.0-20230414095907-0540dde8186d
 	github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c
+	github.com/prometheus/common v0.44.0
 	github.com/satori/go.uuid v1.2.0
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.7.0
@@ -16,6 +17,11 @@ require (
 	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2
 )
 
+require (
+	github.com/montanaflynn/stats v0.7.1 // indirect
+	github.com/prometheus/client_golang v1.15.1 // indirect
+)
+
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/elastic/go-elasticsearch/v7 v7.13.1 // indirect

go.sum

@@ -2,7 +2,9 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/aws/aws-sdk-go v1.42.27/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloud-bulldozer/go-commons v1.0.10 h1:Zjd6sJHhK9MDK7J0IiqMpk9F2WX8NmWRu1U56ytbTS0=
 github.com/cloud-bulldozer/go-commons v1.0.10/go.mod h1:peHBEG4iCC9zKFzA4P16dpINZASWshsCy9T4dVpcBVI=
@@ -69,28 +71,33 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
+github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
 github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU=
 github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc=
 github.com/opensearch-project/opensearch-go v1.1.0 h1:eG5sh3843bbU1itPRjA9QXbxcg8LaZ+DjEzQH9aLN3M=
@@ -102,7 +109,13 @@ github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c/go.mod h1:lFMO
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI=
+github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
+github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
+github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
+github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
@@ -116,6 +129,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=

pkg/config/config.go

@@ -25,9 +25,10 @@ import (
 func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type ConfigDefaulted Config
 	defaultCfg := ConfigDefaulted{
-		Warmup:         false, // Disable warmup by default
-		RequestTimeout: time.Second,
-		Procs:          1,
+		Warmup:            false, // Disable warmup by default
+		RequestTimeout:    time.Second,
+		Procs:             1,
+		PrometheusMetrics: prometheusQueries,
 	}
 	if err := unmarshal(&defaultCfg); err != nil {
 		return err

pkg/config/types.go

@@ -46,4 +46,13 @@ type Config struct {
 	Warmup bool `yaml:"warmup" json:"-"`
 	// RequestTimeout defines the tool request timeout
 	RequestTimeout time.Duration `yaml:"requestTimeout" json:"requestTimeout"`
+	// Prometheus metrics
+	PrometheusMetrics map[string]string `json:"-"`
+}
+
+var prometheusQueries = map[string]string{
+	"avg_cpu_usage_router_pods":           "round(avg(avg_over_time(sum(irate(container_cpu_usage_seconds_total{name!='', namespace='openshift-ingress', pod=~'router-default.+'}[2m])) by (pod)[ELAPSED:]))*100, 1)/100",
+	"avg_memory_usage_router_pods_bytes":  "round(avg(avg_over_time(sum(container_memory_working_set_bytes{name!='', namespace='openshift-ingress', pod=~'router-default.+'}) by (pod)[ELAPSED:]))*100,1)/100",
+	"avg_cpu_usage_router_nodes":          `round(avg(avg_over_time(sum(irate(node_cpu_seconds_total{mode!~'idle|steal'}[2m]) and on (instance) label_replace(kube_pod_info{namespace='openshift-ingress'},'instance', '$1', 'node', '(.+)')) by (instance,mode)[ELAPSED:]))*100,1)/100`,
+	"avg_memory_usage_router_nodes_bytes": `round(avg(avg_over_time(sum(node_memory_MemTotal_bytes-node_memory_MemAvailable_bytes{mode!~'idle|steal'} and on (instance) label_replace(kube_pod_info{namespace='openshift-ingress'},'instance', '$1', 'node', '(.+)')) by (instance,mode)[ELAPSED:]))*100,1)/100`,
 }

pkg/runner/exec.go

@@ -18,12 +18,15 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 
+	"github.com/cloud-bulldozer/go-commons/prometheus"
 	"github.com/cloud-bulldozer/go-commons/version"
 	"github.com/cloud-bulldozer/ingress-perf/pkg/config"
 	"github.com/cloud-bulldozer/ingress-perf/pkg/runner/tools"
+	"github.com/prometheus/common/model"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 	corev1 "k8s.io/api/core/v1"
@@ -35,8 +38,8 @@ import (
 
 var lock = &sync.Mutex{}
 
-func runBenchmark(cfg config.Config, clusterMetadata tools.ClusterMetadata) ([]tools.Result, error) {
-	var aggAvgRps, aggAvgLatency, aggP99Latency float64
+func runBenchmark(cfg config.Config, clusterMetadata tools.ClusterMetadata, p *prometheus.Prometheus) ([]tools.Result, error) {
+	var aggAvgRps, aggAvgLatency, aggP95Latency float64
 	var timeouts, httpErrors int64
 	var benchmarkResult []tools.Result
 	var clientPods []corev1.Pod
@@ -62,12 +65,14 @@ func runBenchmark(cfg config.Config, clusterMetadata tools.ClusterMetadata) ([]t
 	}
 	ts := time.Now().UTC()
 	for i := 1; i <= cfg.Samples; i++ {
+		sampleTs := time.Now().UTC()
 		result := tools.Result{
 			UUID:            cfg.UUID,
 			Sample:          i,
 			Config:          cfg,
 			Timestamp:       ts,
 			ClusterMetadata: clusterMetadata,
+			InfraMetrics:    make(map[string]float64),
 		}
 		result.Config.Tuning = currentTuning // It's useful to index the current tuning patch in the all benchmark's documents
 		log.Infof("Running sample %d/%d: %v", i, cfg.Samples, cfg.Duration)
@@ -98,22 +103,40 @@ func runBenchmark(cfg config.Config, clusterMetadata tools.ClusterMetadata) ([]t
 		genResultSummary(&result)
 		aggAvgRps += result.TotalAvgRps
 		aggAvgLatency += result.AvgLatency
-		aggP99Latency += result.P99Latency
+		aggP95Latency += result.P95Latency
 		timeouts += result.Timeouts
 		httpErrors += result.HTTPErrors
-		log.Infof("%s: Rps=%.0f avgLatency=%.0fms P99Latency=%.0fms", cfg.Termination, result.TotalAvgRps, result.AvgLatency/1e3, result.P99Latency/1e3)
+		elapsed := fmt.Sprintf("%ds", int(time.Since(sampleTs).Seconds()))
+		for field, query := range cfg.PrometheusMetrics {
+			promQuery := strings.ReplaceAll(query, "ELAPSED", elapsed)
+			log.Debugf("Running query: %s", promQuery)
+			value, err := p.Query(promQuery, time.Time{}.UTC())
+			if err != nil {
+				log.Errorf("Query error: %v", err)
+				continue
+			}
+			data, ok := value.(model.Vector)
+			if !ok {
+				log.Errorf("Unsupported result format: %s", value.Type().String())
+				continue
+			}
+			for _, vector := range data {
+				result.InfraMetrics[field] = float64(vector.Value)
+			}
+		}
+		log.Infof("%s: Rps=%.0f avgLatency=%.0fms P95Latency=%.0fms", cfg.Termination, result.TotalAvgRps, result.AvgLatency/1e3, result.P95Latency/1e3)
 		benchmarkResult = append(benchmarkResult, result)
 		if cfg.Delay != 0 {
 			log.Info("Sleeping for ", cfg.Delay)
 			time.Sleep(cfg.Delay)
 		}
 	}
 	validSamples := float64(len(benchmarkResult))
-	log.Infof("Scenario summary %s: Rps=%.0f avgLatency=%.0fms P99Latency=%.0fms timeouts=%d http_errors=%d",
+	log.Infof("Scenario summary %s: Rps=%.0f avgLatency=%.0fms P95Latency=%.0fms timeouts=%d http_errors=%d",
 		cfg.Termination,
 		aggAvgRps/validSamples,
 		aggAvgLatency/validSamples/1e3,
-		aggP99Latency/validSamples/1e3,
+		aggP95Latency/validSamples/1e3,
 		timeouts,
 		httpErrors,
 	)

pkg/runner/runner.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/cloud-bulldozer/go-commons/comparison"
 	"github.com/cloud-bulldozer/go-commons/indexers"
+	"github.com/cloud-bulldozer/go-commons/prometheus"
 
 	ocpmetadata "github.com/cloud-bulldozer/go-commons/ocp-metadata"
 	"github.com/cloud-bulldozer/ingress-perf/pkg/config"
@@ -79,10 +80,21 @@ func Start(uuid, baseUUID, baseIndex string, tolerancy int, indexer *indexers.In
 		return err
 	}
 	clusterMetadata.HAProxyVersion, err = getHAProxyVersion()
-	log.Infof("HAProxy version: %s", clusterMetadata.HAProxyVersion)
+	promURL, promToken, err := ocpMetadata.GetPrometheus()
 	if err != nil {
+		log.Error("Error fetching prometheus information")
 		return err
 	}
+	p, err := prometheus.NewClient(promURL, promToken, "", "", true)
+	if err != nil {
+		log.Error("Error creating prometheus client")
+		return err
+	}
+	if err != nil {
+		log.Errorf("Couldn't fetch haproxy version: %v", err)
+	} else {
+		log.Infof("HAProxy version: %s", clusterMetadata.HAProxyVersion)
+	}
 	if indexer != nil {
 		if _, ok := (*indexer).(*indexers.Elastic); ok {
 			comparator = comparison.NewComparator(*indexers.ESClient, baseIndex)
@@ -112,7 +124,7 @@ func Start(uuid, baseUUID, baseIndex string, tolerancy int, indexer *indexers.In
 				return err
 			}
 		}
-		if benchmarkResult, err = runBenchmark(cfg, clusterMetadata); err != nil {
+		if benchmarkResult, err = runBenchmark(cfg, clusterMetadata, p); err != nil {
 			return err
 		}
 		if indexer != nil {

pkg/runner/tools/types.go

@@ -52,24 +52,25 @@ type PodResult struct {
 }
 
 type Result struct {
-	UUID         string        `json:"uuid"`
-	Sample       int           `json:"sample"`
-	Config       config.Config `json:"config"`
-	Pods         []PodResult   `json:"pods"`
-	Timestamp    time.Time     `json:"timestamp"`
-	TotalAvgRps  float64       `json:"total_avg_rps"`
-	StdevRps     float64       `json:"rps_stdev"`
-	StdevLatency float64       `json:"stdev_lat"`
-	AvgLatency   float64       `json:"avg_lat_us"`
-	MaxLatency   float64       `json:"max_lat_us"`
-	P90Latency   float64       `json:"p90_lat_us"`
-	P95Latency   float64       `json:"p95_lat_us"`
-	P99Latency   float64       `json:"p99_lat_us"`
-	HTTPErrors   int64         `json:"http_errors"`
-	ReadErrors   int64         `json:"read_errors"`
-	WriteErrors  int64         `json:"write_errors"`
-	Requests     int64         `json:"requests"`
-	Timeouts     int64         `json:"timeouts"`
-	Version      string        `json:"version"`
+	UUID         string             `json:"uuid"`
+	Sample       int                `json:"sample"`
+	Config       config.Config      `json:"config"`
+	Pods         []PodResult        `json:"pods"`
+	Timestamp    time.Time          `json:"timestamp"`
+	TotalAvgRps  float64            `json:"total_avg_rps"`
+	StdevRps     float64            `json:"rps_stdev"`
+	StdevLatency float64            `json:"stdev_lat"`
+	AvgLatency   float64            `json:"avg_lat_us"`
+	MaxLatency   float64            `json:"max_lat_us"`
+	P90Latency   float64            `json:"p90_lat_us"`
+	P95Latency   float64            `json:"p95_lat_us"`
+	P99Latency   float64            `json:"p99_lat_us"`
+	HTTPErrors   int64              `json:"http_errors"`
+	ReadErrors   int64              `json:"read_errors"`
+	WriteErrors  int64              `json:"write_errors"`
+	Requests     int64              `json:"requests"`
+	Timeouts     int64              `json:"timeouts"`
+	Version      string             `json:"version"`
+	InfraMetrics map[string]float64 `json:"infra_metrics"`
 	ClusterMetadata
 }