Deprecate comparisons

Signed-off-by: Raul Sevilla <[email protected]>
cloud-bulldozer · Nov 23, 2023 · 5417923 · 5417923
1 parent 7e94871
commit 5417923
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 152 deletions.
diff --git a/cmd/ingress-perf.go b/cmd/ingress-perf.go
@@ -43,9 +43,8 @@ var versionCmd = &cobra.Command{
 }
 
 func run() *cobra.Command {
-	var cfg, uuid, baseUUID, esServer, esIndex, logLevel, baseIndex, outputDir string
+	var cfg, uuid, esServer, esIndex, logLevel, outputDir string
 	var cleanup, podMetrics bool
-	var tolerancy int
 	cmd := &cobra.Command{
 		Use:           "run",
 		Short:         "Run benchmark",
@@ -66,17 +65,13 @@ func run() *cobra.Command {
 			}
 			r := runner.New(
 				uuid, cleanup,
-				runner.WithComparison(baseUUID, baseIndex, tolerancy),
 				runner.WithIndexer(esServer, esIndex, outputDir, podMetrics),
 			)
 			return r.Start()
 		},
 	}
 	cmd.Flags().StringVarP(&cfg, "cfg", "c", "", "Configuration file")
 	cmd.Flags().StringVar(&uuid, "uuid", uid.NewV4().String(), "Benchmark uuid")
-	cmd.Flags().StringVar(&baseUUID, "baseline-uuid", "", "Baseline uuid to compare the results with")
-	cmd.Flags().StringVar(&baseIndex, "baseline-index", "ingress-performance", "Baseline Elasticsearch index")
-	cmd.Flags().IntVar(&tolerancy, "tolerancy", 20, "Comparison tolerancy, must be an integer between 1 and 100")
 	cmd.Flags().StringVar(&esServer, "es-server", "", "Elastic Search endpoint")
 	cmd.Flags().StringVar(&esIndex, "es-index", "ingress-performance", "Elasticsearch index")
 	cmd.Flags().StringVar(&outputDir, "output-dir", "output", "Store collected metrics in this directory")

diff --git a/config/standard.yml b/config/standard.yml
@@ -3,98 +3,10 @@
 # First sample is set as warmup and also will tune the default ingress-controller to place the routers on infra nodes
 - termination: http
   connections: 20
-  samples: 1
-  duration: 1m
-  path: /1024.html
-  concurrency: 1
-  tool: wrk
-  serverReplicas: 90
-  requestTimeout: 2s
-  tuningPatch: '{"spec":{"nodePlacement": {"nodeSelector": {"matchLabels": {"node-role.kubernetes.io/infra": ""}}}, "replicas": 2}}'
-  warmup: true
-
-- termination: http
-  connections: 200
-  samples: 2
-  duration: 2m
-  path: /1024.html
-  concurrency: 1
-  tool: wrk
-  serverReplicas: 90
-  delay: 10s
-  procs: 2
-
-- termination: edge
-  connections: 200
-  samples: 2
-  duration: 2m
-  path: /1024.html
-  concurrency: 1
-  tool: wrk
-  serverReplicas: 90
-  delay: 10s
-  procs: 2
-
-- termination: reencrypt
-  connections: 200
-  samples: 2
-  duration: 2m
-  path: /1024.html
-  concurrency: 1
-  tool: wrk
-  serverReplicas: 90
-  delay: 10s
-  procs: 2
-
-- termination: passthrough
-  connections: 200
   samples: 2
-  duration: 2m
+  duration: 30s
   path: /1024.html
   concurrency: 1
   tool: wrk
-  serverReplicas: 90
-  delay: 10s
-  procs: 2
-
-- termination: http
-  connections: 200
-  samples: 2
-  duration: 2m
-  path: /1024.html
-  concurrency: 20
-  tool: wrk
-  serverReplicas: 90
-  delay: 10s
-
-- termination: edge
-  connections: 200
-  samples: 2
-  duration: 2m
-  path: /1024.html
-  concurrency: 20
-  tool: wrk
-  serverReplicas: 90
-  delay: 10s
-
-- termination: reencrypt
-  connections: 200
-  samples: 2
-  duration: 2m
-  path: /1024.html
-  concurrency: 20
-  tool: wrk
-  serverReplicas: 90
-  delay: 10s
-
-- termination: passthrough
-  connections: 200
-  samples: 2
-  duration: 2m
-  path: /1024.html
-  concurrency: 20
-  tool: wrk
-  serverReplicas: 90
-  delay: 10s
-
-
+  serverReplicas: 30
+  requestTimeout: 2s
diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go
@@ -21,7 +21,6 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/cloud-bulldozer/go-commons/comparison"
 	"github.com/cloud-bulldozer/go-commons/indexers"
 	"github.com/cloud-bulldozer/go-commons/prometheus"
 
@@ -61,18 +60,6 @@ func New(uuid string, cleanup bool, opts ...OptsFunctions) *Runner {
 	return r
 }
 
-func WithComparison(baseUUID, baseIndex string, tolerancy int) OptsFunctions {
-	return func(r *Runner) {
-		if baseUUID != "" {
-			if tolerancy > 100 || tolerancy < 1 {
-				log.Fatalf("Tolerancy must be an integer between 1 and 100: %d", tolerancy)
-			}
-			r.baseUUID = baseUUID
-			r.tolerancy = tolerancy
-			r.baseIndex = baseIndex
-		}
-	}
-}
 func WithIndexer(esServer, esIndex, resultsDir string, podMetrics bool) OptsFunctions {
 	return func(r *Runner) {
 		if esServer != "" || resultsDir != "" {
@@ -105,7 +92,6 @@ func (r *Runner) Start() error {
 	var err error
 	var kubeconfig string
 	var benchmarkResult []tools.Result
-	var comparator comparison.Comparator
 	var clusterMetadata tools.ClusterMetadata
 	var benchmarkResultDocuments []interface{}
 	passed := true
@@ -131,7 +117,6 @@ func (r *Runner) Start() error {
 	if err != nil {
 		return err
 	}
-	clusterMetadata.HAProxyVersion, err = getHAProxyVersion()
 	promURL, promToken, err := ocpMetadata.GetPrometheus()
 	if err != nil {
 		log.Error("Error fetching prometheus information")
@@ -142,16 +127,12 @@ func (r *Runner) Start() error {
 		log.Error("Error creating prometheus client")
 		return err
 	}
+	clusterMetadata.HAProxyVersion, err = getHAProxyVersion()
 	if err != nil {
 		log.Errorf("Couldn't fetch haproxy version: %v", err)
 	} else {
 		log.Infof("HAProxy version: %s", clusterMetadata.HAProxyVersion)
 	}
-	if r.indexer != nil {
-		if _, ok := (*r.indexer).(*indexers.Elastic); ok {
-			comparator = comparison.NewComparator(*indexers.ESClient, r.baseIndex)
-		}
-	}
 	if err := deployAssets(); err != nil {
 		return err
 	}
@@ -179,42 +160,20 @@ func (r *Runner) Start() error {
 		if benchmarkResult, err = runBenchmark(cfg, clusterMetadata, p, r.podMetrics); err != nil {
 			return err
 		}
-		if r.indexer != nil {
-			if !cfg.Warmup {
-				for _, res := range benchmarkResult {
-					benchmarkResultDocuments = append(benchmarkResultDocuments, res)
-				}
-				// When not using local indexer, empty the documents array when all documents after indexing them
-				if _, ok := (*r.indexer).(*indexers.Local); !ok {
-					if indexDocuments(*r.indexer, benchmarkResultDocuments, indexers.IndexingOpts{}) != nil {
-						log.Errorf("Indexing error: %v", err.Error())
-					}
-					benchmarkResultDocuments = []interface{}{}
-				}
-				if r.baseUUID != "" {
-					log.Infof("Comparing total_avg_rps with baseline: %v in index %s", r.baseUUID, r.baseIndex)
-					var totalAvgRps float64
-					query := fmt.Sprintf("uuid.keyword: %s AND config.termination.keyword: %s AND config.concurrency: %d AND config.connections: %d AND config.serverReplicas: %d AND config.path.keyword: \\%s",
-						r.baseUUID, cfg.Termination, cfg.Concurrency, cfg.Connections, cfg.ServerReplicas, cfg.Path)
-					log.Debugf("Query: %s", query)
-					for _, b := range benchmarkResult {
-						totalAvgRps += b.TotalAvgRps
-					}
-					totalAvgRps = totalAvgRps / float64(len(benchmarkResult))
-					msg, err := comparator.Compare("total_avg_rps", query, comparison.Avg, totalAvgRps, r.tolerancy)
-					if err != nil {
-						log.Error(err.Error())
-						passed = false
-					} else {
-						log.Info(msg)
-					}
+		if r.indexer != nil && !cfg.Warmup {
+			for _, res := range benchmarkResult {
+				benchmarkResultDocuments = append(benchmarkResultDocuments, res)
+			}
+			// When not using local indexer, empty the documents array when all documents after indexing them
+			if _, ok := (*r.indexer).(*indexers.Local); !ok {
+				if indexDocuments(*r.indexer, benchmarkResultDocuments, indexers.IndexingOpts{}) != nil {
+					log.Errorf("Indexing error: %v", err.Error())
 				}
-			} else {
-				log.Info("Warmup is enabled, skipping indexing")
+				benchmarkResultDocuments = []interface{}{}
 			}
 		}
 	}
-	if r.indexer != nil {
+	if _, ok := (*r.indexer).(*indexers.Local); r.indexer != nil && ok {
 		if err := indexDocuments(*r.indexer, benchmarkResultDocuments, indexers.IndexingOpts{MetricName: r.uuid}); err != nil {
 			log.Errorf("Indexing error: %v", err.Error())
 		}

diff --git a/pkg/runner/types.go b/pkg/runner/types.go
@@ -37,9 +37,6 @@ const (
 
 type Runner struct {
 	uuid       string
-	baseUUID   string
-	baseIndex  string
-	tolerancy  int
 	indexer    *indexers.Indexer
 	podMetrics bool
 	cleanup    bool