From 891b4a38f3aaf4ed8a0294ba3ded56b8eb34950b Mon Sep 17 00:00:00 2001 From: Manda-supraja26 Date: Tue, 30 Jan 2024 20:04:56 +0530 Subject: [PATCH] Adding YCSB Dashboard to grafonet --- assets/ycsb/panels.libsonnet | 86 +++++++++++++++++++ assets/ycsb/queries.libsonnet | 138 ++++++++++++++++++++++++++++++ assets/ycsb/variables.libsonnet | 50 +++++++++++ templates/General/ycsb-v2.jsonnet | 29 +++++++ 4 files changed, 303 insertions(+) create mode 100644 assets/ycsb/panels.libsonnet create mode 100644 assets/ycsb/queries.libsonnet create mode 100644 assets/ycsb/variables.libsonnet create mode 100644 templates/General/ycsb-v2.jsonnet diff --git a/assets/ycsb/panels.libsonnet b/assets/ycsb/panels.libsonnet new file mode 100644 index 0000000..73afc4a --- /dev/null +++ b/assets/ycsb/panels.libsonnet @@ -0,0 +1,86 @@ +local g = import 'github.com/grafana/grafonnet/gen/grafonnet-latest/main.libsonnet'; + +{ + timeSeries: { + local timeSeries = g.panel.timeSeries, + local custom = timeSeries.fieldConfig.defaults.custom, + local options = timeSeries.options, + + base(title, datasource,unit, targets, gridPos): + timeSeries.new(title) + + timeSeries.queryOptions.withTargets(targets) + + timeSeries.datasource.withType('elasticsearch') + + timeSeries.datasource.withUid(datasource) + + timeSeries.standardOptions.withUnit(unit) + + timeSeries.gridPos.withX(gridPos.x) + + timeSeries.gridPos.withY(gridPos.y) + + timeSeries.gridPos.withH(gridPos.h) + + timeSeries.gridPos.withW(gridPos.w) + + custom.withSpanNulls(false) + + custom.withFillOpacity(25) + + options.tooltip.withMode('multi') + + options.tooltip.withSort('none') + + options.legend.withShowLegend(true) + + custom.withLineWidth(2), + + overallThroughputPerYCSB(title, datasource, unit, targets, gridPos): + self.base(title, datasource, unit, targets, gridPos) + + options.legend.withDisplayMode('table') + + options.legend.withPlacement('right') + + custom.withDrawStyle('bars') + + custom.withFillOpacity(100) + + custom.withPointSize(4) + + custom.withShowPoints('never') + + options.legend.withCalcs([ + 'sum' + ]), + + LatancyofEachWorkloadPerYCSBOperation(title, datasource, unit, targets, gridPos): + self.base(title, datasource, unit, targets, gridPos) + + options.legend.withDisplayMode('list') + + options.legend.withPlacement('bottom') + + custom.withFillOpacity(100) + + custom.withPointSize(4) + + custom.withDrawStyle('bars') + + custom.withShowPoints('never'), + + latency90percReportedFromYCSB(title, datasource, unit, targets, gridPos): + self.base(title, datasource, unit, targets, gridPos) + + custom.withDrawStyle('points') + + custom.withFillOpacity(20) + + options.legend.withDisplayMode('list') + + options.legend.withPlacement('bottom') + + options.legend.withCalcs([]) + + custom.withPointSize(4) + + custom.withShowPoints('always'), + + throughputOvertimePhase(title, datasource, unit, targets, gridPos): + self.base(title, datasource, unit, targets, gridPos) + + custom.withDrawStyle('line') + + custom.withFillOpacity(20) + + custom.withPointSize(5) + + options.legend.withDisplayMode('list') + + options.legend.withPlacement('bottom') + + options.legend.withCalcs([]) + + custom.withShowPoints('never'), + }, + + table: { + local table = g.panel.table, + local custom = table.fieldConfig.defaults.custom, + local options = table.options, + + base(title, datasource, targets, gridPos): + table.new(title) + + table.queryOptions.withTargets(targets) + + table.datasource.withType('elasticsearch') + + table.datasource.withUid(datasource) + + table.gridPos.withX(gridPos.x) + + table.gridPos.withY(gridPos.y) + + table.gridPos.withH(gridPos.h) + + table.gridPos.withW(gridPos.w) + + custom.cellOptions.TableSparklineCellOptions.withTransform('timeseries_to_columns') + + options.withShowHeader(true) + + options.sortBy.withDesc(true), + }, +} \ No newline at end of file diff --git a/assets/ycsb/queries.libsonnet b/assets/ycsb/queries.libsonnet new file mode 100644 index 0000000..7e948d7 --- /dev/null +++ b/assets/ycsb/queries.libsonnet @@ -0,0 +1,138 @@ +local g = import 'github.com/grafana/grafonnet/gen/grafonnet-latest/main.libsonnet'; +local variables = import './variables.libsonnet'; +local elasticsearch = g.query.elasticsearch; + +{ + throughput_overtime: { + query(): + elasticsearch.withAlias(null) + + elasticsearch.withBucketAggs([ + elasticsearch.bucketAggs.Terms.withField("action.keyword") + + elasticsearch.bucketAggs.Terms.withId("4") + + elasticsearch.bucketAggs.Terms.withType('terms') + + elasticsearch.bucketAggs.Terms.settings.withOrder('desc') + + elasticsearch.bucketAggs.Terms.settings.withOrderBy('_term') + + elasticsearch.bucketAggs.Terms.settings.withMinDocCount(1) + + elasticsearch.bucketAggs.Terms.settings.withSize("10"), + elasticsearch.bucketAggs.DateHistogram.withField("timestamp") + + elasticsearch.bucketAggs.DateHistogram.withId("3") + + elasticsearch.bucketAggs.DateHistogram.withType('date_histogram') + + elasticsearch.bucketAggs.DateHistogram.settings.withInterval('auto') + + elasticsearch.bucketAggs.DateHistogram.settings.withMinDocCount(0) + + elasticsearch.bucketAggs.DateHistogram.settings.withTimeZone("utc") + + elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges(0), + ]) + + elasticsearch.withMetrics([ + elasticsearch.metrics.MetricAggregationWithSettings.Average.withField("overall_rate") + + elasticsearch.metrics.MetricAggregationWithSettings.Average.withId("1") + + elasticsearch.metrics.MetricAggregationWithSettings.Average.withType('avg') + ]) + + elasticsearch.withQuery('(uuid.keyword = $uuid) AND (phase.keyword = $phase) AND (user.keyword=$user) AND (action.keyword=$operation)') + + elasticsearch.withTimeField('timestamp') + }, + + phase_average_latency: { + query(): + elasticsearch.withAlias("{{ocpMajorVersion.keyword}}") + + elasticsearch.withBucketAggs([ + elasticsearch.bucketAggs.Terms.withField("action.keyword") + + elasticsearch.bucketAggs.Terms.withId("3") + + elasticsearch.bucketAggs.Terms.withType('terms') + + elasticsearch.bucketAggs.Terms.settings.withOrder('desc') + + elasticsearch.bucketAggs.Terms.settings.withOrderBy('_term') + + elasticsearch.bucketAggs.Terms.settings.withMinDocCount(1) + + elasticsearch.bucketAggs.Terms.settings.withSize("10"), + elasticsearch.bucketAggs.DateHistogram.withField("timestamp") + + elasticsearch.bucketAggs.DateHistogram.withId("2") + + elasticsearch.bucketAggs.DateHistogram.withType('date_histogram') + + elasticsearch.bucketAggs.DateHistogram.settings.withInterval('auto') + + elasticsearch.bucketAggs.DateHistogram.settings.withMinDocCount(0) + + elasticsearch.bucketAggs.DateHistogram.settings.withTimeZone("utc") + + elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges(0), + ]) + + elasticsearch.withMetrics([ + elasticsearch.metrics.MetricAggregationWithSettings.Average.withField("latency_90") + + elasticsearch.metrics.MetricAggregationWithSettings.Average.withId("1") + + elasticsearch.metrics.MetricAggregationWithSettings.Average.withType('avg') + ]) + + elasticsearch.withQuery('(uuid.keyword = $uuid) AND (phase.keyword = $phase) AND (user.keyword=$user) AND (action.keyword=$operation)') + + elasticsearch.withTimeField('timestamp') + }, + + latency_95: { + query(): + elasticsearch.withAlias("{{ocpMajorVersion.keyword}}") + + elasticsearch.withBucketAggs([ + elasticsearch.bucketAggs.Terms.withField("workload_type.keyword") + + elasticsearch.bucketAggs.Terms.withId("5") + + elasticsearch.bucketAggs.Terms.withType('terms') + + elasticsearch.bucketAggs.Terms.settings.withOrder('desc') + + elasticsearch.bucketAggs.Terms.settings.withOrderBy('_term') + + elasticsearch.bucketAggs.Terms.settings.withMinDocCount(1) + + elasticsearch.bucketAggs.Terms.settings.withSize("10"), + elasticsearch.bucketAggs.DateHistogram.withField("timestamp") + + elasticsearch.bucketAggs.DateHistogram.withId("3") + + elasticsearch.bucketAggs.DateHistogram.withType('date_histogram') + + elasticsearch.bucketAggs.DateHistogram.settings.withInterval('auto') + + elasticsearch.bucketAggs.DateHistogram.settings.withMinDocCount(0) + + elasticsearch.bucketAggs.DateHistogram.settings.withTimeZone("utc") + + elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges(0), + ]) + + elasticsearch.withMetrics([ + elasticsearch.metrics.MetricAggregationWithSettings.Average.withField("data.$operation.95thPercentileLatency(us)") + + elasticsearch.metrics.MetricAggregationWithSettings.Average.withId("1") + + elasticsearch.metrics.MetricAggregationWithSettings.Average.withType('avg') + ]) + + elasticsearch.withQuery('(uuid.keyword = $uuid) AND (phase.keyword = $phase) AND (user.keyword=$user)') + + elasticsearch.withTimeField('timestamp') + }, + + overall_workload_throughput: { + query(): + elasticsearch.withAlias("{{ocpMajorVersion.keyword}}") + + elasticsearch.withBucketAggs([ + elasticsearch.bucketAggs.Terms.withField("workload_type.keyword") + + elasticsearch.bucketAggs.Terms.withId("5") + + elasticsearch.bucketAggs.Terms.withType('terms') + + elasticsearch.bucketAggs.Terms.settings.withOrder('desc') + + elasticsearch.bucketAggs.Terms.settings.withOrderBy('_term') + + elasticsearch.bucketAggs.Terms.settings.withMinDocCount(1) + + elasticsearch.bucketAggs.Terms.settings.withSize("10"), + elasticsearch.bucketAggs.DateHistogram.withField("timestamp") + + elasticsearch.bucketAggs.DateHistogram.withId("3") + + elasticsearch.bucketAggs.DateHistogram.withType('date_histogram') + + elasticsearch.bucketAggs.DateHistogram.settings.withInterval('auto') + + elasticsearch.bucketAggs.DateHistogram.settings.withMinDocCount(0) + + elasticsearch.bucketAggs.DateHistogram.settings.withTimeZone("utc") + + elasticsearch.bucketAggs.DateHistogram.settings.withTrimEdges(0), + ]) + + elasticsearch.withMetrics([ + elasticsearch.metrics.MetricAggregationWithSettings.Sum.withField("data.OVERALL.Throughput(ops/sec)") + + elasticsearch.metrics.MetricAggregationWithSettings.Sum.withId("1") + + elasticsearch.metrics.MetricAggregationWithSettings.Sum.withType('sum') + ]) + + elasticsearch.withQuery('(uuid.keyword = $uuid) AND (phase.keyword = $phase) AND (user.keyword=$user)') + + elasticsearch.withTimeField('timestamp') + }, + + aggregate_operation_sum: { + query(): + elasticsearch.withAlias("$operation - Operations") + + elasticsearch.withBucketAggs([ + elasticsearch.bucketAggs.Terms.withField("workload_type.keyword") + + elasticsearch.bucketAggs.Terms.withId("3") + + elasticsearch.bucketAggs.Terms.withType('terms') + + elasticsearch.bucketAggs.Terms.settings.withOrder('desc') + + elasticsearch.bucketAggs.Terms.settings.withOrderBy('_term') + + elasticsearch.bucketAggs.Terms.settings.withMinDocCount(1) + + elasticsearch.bucketAggs.Terms.settings.withSize("10"), + ]) + + elasticsearch.withMetrics([ + elasticsearch.metrics.MetricAggregationWithSettings.Sum.withField("data.$operation.Operations") + + elasticsearch.metrics.MetricAggregationWithSettings.Sum.withId("1") + + elasticsearch.metrics.MetricAggregationWithSettings.Sum.withType('sum') + ]) + + elasticsearch.withQuery('(uuid.keyword = $uuid) AND (phase.keyword = $phase) AND (user.keyword=$user)') + + elasticsearch.withTimeField('timestamp') + } +} \ No newline at end of file diff --git a/assets/ycsb/variables.libsonnet b/assets/ycsb/variables.libsonnet new file mode 100644 index 0000000..8dfedd1 --- /dev/null +++ b/assets/ycsb/variables.libsonnet @@ -0,0 +1,50 @@ +local g = import 'github.com/grafana/grafonnet/gen/grafonnet-latest/main.libsonnet'; +local var = g.dashboard.variable; + +{ + Datasource1: + var.datasource.new('Datasource1','elasticsearch') + + var.datasource.withRegex('') + + var.query.withRefresh(1) + + var.query.generalOptions.withLabel('ycsb-results datasource'), + + Datasource2: + var.datasource.new('Datasource2','elasticsearch') + + var.datasource.withRegex('') + + var.query.withRefresh(1) + + var.query.generalOptions.withLabel('ycsb-summary datasource'), + + uuid: + var.query.new('uuid','{"find": "terms", "field": "uuid.keyword"}') + + var.query.withDatasourceFromVariable(self.Datasource2) + + var.query.withRefresh(2) + + var.datasource.withRegex('') + + var.query.selectionOptions.withMulti(false) + + var.query.selectionOptions.withIncludeAll(true), + + user: + var.query.new('user','{"find": "terms", "field": "user.keyword"}') + + var.query.withDatasourceFromVariable(self.Datasource2) + + var.query.withRefresh(2) + + var.datasource.withRegex('') + + var.query.selectionOptions.withMulti(false) + + var.query.selectionOptions.withIncludeAll(true), + + phase: + var.query.new('phase','{"find": "terms", "field": "phase.keyword"}') + + var.query.withDatasourceFromVariable(self.Datasource2) + + var.query.withRefresh(2) + + var.datasource.withRegex('') + + var.query.generalOptions.withCurrent('run','run') + + var.query.selectionOptions.withMulti(false) + + var.query.selectionOptions.withIncludeAll(true), + + operation: + var.query.new('operation','{"find": "fields", "field": "data.*.Operations"}') + + var.query.withDatasourceFromVariable(self.Datasource2) + + var.query.withRefresh(2) + + var.query.generalOptions.withCurrent('READ','READ') + + var.datasource.withRegex('/data.(.*).Operations/') + + var.query.selectionOptions.withMulti(false) + + var.query.selectionOptions.withIncludeAll(true), +} \ No newline at end of file diff --git a/templates/General/ycsb-v2.jsonnet b/templates/General/ycsb-v2.jsonnet new file mode 100644 index 0000000..14eab24 --- /dev/null +++ b/templates/General/ycsb-v2.jsonnet @@ -0,0 +1,29 @@ +local panels = import '../../assets/ycsb/panels.libsonnet'; +local queries = import '../../assets/ycsb/queries.libsonnet'; +local variables = import '../../assets/ycsb/variables.libsonnet'; +local g = import 'github.com/grafana/grafonnet/gen/grafonnet-latest/main.libsonnet'; + +g.dashboard.new('YCSB') ++ g.dashboard.time.withFrom('now/y') ++ g.dashboard.time.withTo('now') ++ g.dashboard.withTimezone('utc') ++ g.dashboard.timepicker.withRefreshIntervals(['5s', '10s', '30s', '1m', '5m', '15m', '30m', '1h', '2h', '1d']) ++ g.dashboard.timepicker.withTimeOptions(['5m', '15m', '1h', '6h', '12h', '24h', '2d', '7d', '30d']) ++ g.dashboard.withRefresh('') ++ g.dashboard.withEditable(false) ++ g.dashboard.graphTooltip.withSharedCrosshair() ++ g.dashboard.withVariables([ + variables.Datasource1, + variables.Datasource2, + variables.uuid, + variables.user, + variables.phase, + variables.operation, +]) ++ g.dashboard.withPanels([ + panels.timeSeries.throughputOvertimePhase('Throughput overtime - Phase = $phase : Operation = $operation', '$Datasource1', 'ops', queries.throughput_overtime.query(), { x: 0, y: 0, w: 12, h: 9 }), + panels.timeSeries.latency90percReportedFromYCSB('Phase = $phase :: Latency - 90%tile Reported from YCSB', '$Datasource1', 'µs', queries.phase_average_latency.query(), { x: 12, y: 0, w: 12, h: 9 }), + panels.timeSeries.LatancyofEachWorkloadPerYCSBOperation('95th% Latency of each workload per YCSB Operation', '$Datasource2', 'µs', queries.latency_95.query(), { x: 0, y: 9, w: 24, h: 6 }), + panels.timeSeries.overallThroughputPerYCSB('Overall Throughput per YCSB Workload', '$Datasource2', 'ops', queries.overall_workload_throughput.query(), { x: 0, y: 15, w: 16, h: 10 }), + panels.table.base('Phase = $phase :: $operation - Count', '$Datasource2', queries.aggregate_operation_sum.query(), { x: 16, y: 15, w: 8, h: 10 }), +])