-
Notifications
You must be signed in to change notification settings - Fork 15
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add saturation gauge sli library (#594)
* feat: add saturation gauge sli library * feat: adding new avail slis for status check and inservice instances * fix: correction to legends and formula for calculating colour thresholds * refactor: correction status metric with integer to support avg success prom target, also update fill on all graph panels to 4 * fix: correction to saturation SLI promql expression to use sum and remove detailed dashboard gen * feat: update output folder permissions
- Loading branch information
1 parent
b823360
commit 12c834d
Showing
19 changed files
with
503 additions
and
22 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
123 changes: 123 additions & 0 deletions
123
...value-libraries/availability-gauge-using-inservice-and-desired-instance-metrics.libsonnet
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,123 @@ | ||
| // Divides the Count where inservice instances do not match desired instances by the overall count of samples | ||
| // target metric samples taken from average-using-single-metric | ||
|
|
||
| // Target metrics: | ||
| // target - Metric to get the average value of over evaluation interval | ||
|
|
||
| // Additional config: | ||
| // counterIntegerTarget in SLI spec | ||
|
|
||
| // MaC imports | ||
| local sliValueLibraryFunctions = import '../util/sli-value-library-functions.libsonnet'; | ||
|
|
||
| // Grafana imports | ||
| local grafana = import 'grafonnet/grafana.libsonnet'; | ||
| local prometheus = grafana.prometheus; | ||
| local graphPanel = grafana.graphPanel; | ||
|
|
||
| // Creates the custom SLI value rule | ||
| // @param sliSpec The spec for the SLI having its recording rules created | ||
| // @param sliMetadata Metadata about the type and category of the SLI | ||
| // @param config The config for the service defined in the mixin file | ||
| // @returns JSON defining the recording rule | ||
| local createSliValueRule(sliSpec, sliMetadata, config) = | ||
| local metricConfig = sliValueLibraryFunctions.getMetricConfig(sliSpec); | ||
| local ruleSelectors = sliValueLibraryFunctions.createRuleSelectors(metricConfig, sliSpec, config); | ||
| local targetMetrics = sliValueLibraryFunctions.getTargetMetrics(metricConfig, sliSpec); | ||
| local selectorLabels = sliValueLibraryFunctions.getSelectorLabels(metricConfig); | ||
|
|
||
| [ | ||
| { | ||
| record: 'sli_value', | ||
| expr: ||| | ||
| sum without (%(selectorLabels)s) (label_replace(label_replace( | ||
| ( | ||
| ( | ||
| sum by(%(selectorLabels)s) (avg_over_time(%(desiredMetric)s{%(selectors)s}[%(evalInterval)s])) | ||
| - | ||
| sum by(%(selectorLabels)s) (avg_over_time(%(inserviceMetric)s{%(selectors)s}[%(evalInterval)s])) | ||
| ) >= 1 | ||
| / | ||
| count by(%(selectorLabels)s) (count_over_time(%(desiredMetric)s{%(selectors)s}[%(evalInterval)s])) | ||
| ), | ||
| "sli_environment", "$1", "%(environmentSelectorLabel)s", "(.*)"), "sli_product", "$1", "%(productSelectorLabel)s", "(.*)")) | ||
| ||| % { | ||
| desiredMetric: targetMetrics.desired, | ||
| inserviceMetric: targetMetrics.inservice, | ||
| selectorLabels: std.join(', ', std.objectValues(selectorLabels)), | ||
| environmentSelectorLabel: selectorLabels.environment, | ||
| productSelectorLabel: selectorLabels.product, | ||
| selectors: std.join(', ', ruleSelectors), | ||
| evalInterval: sliSpec.evalInterval, | ||
| }, | ||
| labels: sliSpec.sliLabels + sliMetadata, | ||
| }, | ||
| ]; | ||
|
|
||
| // Creates Grafana dashboard graph panel | ||
| // @param sliSpec The spec for the SLI having its dashboard created | ||
| // @returns Grafana graph panel object | ||
| local createGraphPanel(sliSpec) = | ||
| local metricConfig = sliValueLibraryFunctions.getMetricConfig(sliSpec); | ||
| local dashboardSelectors = sliValueLibraryFunctions.createDashboardSelectors(metricConfig, sliSpec); | ||
| local targetMetrics = sliValueLibraryFunctions.getTargetMetrics(metricConfig, sliSpec); | ||
|
|
||
| graphPanel.new( | ||
| title='%s' % sliSpec.sliDescription, | ||
| datasource='prometheus', | ||
| description=||| | ||
| * Sample interval is %(evalInterval)s | ||
| * Selectors are %(selectors)s | ||
| ||| % { | ||
| selectors: std.strReplace(std.join(', ', sliValueLibraryFunctions.getSelectors(metricConfig, sliSpec)), '~', '\\~'), | ||
| evalInterval: sliSpec.evalInterval, | ||
| }, | ||
| min=0, | ||
| fill=4, | ||
| ).addTarget( | ||
| prometheus.target( | ||
| ||| | ||
| sum(avg_over_time(%(inserviceMetric)s{%(selectors)s}[%(evalInterval)s]) >= 0 or vector(0)) | ||
| ||| % { | ||
| desiredMetric: targetMetrics.desired, | ||
| inserviceMetric: targetMetrics.inservice, | ||
| selectors: std.join(',', dashboardSelectors), | ||
| evalInterval: sliSpec.evalInterval, | ||
| }, | ||
| legendFormat='avg inservice instances', | ||
| ), | ||
| ).addTarget( | ||
| prometheus.target( | ||
| ||| | ||
| ( | ||
| sum(avg_over_time(%(inserviceMetric)s{%(selectors)s}[%(evalInterval)s]) >= 0 or vector(0)) | ||
| - | ||
| sum(avg_over_time(%(desiredMetric)s{%(selectors)s}[%(evalInterval)s]) >= 0 or vector(0)) | ||
| ) >= 1 | ||
| / | ||
| count(count_over_time(%(desiredMetric)s{%(selectors)s}[%(evalInterval)s]) >= 0 or vector(0)) | ||
| ||| % { | ||
| desiredMetric: targetMetrics.desired, | ||
| inserviceMetric: targetMetrics.inservice, | ||
| selectors: std.join(',', dashboardSelectors), | ||
| evalInterval: sliSpec.evalInterval, | ||
| }, | ||
| legendFormat='avg period where inservice instances < desired instances' | ||
| ) | ||
| ).addSeriesOverride( | ||
| { | ||
| alias: '/avg period where inservice instances < desired instances/', | ||
| color: 'red', | ||
| }, | ||
| ).addSeriesOverride( | ||
| { | ||
| alias: '/avg inservice instances/', | ||
| color: 'green', | ||
| }, | ||
| ); | ||
|
|
||
| // File exports | ||
| { | ||
| createSliValueRule(sliSpec, sliMetadata, config): createSliValueRule(sliSpec, sliMetadata, config), | ||
| createGraphPanel(sliSpec): createGraphPanel(sliSpec), | ||
| } |
Oops, something went wrong.