Skip to content

Commit

Permalink
Add gauge on all async tasks and max async task id
Browse files Browse the repository at this point in the history
  • Loading branch information
@willgorman
willgorman committed May 22, 2024
1 parent e6c59b9 commit 718d3ed
Show file tree
Hide file tree
Showing 5 changed files with 94 additions and 33 deletions.
35 changes: 29 additions & 6 deletions pkg/prom/collector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,8 @@ func (c *SolidfireCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- MetricDescriptions.VirtualVolumeTasks
ch <- MetricDescriptions.BulkVolumeJobs
ch <- MetricDescriptions.AsyncResultsActive
ch <- MetricDescriptions.AsyncResults
ch <- MetricDescriptions.MaxAsyncResultID
}

func (c *SolidfireCollector) collectVolumeMeta(ctx context.Context, ch chan<- prometheus.Metric) error {
Expand Down Expand Up @@ -1296,31 +1298,52 @@ func (c *SolidfireCollector) collectAsyncResults(ctx context.Context, ch chan<-
if err != nil {
return err
}

m := make(map[string]int64)
maxAsyncResultID := 0
activeAsyncResults := make(map[string]int64)
allAsyncResults := make(map[string]int64)
for _, v := range ar.Result.AsyncHandles {
allAsyncResults[v.ResultType]++
if !v.Completed && !v.Success {
m[v.ResultType]++
activeAsyncResults[v.ResultType]++
}
if v.AsyncResultID > int64(maxAsyncResultID) {
maxAsyncResultID = int(v.AsyncResultID)
}
}

types := []string{"DriveAdd", "BulkVolume", "Clone", "DriveRemoval", "RtfiPendingNode"}
for _, t := range types {
if _, ok := m[t]; !ok {
m[t] = 0
if _, ok := activeAsyncResults[t]; !ok {
activeAsyncResults[t] = 0
}
if _, ok := allAsyncResults[t]; !ok {
allAsyncResults[t] = 0
}
}

mu.Lock()
defer mu.Unlock()
for k, v := range m {
for k, v := range activeAsyncResults {
ch <- prometheus.MustNewConstMetric(
MetricDescriptions.AsyncResultsActive,
prometheus.GaugeValue,
float64(v),
k,
)
}
for k, v := range allAsyncResults {
ch <- prometheus.MustNewConstMetric(
MetricDescriptions.AsyncResults,
prometheus.GaugeValue,
float64(v),
k,
)
}
ch <- prometheus.MustNewConstMetric(
MetricDescriptions.MaxAsyncResultID,
prometheus.GaugeValue,
float64(maxAsyncResultID),
)
return nil
}

Expand Down
16 changes: 15 additions & 1 deletion pkg/prom/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,8 @@ type Descriptions struct {
VirtualVolumeTasks *prometheus.Desc
BulkVolumeJobs *prometheus.Desc
AsyncResultsActive *prometheus.Desc
AsyncResults *prometheus.Desc
MaxAsyncResultID *prometheus.Desc
}

func NewMetricDescriptions(namespace string) *Descriptions {
Expand Down Expand Up @@ -920,10 +922,22 @@ func NewMetricDescriptions(namespace string) *Descriptions {
)
d.AsyncResultsActive = prometheus.NewDesc(
prometheus.BuildFQName(namespace, "", "cluster_volume_async_result_active"),
"The active jobs return by async results",
"The active jobs returned by async results",
[]string{"type"},
nil,
)
d.AsyncResults = prometheus.NewDesc(
prometheus.BuildFQName(namespace, "", "cluster_volume_async_result"),
"All (active and completed) jobs returned by async results",
[]string{"type"},
nil,
)
d.MaxAsyncResultID = prometheus.NewDesc(
prometheus.BuildFQName(namespace, "", "cluster_max_async_result_id"),
"The maximum id used by async result handles",
nil,
nil,
)

return &d
}
48 changes: 29 additions & 19 deletions pkg/solidfire/solidfire_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func TestClient_ListVolumeStats(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer gock.Off()
//gock.Observe(gock.DumpRequest)
// gock.Observe(gock.DumpRequest)
gock.New(sfHost).
Post(sfRPCEndpoint).
MatchType("json").
Expand All @@ -54,7 +54,8 @@ func TestClient_ListVolumeStats(t *testing.T) {
Params: solidfire.ListVolumeStatsRPCParams{
VolumeIDs: []int{},
IncludeVirtualVolumes: true,
}}).
},
}).
Reply(200).
BodyString(string(fixture))
gotRaw, err := sfClient.ListVolumeStats(context.Background())
Expand Down Expand Up @@ -90,7 +91,7 @@ func TestClient_ListVolumes(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer gock.Off()
//gock.Observe(gock.DumpRequest)
// gock.Observe(gock.DumpRequest)
gock.New(sfHost).
Post(sfRPCEndpoint).
MatchType("json").
Expand All @@ -99,7 +100,8 @@ func TestClient_ListVolumes(t *testing.T) {
Method: solidfire.RPCListVolumes,
Params: solidfire.ListVolumesRPCParams{
IncludeVirtualVolumes: true,
}}).
},
}).
Reply(200).
BodyString(string(fixture))
gotRaw, err := sfClient.ListVolumes(context.Background())
Expand All @@ -114,6 +116,7 @@ func TestClient_ListVolumes(t *testing.T) {
})
}
}

func TestClient_GetClusterCapacity(t *testing.T) {
fixture, err := ioutil.ReadFile(testutils.ResolveFixturePath(fixtureBasePath, solidfire.RPCGetClusterCapacity))
if err != nil {
Expand All @@ -134,14 +137,15 @@ func TestClient_GetClusterCapacity(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer gock.Off()
//gock.Observe(gock.DumpRequest)
// gock.Observe(gock.DumpRequest)
gock.New(sfHost).
Post(sfRPCEndpoint).
MatchType("json").
JSON(solidfire.RPCBody{
ID: 1,
Method: solidfire.RPCGetClusterCapacity,
Params: solidfire.GetClusterCapacityRPCParams{}}).
Params: solidfire.GetClusterCapacityRPCParams{},
}).
Reply(200).
BodyString(string(fixture))
gotRaw, err := sfClient.GetClusterCapacity(context.Background())
Expand Down Expand Up @@ -187,7 +191,8 @@ func TestClient_ListClusterFaults(t *testing.T) {
Params: solidfire.ListClusterFaultsRPCParams{
FaultTypes: "current",
BestPractices: true,
}}).
},
}).
Reply(200).
BodyString(string(fixture))

Expand Down Expand Up @@ -232,7 +237,8 @@ func TestClient_ListNodeStats(t *testing.T) {
JSON(solidfire.RPCBody{
ID: 1,
Method: solidfire.RPCListNodeStats,
Params: solidfire.ListNodeStatsRPCParams{}}).
Params: solidfire.ListNodeStatsRPCParams{},
}).
Reply(200).
BodyString(string(fixture))

Expand Down Expand Up @@ -279,7 +285,8 @@ func TestClient_ListVolumeQoSHistograms(t *testing.T) {
Method: solidfire.RPCListVolumeQoSHistograms,
Params: solidfire.ListVolumeQoSHistogramsRPCParams{
VolumeIDs: []int{}, // blank gives us all of them
}}).
},
}).
Reply(200).
BodyString(string(fixture))

Expand Down Expand Up @@ -324,7 +331,8 @@ func TestClient_ListAllNodes(t *testing.T) {
JSON(solidfire.RPCBody{
ID: 1,
Method: solidfire.RPCListAllNodes,
Params: solidfire.ListAllNodesRPCParams{}}).
Params: solidfire.ListAllNodesRPCParams{},
}).
Reply(200).
BodyString(string(fixture))

Expand Down Expand Up @@ -369,7 +377,8 @@ func TestClient_GetClusterStats(t *testing.T) {
JSON(solidfire.RPCBody{
ID: 1,
Method: solidfire.RPCGetClusterStats,
Params: solidfire.GetClusterStatsRPCParams{}}).
Params: solidfire.GetClusterStatsRPCParams{},
}).
Reply(200).
BodyString(string(fixture))

Expand Down Expand Up @@ -414,7 +423,8 @@ func TestClient_GetClusterFullThreshold(t *testing.T) {
JSON(solidfire.RPCBody{
ID: 1,
Method: solidfire.RPCGetClusterFullThreshold,
Params: solidfire.GetClusterFullThresholdParams{}}).
Params: solidfire.GetClusterFullThresholdParams{},
}).
Reply(200).
BodyString(string(fixture))

Expand Down Expand Up @@ -452,7 +462,7 @@ func TestClient_ListAccounts(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer gock.Off()
//gock.Observe(gock.DumpRequest)
// gock.Observe(gock.DumpRequest)
gock.New(sfHost).
Post(sfRPCEndpoint).
MatchType("json").
Expand Down Expand Up @@ -496,7 +506,7 @@ func TestClient_ListInitiators(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer gock.Off()
//gock.Observe(gock.DumpRequest)
// gock.Observe(gock.DumpRequest)
gock.New(sfHost).
Post(sfRPCEndpoint).
MatchType("json").
Expand Down Expand Up @@ -539,7 +549,7 @@ func TestClient_ListVirtualVolumeTasks(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer gock.Off()
//gock.Observe(gock.DumpRequest)
// gock.Observe(gock.DumpRequest)
gock.New(sfHost).
Post(sfRPCEndpoint).
MatchType("json").
Expand Down Expand Up @@ -575,14 +585,14 @@ func TestClient_ListAsyncResults(t *testing.T) {
wantErr bool
}{
{
want: 47,
want: 43,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer gock.Off()
//gock.Observe(gock.DumpRequest)
// gock.Observe(gock.DumpRequest)
gock.New(sfHost).
Post(sfRPCEndpoint).
MatchType("json").
Expand Down Expand Up @@ -625,7 +635,7 @@ func TestClient_ListBulkVolumeJobs(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer gock.Off()
//gock.Observe(gock.DumpRequest)
// gock.Observe(gock.DumpRequest)
gock.New(sfHost).
Post(sfRPCEndpoint).
MatchType("json").
Expand Down Expand Up @@ -669,7 +679,7 @@ func TestClient_ListVolumeAccessGroups(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer gock.Off()
//gock.Observe(gock.DumpRequest)
// gock.Observe(gock.DumpRequest)
gock.New(sfHost).
Post(sfRPCEndpoint).
MatchType("json").
Expand Down
14 changes: 14 additions & 0 deletions pkg/testutils/collector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ solidfire_cluster_last_sample_read_ops 0
solidfire_cluster_last_sample_write_bytes 0
solidfire_cluster_last_sample_write_ops 0
solidfire_cluster_latency_seconds 0
solidfire_cluster_max_async_result_id 47
solidfire_cluster_max_iops 3000
solidfire_cluster_max_metadata_over_provision_factor 5
solidfire_cluster_max_over_provisionable_space_bytes 1.855425871872e+13
Expand Down Expand Up @@ -85,6 +86,12 @@ solidfire_cluster_volume_async_result_active{type="DriveAdd"} 0
solidfire_cluster_volume_async_result_active{type="DriveRemoval"} 0
solidfire_cluster_volume_async_result_active{type="NotClone"} 1
solidfire_cluster_volume_async_result_active{type="RtfiPendingNode"} 0
solidfire_cluster_volume_async_result{type="BulkVolume"} 0
solidfire_cluster_volume_async_result{type="Clone"} 5
solidfire_cluster_volume_async_result{type="DriveAdd"} 0
solidfire_cluster_volume_async_result{type="DriveRemoval"} 0
solidfire_cluster_volume_async_result{type="NotClone"} 2
solidfire_cluster_volume_async_result{type="RtfiPendingNode"} 0
solidfire_cluster_volume_bulk_volume_job_count 1
solidfire_cluster_write_bytes_total 1.21720639488e+11
solidfire_cluster_write_latency_seconds 0
Expand Down Expand Up @@ -315,6 +322,7 @@ solidfire_cluster_last_sample_read_ops 0
solidfire_cluster_last_sample_write_bytes 0
solidfire_cluster_last_sample_write_ops 0
solidfire_cluster_latency_seconds 0
solidfire_cluster_max_async_result_id 47
solidfire_cluster_max_iops 3000
solidfire_cluster_max_metadata_over_provision_factor 5
solidfire_cluster_max_over_provisionable_space_bytes 1.855425871872e+13
Expand Down Expand Up @@ -369,6 +377,12 @@ solidfire_cluster_volume_async_result_active{type="DriveAdd"} 0
solidfire_cluster_volume_async_result_active{type="DriveRemoval"} 0
solidfire_cluster_volume_async_result_active{type="NotClone"} 1
solidfire_cluster_volume_async_result_active{type="RtfiPendingNode"} 0
solidfire_cluster_volume_async_result{type="BulkVolume"} 0
solidfire_cluster_volume_async_result{type="Clone"} 5
solidfire_cluster_volume_async_result{type="DriveAdd"} 0
solidfire_cluster_volume_async_result{type="DriveRemoval"} 0
solidfire_cluster_volume_async_result{type="NotClone"} 2
solidfire_cluster_volume_async_result{type="RtfiPendingNode"} 0
solidfire_cluster_volume_bulk_volume_job_count 1
solidfire_cluster_write_bytes_total 1.21720639488e+11
solidfire_cluster_write_latency_seconds 0
Expand Down
14 changes: 7 additions & 7 deletions test/fixtures/ListAsyncResults.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"result": {
"asyncHandles": [
{
"asyncResultID": 47,
"asyncResultID": 43,
"completed": true,
"createTime": "2016-01-01T22:29:19Z",
"data": {
Expand All @@ -16,7 +16,7 @@
"success": true
},
{
"asyncResultID": 47,
"asyncResultID": 46,
"completed": true,
"createTime": "2016-01-01T22:29:19Z",
"data": {
Expand All @@ -29,7 +29,7 @@
"success": true
},
{
"asyncResultID": 47,
"asyncResultID": 45,
"completed": false,
"createTime": "2016-01-01T22:29:19Z",
"data": {
Expand All @@ -55,7 +55,7 @@
"success": true
},
{
"asyncResultID": 47,
"asyncResultID": 40,
"completed": true,
"createTime": "2016-01-01T22:29:19Z",
"data": {
Expand All @@ -68,7 +68,7 @@
"success": false
},
{
"asyncResultID": 47,
"asyncResultID": 42,
"completed": true,
"createTime": "2016-01-01T22:29:19Z",
"data": {
Expand All @@ -81,7 +81,7 @@
"success": true
},
{
"asyncResultID": 47,
"asyncResultID": 39,
"completed": false,
"createTime": "2016-01-01T22:29:19Z",
"data": {
Expand All @@ -95,4 +95,4 @@
}
]
}
}
}

0 comments on commit 718d3ed

Please sign in to comment.