From 8339fcc39230ede0fa42f322976caf90ffd3faa0 Mon Sep 17 00:00:00 2001 From: Thomas Krampl Date: Mon, 15 Apr 2024 11:49:55 +0200 Subject: [PATCH] Utilization: When fetching specific data for team, group rows usable for cost When calculating trends and utilization, we want to include the overuse. This PR switches to return up to two rows when requesting data which previusly discarded in the case where usage was higher than request. The code is not pretty, but seems to provide better values in the case mentioned in #24 Fixes #24 --- internal/database/gensql/mock_querier.go | 14 +++--- internal/database/gensql/querier.go | 2 +- internal/database/gensql/resourceusage.sql.go | 44 +++++++++++----- internal/database/mock_database.go | 14 +++--- internal/database/queries/resourceusage.sql | 9 ++-- internal/database/resource_utilization.go | 4 +- internal/resourceusage/client.go | 50 +++++++++++++++---- 7 files changed, 95 insertions(+), 42 deletions(-) diff --git a/internal/database/gensql/mock_querier.go b/internal/database/gensql/mock_querier.go index 1640fd209..53aa30666 100644 --- a/internal/database/gensql/mock_querier.go +++ b/internal/database/gensql/mock_querier.go @@ -6112,23 +6112,23 @@ func (_c *MockQuerier_SpecificResourceUtilizationForApp_Call) RunAndReturn(run f } // SpecificResourceUtilizationForTeam provides a mock function with given fields: ctx, arg -func (_m *MockQuerier) SpecificResourceUtilizationForTeam(ctx context.Context, arg SpecificResourceUtilizationForTeamParams) (*SpecificResourceUtilizationForTeamRow, error) { +func (_m *MockQuerier) SpecificResourceUtilizationForTeam(ctx context.Context, arg SpecificResourceUtilizationForTeamParams) ([]*SpecificResourceUtilizationForTeamRow, error) { ret := _m.Called(ctx, arg) if len(ret) == 0 { panic("no return value specified for SpecificResourceUtilizationForTeam") } - var r0 *SpecificResourceUtilizationForTeamRow + var r0 []*SpecificResourceUtilizationForTeamRow var r1 error - if rf, ok := ret.Get(0).(func(context.Context, SpecificResourceUtilizationForTeamParams) (*SpecificResourceUtilizationForTeamRow, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, SpecificResourceUtilizationForTeamParams) ([]*SpecificResourceUtilizationForTeamRow, error)); ok { return rf(ctx, arg) } - if rf, ok := ret.Get(0).(func(context.Context, SpecificResourceUtilizationForTeamParams) *SpecificResourceUtilizationForTeamRow); ok { + if rf, ok := ret.Get(0).(func(context.Context, SpecificResourceUtilizationForTeamParams) []*SpecificResourceUtilizationForTeamRow); ok { r0 = rf(ctx, arg) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*SpecificResourceUtilizationForTeamRow) + r0 = ret.Get(0).([]*SpecificResourceUtilizationForTeamRow) } } @@ -6160,12 +6160,12 @@ func (_c *MockQuerier_SpecificResourceUtilizationForTeam_Call) Run(run func(ctx return _c } -func (_c *MockQuerier_SpecificResourceUtilizationForTeam_Call) Return(_a0 *SpecificResourceUtilizationForTeamRow, _a1 error) *MockQuerier_SpecificResourceUtilizationForTeam_Call { +func (_c *MockQuerier_SpecificResourceUtilizationForTeam_Call) Return(_a0 []*SpecificResourceUtilizationForTeamRow, _a1 error) *MockQuerier_SpecificResourceUtilizationForTeam_Call { _c.Call.Return(_a0, _a1) return _c } -func (_c *MockQuerier_SpecificResourceUtilizationForTeam_Call) RunAndReturn(run func(context.Context, SpecificResourceUtilizationForTeamParams) (*SpecificResourceUtilizationForTeamRow, error)) *MockQuerier_SpecificResourceUtilizationForTeam_Call { +func (_c *MockQuerier_SpecificResourceUtilizationForTeam_Call) RunAndReturn(run func(context.Context, SpecificResourceUtilizationForTeamParams) ([]*SpecificResourceUtilizationForTeamRow, error)) *MockQuerier_SpecificResourceUtilizationForTeam_Call { _c.Call.Return(run) return _c } diff --git a/internal/database/gensql/querier.go b/internal/database/gensql/querier.go index 80cfe4e14..992b61591 100644 --- a/internal/database/gensql/querier.go +++ b/internal/database/gensql/querier.go @@ -142,7 +142,7 @@ type Querier interface { SpecificResourceUtilizationForApp(ctx context.Context, arg SpecificResourceUtilizationForAppParams) (*SpecificResourceUtilizationForAppRow, error) // SpecificResourceUtilizationForTeam will return resource utilization for a team at a specific timestamp. Applications // with a usage greater than request will be ignored. - SpecificResourceUtilizationForTeam(ctx context.Context, arg SpecificResourceUtilizationForTeamParams) (*SpecificResourceUtilizationForTeamRow, error) + SpecificResourceUtilizationForTeam(ctx context.Context, arg SpecificResourceUtilizationForTeamParams) ([]*SpecificResourceUtilizationForTeamRow, error) TeamExists(ctx context.Context, argSlug slug.Slug) (bool, error) UpdateTeam(ctx context.Context, arg UpdateTeamParams) (*Team, error) UpdateTeamExternalReferences(ctx context.Context, arg UpdateTeamExternalReferencesParams) (*Team, error) diff --git a/internal/database/gensql/resourceusage.sql.go b/internal/database/gensql/resourceusage.sql.go index ecb6b791c..5c7e62598 100644 --- a/internal/database/gensql/resourceusage.sql.go +++ b/internal/database/gensql/resourceusage.sql.go @@ -339,20 +339,21 @@ func (q *Queries) SpecificResourceUtilizationForApp(ctx context.Context, arg Spe return &i, err } -const specificResourceUtilizationForTeam = `-- name: SpecificResourceUtilizationForTeam :one +const specificResourceUtilizationForTeam = `-- name: SpecificResourceUtilizationForTeam :many SELECT SUM(usage)::double precision AS usage, SUM(request)::double precision AS request, - timestamp + timestamp, + request > usage as usable_for_cost FROM resource_utilization_metrics WHERE team_slug = $1 AND resource_type = $2 AND timestamp = $3 - AND request > usage GROUP BY - timestamp + timestamp, usable_for_cost +ORDER BY usable_for_cost DESC ` type SpecificResourceUtilizationForTeamParams struct { @@ -362,16 +363,35 @@ type SpecificResourceUtilizationForTeamParams struct { } type SpecificResourceUtilizationForTeamRow struct { - Usage float64 - Request float64 - Timestamp pgtype.Timestamptz + Usage float64 + Request float64 + Timestamp pgtype.Timestamptz + UsableForCost bool } // SpecificResourceUtilizationForTeam will return resource utilization for a team at a specific timestamp. Applications // with a usage greater than request will be ignored. -func (q *Queries) SpecificResourceUtilizationForTeam(ctx context.Context, arg SpecificResourceUtilizationForTeamParams) (*SpecificResourceUtilizationForTeamRow, error) { - row := q.db.QueryRow(ctx, specificResourceUtilizationForTeam, arg.TeamSlug, arg.ResourceType, arg.Timestamp) - var i SpecificResourceUtilizationForTeamRow - err := row.Scan(&i.Usage, &i.Request, &i.Timestamp) - return &i, err +func (q *Queries) SpecificResourceUtilizationForTeam(ctx context.Context, arg SpecificResourceUtilizationForTeamParams) ([]*SpecificResourceUtilizationForTeamRow, error) { + rows, err := q.db.Query(ctx, specificResourceUtilizationForTeam, arg.TeamSlug, arg.ResourceType, arg.Timestamp) + if err != nil { + return nil, err + } + defer rows.Close() + items := []*SpecificResourceUtilizationForTeamRow{} + for rows.Next() { + var i SpecificResourceUtilizationForTeamRow + if err := rows.Scan( + &i.Usage, + &i.Request, + &i.Timestamp, + &i.UsableForCost, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil } diff --git a/internal/database/mock_database.go b/internal/database/mock_database.go index e28b0e0af..32cfd1d0b 100644 --- a/internal/database/mock_database.go +++ b/internal/database/mock_database.go @@ -5767,23 +5767,23 @@ func (_c *MockDatabase_SpecificResourceUtilizationForApp_Call) RunAndReturn(run } // SpecificResourceUtilizationForTeam provides a mock function with given fields: ctx, teamSlug, resourceType, timestamp -func (_m *MockDatabase) SpecificResourceUtilizationForTeam(ctx context.Context, teamSlug slug.Slug, resourceType gensql.ResourceType, timestamp pgtype.Timestamptz) (*gensql.SpecificResourceUtilizationForTeamRow, error) { +func (_m *MockDatabase) SpecificResourceUtilizationForTeam(ctx context.Context, teamSlug slug.Slug, resourceType gensql.ResourceType, timestamp pgtype.Timestamptz) ([]*gensql.SpecificResourceUtilizationForTeamRow, error) { ret := _m.Called(ctx, teamSlug, resourceType, timestamp) if len(ret) == 0 { panic("no return value specified for SpecificResourceUtilizationForTeam") } - var r0 *gensql.SpecificResourceUtilizationForTeamRow + var r0 []*gensql.SpecificResourceUtilizationForTeamRow var r1 error - if rf, ok := ret.Get(0).(func(context.Context, slug.Slug, gensql.ResourceType, pgtype.Timestamptz) (*gensql.SpecificResourceUtilizationForTeamRow, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, slug.Slug, gensql.ResourceType, pgtype.Timestamptz) ([]*gensql.SpecificResourceUtilizationForTeamRow, error)); ok { return rf(ctx, teamSlug, resourceType, timestamp) } - if rf, ok := ret.Get(0).(func(context.Context, slug.Slug, gensql.ResourceType, pgtype.Timestamptz) *gensql.SpecificResourceUtilizationForTeamRow); ok { + if rf, ok := ret.Get(0).(func(context.Context, slug.Slug, gensql.ResourceType, pgtype.Timestamptz) []*gensql.SpecificResourceUtilizationForTeamRow); ok { r0 = rf(ctx, teamSlug, resourceType, timestamp) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*gensql.SpecificResourceUtilizationForTeamRow) + r0 = ret.Get(0).([]*gensql.SpecificResourceUtilizationForTeamRow) } } @@ -5817,12 +5817,12 @@ func (_c *MockDatabase_SpecificResourceUtilizationForTeam_Call) Run(run func(ctx return _c } -func (_c *MockDatabase_SpecificResourceUtilizationForTeam_Call) Return(_a0 *gensql.SpecificResourceUtilizationForTeamRow, _a1 error) *MockDatabase_SpecificResourceUtilizationForTeam_Call { +func (_c *MockDatabase_SpecificResourceUtilizationForTeam_Call) Return(_a0 []*gensql.SpecificResourceUtilizationForTeamRow, _a1 error) *MockDatabase_SpecificResourceUtilizationForTeam_Call { _c.Call.Return(_a0, _a1) return _c } -func (_c *MockDatabase_SpecificResourceUtilizationForTeam_Call) RunAndReturn(run func(context.Context, slug.Slug, gensql.ResourceType, pgtype.Timestamptz) (*gensql.SpecificResourceUtilizationForTeamRow, error)) *MockDatabase_SpecificResourceUtilizationForTeam_Call { +func (_c *MockDatabase_SpecificResourceUtilizationForTeam_Call) RunAndReturn(run func(context.Context, slug.Slug, gensql.ResourceType, pgtype.Timestamptz) ([]*gensql.SpecificResourceUtilizationForTeamRow, error)) *MockDatabase_SpecificResourceUtilizationForTeam_Call { _c.Call.Return(run) return _c } diff --git a/internal/database/queries/resourceusage.sql b/internal/database/queries/resourceusage.sql index 1f7a1bb2c..4e3ec97bb 100644 --- a/internal/database/queries/resourceusage.sql +++ b/internal/database/queries/resourceusage.sql @@ -106,20 +106,21 @@ WHERE -- SpecificResourceUtilizationForTeam will return resource utilization for a team at a specific timestamp. Applications -- with a usage greater than request will be ignored. --- name: SpecificResourceUtilizationForTeam :one +-- name: SpecificResourceUtilizationForTeam :many SELECT SUM(usage)::double precision AS usage, SUM(request)::double precision AS request, - timestamp + timestamp, + request > usage as usable_for_cost FROM resource_utilization_metrics WHERE team_slug = @team_slug AND resource_type = @resource_type AND timestamp = @timestamp - AND request > usage GROUP BY - timestamp; + timestamp, usable_for_cost +ORDER BY usable_for_cost DESC; -- AverageResourceUtilizationForTeam will return the average resource utilization for a team for a week. -- name: AverageResourceUtilizationForTeam :one diff --git a/internal/database/resource_utilization.go b/internal/database/resource_utilization.go index 39ad14ea1..0b0fcf83c 100644 --- a/internal/database/resource_utilization.go +++ b/internal/database/resource_utilization.go @@ -18,7 +18,7 @@ type ResourceUtilizationRepo interface { ResourceUtilizationRangeForTeam(ctx context.Context, teamSlug slug.Slug) (*gensql.ResourceUtilizationRangeForTeamRow, error) ResourceUtilizationUpsert(ctx context.Context, arg []gensql.ResourceUtilizationUpsertParams) *gensql.ResourceUtilizationUpsertBatchResults SpecificResourceUtilizationForApp(ctx context.Context, environment string, teamSlug slug.Slug, app string, resourceType gensql.ResourceType, timestamp pgtype.Timestamptz) (*gensql.SpecificResourceUtilizationForAppRow, error) - SpecificResourceUtilizationForTeam(ctx context.Context, teamSlug slug.Slug, resourceType gensql.ResourceType, timestamp pgtype.Timestamptz) (*gensql.SpecificResourceUtilizationForTeamRow, error) + SpecificResourceUtilizationForTeam(ctx context.Context, teamSlug slug.Slug, resourceType gensql.ResourceType, timestamp pgtype.Timestamptz) ([]*gensql.SpecificResourceUtilizationForTeamRow, error) } var _ ResourceUtilizationRepo = (*database)(nil) @@ -83,7 +83,7 @@ func (d *database) SpecificResourceUtilizationForApp(ctx context.Context, enviro }) } -func (d *database) SpecificResourceUtilizationForTeam(ctx context.Context, teamSlug slug.Slug, resourceType gensql.ResourceType, timestamp pgtype.Timestamptz) (*gensql.SpecificResourceUtilizationForTeamRow, error) { +func (d *database) SpecificResourceUtilizationForTeam(ctx context.Context, teamSlug slug.Slug, resourceType gensql.ResourceType, timestamp pgtype.Timestamptz) ([]*gensql.SpecificResourceUtilizationForTeamRow, error) { return d.querier.SpecificResourceUtilizationForTeam(ctx, gensql.SpecificResourceUtilizationForTeamParams{ TeamSlug: teamSlug, ResourceType: resourceType, diff --git a/internal/resourceusage/client.go b/internal/resourceusage/client.go index 9ac7ec2e7..167bdae85 100644 --- a/internal/resourceusage/client.go +++ b/internal/resourceusage/client.go @@ -165,8 +165,8 @@ func (c *client) CurrentResourceUtilizationForApp(ctx context.Context, env strin return &model.CurrentResourceUtilization{ Timestamp: timeRange.To.Time, - CPU: resourceUtilization(model.ResourceTypeCPU, cpu.Timestamp.Time.UTC(), cpu.Request, cpu.Usage), - Memory: resourceUtilization(model.ResourceTypeMemory, memory.Timestamp.Time.UTC(), memory.Request, memory.Usage), + CPU: resourceUtilization(model.ResourceTypeCPU, cpu.Timestamp.Time.UTC(), cpu.Request, cpu.Usage, cpu.Request, cpu.Usage), + Memory: resourceUtilization(model.ResourceTypeMemory, memory.Timestamp.Time.UTC(), memory.Request, memory.Usage, memory.Request, memory.Usage), }, nil } @@ -201,10 +201,13 @@ func (c *client) CurrentResourceUtilizationForTeam(ctx context.Context, team slu return nil, fmt.Errorf("fetching current memory: %w", err) } + sMem := joinSpecificRows(currentMemory) + sCpu := joinSpecificRows(currentCpu) + return &model.CurrentResourceUtilization{ Timestamp: timeRange.To.Time, - CPU: resourceUtilization(model.ResourceTypeCPU, currentCpu.Timestamp.Time.UTC(), currentCpu.Request, currentCpu.Usage), - Memory: resourceUtilization(model.ResourceTypeMemory, currentMemory.Timestamp.Time.UTC(), currentMemory.Request, currentMemory.Usage), + CPU: resourceUtilization(model.ResourceTypeCPU, sCpu.utcTime, sCpu.request, sCpu.usage, sCpu.costOnlyRequest, sCpu.costOnlyUsage), + Memory: resourceUtilization(model.ResourceTypeMemory, sMem.utcTime, sMem.request, sMem.usage, sMem.costOnlyRequest, sMem.costOnlyUsage), }, nil } @@ -288,7 +291,7 @@ func (c *client) resourceUtilizationForApp(ctx context.Context, resourceType mod utilizationMap := initUtilizationMap(s, e) for _, row := range rows { ts := row.Timestamp.Time.UTC() - utilizationMap[ts] = resourceUtilization(resourceType, ts, row.Request, row.Usage) + utilizationMap[ts] = resourceUtilization(resourceType, ts, row.Request, row.Usage, row.Request, row.Usage) } data := make([]*model.ResourceUtilization, 0) @@ -335,7 +338,7 @@ func (c *client) resourceUtilizationForTeam(ctx context.Context, resourceType mo utilizationMap := initUtilizationMap(s, e) for _, row := range rows { ts := row.Timestamp.Time.UTC() - utilizationMap[ts] = resourceUtilization(resourceType, ts, row.Request, row.Usage) + utilizationMap[ts] = resourceUtilization(resourceType, ts, row.Request, row.Usage, row.Request, row.Usage) } data := make([]*model.ResourceUtilization, 0) @@ -409,14 +412,14 @@ func getDateRange(from, to pgtype.Timestamptz) *model.ResourceUtilizationDateRan } // resourceUtilization will return a resource utilization model -func resourceUtilization(resource model.ResourceType, ts time.Time, request, usage float64) model.ResourceUtilization { +func resourceUtilization(resource model.ResourceType, ts time.Time, request, usage, costRequest, costUsage float64) model.ResourceUtilization { var utilization float64 if request > 0 { utilization = usage / request * 100 } - requestCost := costPerHour(resource.ToDatabaseEnum(), request) - usageCost := costPerHour(resource.ToDatabaseEnum(), usage) + requestCost := costPerHour(resource.ToDatabaseEnum(), costRequest) + usageCost := costPerHour(resource.ToDatabaseEnum(), costUsage) overageCostPerHour := requestCost - usageCost return model.ResourceUtilization{ @@ -449,3 +452,32 @@ func initUtilizationMap(start, end time.Time) map[time.Time]model.ResourceUtiliz } return utilization } + +type splitResource struct { + request float64 + usage float64 + costOnlyRequest float64 + costOnlyUsage float64 + utcTime time.Time +} + +func joinSpecificRows(r []*gensql.SpecificResourceUtilizationForTeamRow) splitResource { + var request, usage, costOnlyRequest, costOnlyUsage float64 + var utcTime time.Time + for _, row := range r { + utcTime = row.Timestamp.Time.UTC() + request += row.Request + usage += row.Usage + if row.UsableForCost { + costOnlyRequest += row.Request + costOnlyUsage += row.Usage + } + } + return splitResource{ + request: request, + usage: usage, + costOnlyRequest: costOnlyRequest, + costOnlyUsage: costOnlyUsage, + utcTime: utcTime, + } +}