Skip to content

Commit

Permalink
Backport chunk_refs_fetch_time to 2.9.x branch (#11891)
Browse files Browse the repository at this point in the history 
This backports #10362 into the 2.9.x
branch

I recently talked about this metric in a blogpost on query performance
not realizing it wasn't available in the latest release. Would like to
make it available sooner than 3.0

Signed-off-by: Edward Welch <[email protected]>
  • Loading branch information
@slim-bean
slim-bean authored Feb 12, 2024
1 parent 3b74b60 commit e874612
Show file tree
Hide file tree
Showing 10 changed files with 150 additions and 75 deletions.
1 change: 1 addition & 0 deletions pkg/logql/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ func RecordRangeAndInstantQueryMetrics(
"queue_time", logql_stats.ConvertSecondsToNanoseconds(stats.Summary.QueueTime),
"splits", stats.Summary.Splits,
"shards", stats.Summary.Shards,
"chunk_refs_fetch_time", stats.ChunkRefsFetchTime(),
"cache_chunk_req", stats.Caches.Chunk.EntriesRequested,
"cache_chunk_hit", stats.Caches.Chunk.EntriesFound,
"cache_chunk_bytes_stored", stats.Caches.Chunk.BytesSent,
Expand Down
11 changes: 11 additions & 0 deletions pkg/logqlmodel/stats/context.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@ func (s *Store) Merge(m Store) {
s.TotalChunksRef += m.TotalChunksRef
s.TotalChunksDownloaded += m.TotalChunksDownloaded
s.ChunksDownloadTime += m.ChunksDownloadTime
s.ChunkRefsFetchTime += m.ChunkRefsFetchTime
s.Chunk.HeadChunkBytes += m.Chunk.HeadChunkBytes
s.Chunk.HeadChunkStructuredMetadataBytes += m.Chunk.HeadChunkStructuredMetadataBytes
s.Chunk.HeadChunkLines += m.Chunk.HeadChunkLines
Expand Down Expand Up @@ -247,6 +248,10 @@ func (r Result) ChunksDownloadTime() time.Duration {
return time.Duration(r.Querier.Store.ChunksDownloadTime + r.Ingester.Store.ChunksDownloadTime)
}

func (r Result) ChunkRefsFetchTime() time.Duration {
return time.Duration(r.Querier.Store.ChunkRefsFetchTime + r.Ingester.Store.ChunkRefsFetchTime)
}

func (r Result) TotalDuplicates() int64 {
return r.Querier.Store.Chunk.TotalDuplicates + r.Ingester.Store.Chunk.TotalDuplicates
}
Expand Down Expand Up @@ -320,6 +325,10 @@ func (c *Context) AddChunksDownloadTime(i time.Duration) {
atomic.AddInt64(&c.store.ChunksDownloadTime, int64(i))
}

func (c *Context) AddChunkRefsFetchTime(i time.Duration) {
atomic.AddInt64(&c.store.ChunkRefsFetchTime, int64(i))
}

func (c *Context) AddChunksDownloaded(i int64) {
atomic.AddInt64(&c.store.TotalChunksDownloaded, i)
}
Expand Down Expand Up @@ -433,6 +442,7 @@ func (r Result) Log(log log.Logger) {
"Ingester.TotalChunksRef", r.Ingester.Store.TotalChunksRef,
"Ingester.TotalChunksDownloaded", r.Ingester.Store.TotalChunksDownloaded,
"Ingester.ChunksDownloadTime", time.Duration(r.Ingester.Store.ChunksDownloadTime),
"Ingester.ChunkRefsFetchTime", time.Duration(r.Ingester.Store.ChunkRefsFetchTime),
"Ingester.HeadChunkBytes", humanize.Bytes(uint64(r.Ingester.Store.Chunk.HeadChunkBytes)),
"Ingester.HeadChunkLines", r.Ingester.Store.Chunk.HeadChunkLines,
"Ingester.DecompressedBytes", humanize.Bytes(uint64(r.Ingester.Store.Chunk.DecompressedBytes)),
Expand All @@ -444,6 +454,7 @@ func (r Result) Log(log log.Logger) {
"Querier.TotalChunksRef", r.Querier.Store.TotalChunksRef,
"Querier.TotalChunksDownloaded", r.Querier.Store.TotalChunksDownloaded,
"Querier.ChunksDownloadTime", time.Duration(r.Querier.Store.ChunksDownloadTime),
"Querier.ChunkRefsFetchTime", time.Duration(r.Querier.Store.ChunkRefsFetchTime),
"Querier.HeadChunkBytes", humanize.Bytes(uint64(r.Querier.Store.Chunk.HeadChunkBytes)),
"Querier.HeadChunkLines", r.Querier.Store.Chunk.HeadChunkLines,
"Querier.DecompressedBytes", humanize.Bytes(uint64(r.Querier.Store.Chunk.DecompressedBytes)),
Expand Down
184 changes: 113 additions & 71 deletions pkg/logqlmodel/stats/stats.pb.go
View file

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions pkg/logqlmodel/stats/stats.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,9 @@ message Store {
(gogoproto.nullable) = false,
(gogoproto.jsontag) = "chunk"
];

// Time spent fetching chunk refs from index.
int64 chunkRefsFetchTime = 5 [(gogoproto.jsontag) = "chunkRefsFetchTime"];
}

message Chunk {
Expand Down
6 changes: 4 additions & 2 deletions pkg/querier/http_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ var (
},
"chunksDownloadTime": 0,
"totalChunksRef": 0,
"totalChunksDownloaded": 0
"totalChunksDownloaded": 0,
"chunkRefsFetchTime": 0
},
"totalBatches": 6,
"totalChunksMatched": 7,
Expand All @@ -63,7 +64,8 @@ var (
},
"chunksDownloadTime": 16,
"totalChunksRef": 17,
"totalChunksDownloaded": 18
"totalChunksDownloaded": 18,
"chunkRefsFetchTime": 0
}
},
"cache": {
Expand Down
7 changes: 5 additions & 2 deletions pkg/querier/queryrange/codec_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1349,7 +1349,8 @@ var (
},
"chunksDownloadTime": 0,
"totalChunksRef": 0,
"totalChunksDownloaded": 0
"totalChunksDownloaded": 0,
"chunkRefsFetchTime": 0
},
"totalBatches": 6,
"totalChunksMatched": 7,
Expand All @@ -1371,7 +1372,8 @@ var (
},
"chunksDownloadTime": 16,
"totalChunksRef": 17,
"totalChunksDownloaded": 18
"totalChunksDownloaded": 18,
"chunkRefsFetchTime": 19
}
},
"cache": {
Expand Down Expand Up @@ -1607,6 +1609,7 @@ var (
ChunksDownloadTime: 16,
TotalChunksRef: 17,
TotalChunksDownloaded: 18,
ChunkRefsFetchTime: 19,
},
},

Expand Down
2 changes: 2 additions & 0 deletions pkg/querier/queryrange/prometheus_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ var emptyStats = `"stats": {
"chunksDownloadTime": 0,
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunkRefsFetchTime": 0,
"chunk" :{
"compressedBytes": 0,
"decompressedBytes": 0,
Expand All @@ -40,6 +41,7 @@ var emptyStats = `"stats": {
"chunksDownloadTime": 0,
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunkRefsFetchTime": 0,
"chunk" :{
"compressedBytes": 0,
"decompressedBytes": 0,
Expand Down
3 changes: 3 additions & 0 deletions pkg/storage/store.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -379,7 +379,10 @@ func (s *store) lazyChunks(ctx context.Context, matchers []*labels.Matcher, from

stats := stats.FromContext(ctx)

start := time.Now()
chks, fetchers, err := s.GetChunkRefs(ctx, userID, from, through, matchers...)
stats.AddChunkRefsFetchTime(time.Since(start))

if err != nil {
return nil, err
}
Expand Down
2 changes: 2 additions & 0 deletions pkg/util/marshal/legacy/marshal_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ var queryTests = []struct {
"chunksDownloadTime": 0,
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunkRefsFetchTime": 0,
"chunk" :{
"compressedBytes": 0,
"decompressedBytes": 0,
Expand All @@ -87,6 +88,7 @@ var queryTests = []struct {
"chunksDownloadTime": 0,
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunkRefsFetchTime": 0,
"chunk" :{
"compressedBytes": 0,
"decompressedBytes": 0,
Expand Down
6 changes: 6 additions & 0 deletions pkg/util/marshal/marshal_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ var queryTests = []struct {
"chunksDownloadTime": 0,
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunkRefsFetchTime": 0,
"chunk" :{
"compressedBytes": 0,
"decompressedBytes": 0,
Expand All @@ -89,6 +90,7 @@ var queryTests = []struct {
"chunksDownloadTime": 0,
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunkRefsFetchTime": 0,
"chunk" :{
"compressedBytes": 0,
"decompressedBytes": 0,
Expand Down Expand Up @@ -221,6 +223,7 @@ var queryTests = []struct {
"chunksDownloadTime": 0,
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunkRefsFetchTime": 0,
"chunk" :{
"compressedBytes": 0,
"decompressedBytes": 0,
Expand All @@ -243,6 +246,7 @@ var queryTests = []struct {
"chunksDownloadTime": 0,
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunkRefsFetchTime": 0,
"chunk" :{
"compressedBytes": 0,
"decompressedBytes": 0,
Expand Down Expand Up @@ -396,6 +400,7 @@ var queryTests = []struct {
"chunksDownloadTime": 0,
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunkRefsFetchTime": 0,
"chunk" :{
"compressedBytes": 0,
"decompressedBytes": 0,
Expand All @@ -418,6 +423,7 @@ var queryTests = []struct {
"chunksDownloadTime": 0,
"totalChunksRef": 0,
"totalChunksDownloaded": 0,
"chunkRefsFetchTime": 0,
"chunk" :{
"compressedBytes": 0,
"decompressedBytes": 0,
Expand Down

0 comments on commit e874612

Please sign in to comment.