From 88e545fc952d6ff55c61d079db920f00abc04865 Mon Sep 17 00:00:00 2001 From: Kaviraj Kanagaraj Date: Sun, 19 May 2024 17:40:25 +0200 Subject: [PATCH 01/41] feat(cache): Add `Cache-Control: no-cache` support for Loki instant queries. (#12896) Signed-off-by: Kaviraj --- cmd/logcli/main.go | 1 + pkg/chunkenc/interface.go | 2 +- pkg/logcli/client/client.go | 15 +- pkg/logql/evaluator.go | 76 +++- pkg/querier/queryrange/codec.go | 48 +- pkg/querier/queryrange/codec_test.go | 49 +++ pkg/querier/queryrange/downstreamer.go | 6 +- pkg/querier/queryrange/downstreamer_test.go | 80 ++++ pkg/querier/queryrange/queryrange.pb.go | 414 +++++++++++------- pkg/querier/queryrange/queryrange.proto | 3 + .../chunk/cache/resultscache/cache_test.go | 99 +++++ 11 files changed, 626 insertions(+), 167 deletions(-) diff --git a/cmd/logcli/main.go b/cmd/logcli/main.go index e4e04da8d665..08fbcc195ba6 100644 --- a/cmd/logcli/main.go +++ b/cmd/logcli/main.go @@ -475,6 +475,7 @@ func newQueryClient(app *kingpin.Application) client.Client { app.Flag("key", "Path to the client certificate key. Can also be set using LOKI_CLIENT_KEY_PATH env var.").Default("").Envar("LOKI_CLIENT_KEY_PATH").StringVar(&client.TLSConfig.KeyFile) app.Flag("org-id", "adds X-Scope-OrgID to API requests for representing tenant ID. Useful for requesting tenant data when bypassing an auth gateway. Can also be set using LOKI_ORG_ID env var.").Default("").Envar("LOKI_ORG_ID").StringVar(&client.OrgID) app.Flag("query-tags", "adds X-Query-Tags http header to API requests. This header value will be part of `metrics.go` statistics. Useful for tracking the query. Can also be set using LOKI_QUERY_TAGS env var.").Default("").Envar("LOKI_QUERY_TAGS").StringVar(&client.QueryTags) + app.Flag("nocache", "adds Cache-Control: no-cache http header to API requests. Can also be set using LOKI_NO_CACHE env var.").Default("false").Envar("LOKI_NO_CACHE").BoolVar(&client.NoCache) app.Flag("bearer-token", "adds the Authorization header to API requests for authentication purposes. Can also be set using LOKI_BEARER_TOKEN env var.").Default("").Envar("LOKI_BEARER_TOKEN").StringVar(&client.BearerToken) app.Flag("bearer-token-file", "adds the Authorization header to API requests for authentication purposes. Can also be set using LOKI_BEARER_TOKEN_FILE env var.").Default("").Envar("LOKI_BEARER_TOKEN_FILE").StringVar(&client.BearerTokenFile) app.Flag("retries", "How many times to retry each query when getting an error response from Loki. Can also be set using LOKI_CLIENT_RETRIES env var.").Default("0").Envar("LOKI_CLIENT_RETRIES").IntVar(&client.Retries) diff --git a/pkg/chunkenc/interface.go b/pkg/chunkenc/interface.go index 8d6f5e1e8dd6..3825a6520af5 100644 --- a/pkg/chunkenc/interface.go +++ b/pkg/chunkenc/interface.go @@ -24,7 +24,7 @@ var ( ) type errTooFarBehind struct { - // original timestmap of the entry itself. + // original timestamp of the entry itself. entryTs time.Time // cutoff is the oldest acceptable timstamp of the `stream` that entry belongs to. diff --git a/pkg/logcli/client/client.go b/pkg/logcli/client/client.go index e417ccfa3ce5..0c7880d62257 100644 --- a/pkg/logcli/client/client.go +++ b/pkg/logcli/client/client.go @@ -39,6 +39,12 @@ const ( volumeRangePath = "/loki/api/v1/index/volume_range" detectedFieldsPath = "/loki/api/v1/detected_fields" defaultAuthHeader = "Authorization" + + // HTTP header keys + HTTPScopeOrgID = "X-Scope-OrgID" + HTTPQueryTags = "X-Query-Tags" + HTTPCacheControl = "Cache-Control" + HTTPCacheControlNoCache = "no-cache" ) var userAgent = fmt.Sprintf("loki-logcli/%s", build.Version) @@ -77,6 +83,7 @@ type DefaultClient struct { BearerTokenFile string Retries int QueryTags string + NoCache bool AuthHeader string ProxyURL string BackoffConfig BackoffConfig @@ -372,11 +379,15 @@ func (c *DefaultClient) getHTTPRequestHeader() (http.Header, error) { h.Set("User-Agent", userAgent) if c.OrgID != "" { - h.Set("X-Scope-OrgID", c.OrgID) + h.Set(HTTPScopeOrgID, c.OrgID) + } + + if c.NoCache { + h.Set(HTTPCacheControl, HTTPCacheControlNoCache) } if c.QueryTags != "" { - h.Set("X-Query-Tags", c.QueryTags) + h.Set(HTTPQueryTags, c.QueryTags) } if (c.Username != "" || c.Password != "") && (len(c.BearerToken) > 0 || len(c.BearerTokenFile) > 0) { diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index ff887ff9b752..e50d8739c30a 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -18,6 +18,7 @@ import ( "github.com/grafana/loki/v3/pkg/logql/syntax" "github.com/grafana/loki/v3/pkg/logqlmodel" "github.com/grafana/loki/v3/pkg/querier/plan" + "github.com/grafana/loki/v3/pkg/storage/chunk/cache/resultscache" "github.com/grafana/loki/v3/pkg/util" ) @@ -42,6 +43,7 @@ type Params interface { Shards() []string GetExpression() syntax.Expr GetStoreChunks() *logproto.ChunkRefGroup + CachingOptions() resultscache.CachingOptions } func NewLiteralParams( @@ -52,22 +54,70 @@ func NewLiteralParams( limit uint32, shards []string, storeChunks *logproto.ChunkRefGroup, +) (LiteralParams, error) { + return newLiteralParams( + qs, + start, + end, + step, + interval, + direction, + limit, + shards, + storeChunks, + resultscache.CachingOptions{}, + ) +} + +func NewLiteralParamsWithCaching( + qs string, + start, end time.Time, + step, interval time.Duration, + direction logproto.Direction, + limit uint32, + shards []string, + storeChunks *logproto.ChunkRefGroup, + cachingOptions resultscache.CachingOptions, +) (LiteralParams, error) { + return newLiteralParams( + qs, + start, + end, + step, + interval, + direction, + limit, + shards, + storeChunks, + cachingOptions, + ) +} + +func newLiteralParams( + qs string, + start, end time.Time, + step, interval time.Duration, + direction logproto.Direction, + limit uint32, + shards []string, + storeChunks *logproto.ChunkRefGroup, + cachingOptions resultscache.CachingOptions, ) (LiteralParams, error) { p := LiteralParams{ - queryString: qs, - start: start, - end: end, - step: step, - interval: interval, - direction: direction, - limit: limit, - shards: shards, - storeChunks: storeChunks, + queryString: qs, + start: start, + end: end, + step: step, + interval: interval, + direction: direction, + limit: limit, + shards: shards, + storeChunks: storeChunks, + cachingOptions: cachingOptions, } var err error p.queryExpr, err = syntax.ParseExpr(qs) return p, err - } // LiteralParams impls Params @@ -80,6 +130,7 @@ type LiteralParams struct { shards []string queryExpr syntax.Expr storeChunks *logproto.ChunkRefGroup + cachingOptions resultscache.CachingOptions } func (p LiteralParams) Copy() LiteralParams { return p } @@ -114,6 +165,11 @@ func (p LiteralParams) Shards() []string { return p.shards } // StoreChunks impls Params func (p LiteralParams) GetStoreChunks() *logproto.ChunkRefGroup { return p.storeChunks } +// CachingOptions returns whether Loki query created from this params should be cached. +func (p LiteralParams) CachingOptions() resultscache.CachingOptions { + return p.cachingOptions +} + // GetRangeType returns whether a query is an instant query or range query func GetRangeType(q Params) QueryRangeType { if q.Start() == q.End() && q.Step() == 0 { diff --git a/pkg/querier/queryrange/codec.go b/pkg/querier/queryrange/codec.go index 01ff8772a4c7..7bab6f6c5d05 100644 --- a/pkg/querier/queryrange/codec.go +++ b/pkg/querier/queryrange/codec.go @@ -42,6 +42,11 @@ import ( "github.com/grafana/loki/v3/pkg/util/querylimits" ) +const ( + cacheControlHeader = "Cache-Control" + noCacheVal = "no-cache" +) + var DefaultCodec = &Codec{} type Codec struct{} @@ -95,8 +100,6 @@ func (r *LokiRequest) LogToSpan(sp opentracing.Span) { ) } -func (*LokiRequest) GetCachingOptions() (res queryrangebase.CachingOptions) { return } - func (r *LokiInstantRequest) GetStep() int64 { return 0 } @@ -142,8 +145,6 @@ func (r *LokiInstantRequest) LogToSpan(sp opentracing.Span) { ) } -func (*LokiInstantRequest) GetCachingOptions() (res queryrangebase.CachingOptions) { return } - func (r *LokiSeriesRequest) GetEnd() time.Time { return r.EndTs } @@ -329,13 +330,18 @@ func (Codec) DecodeRequest(_ context.Context, r *http.Request, _ []string) (quer return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) } + disableCacheReq := false + + if strings.ToLower(strings.TrimSpace(r.Header.Get(cacheControlHeader))) == noCacheVal { + disableCacheReq = true + } + switch op := getOperation(r.URL.Path); op { case QueryRangeOp: req, err := parseRangeQuery(r) if err != nil { return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) } - return req, nil case InstantQueryOp: req, err := parseInstantQuery(r) @@ -343,6 +349,10 @@ func (Codec) DecodeRequest(_ context.Context, r *http.Request, _ []string) (quer return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) } + req.CachingOptions = queryrangebase.CachingOptions{ + Disabled: disableCacheReq, + } + return req, nil case SeriesOp: req, err := loghttp.ParseAndValidateSeriesQuery(r) @@ -1808,6 +1818,10 @@ func (p paramsRangeWrapper) Shards() []string { return p.GetShards() } +func (p paramsRangeWrapper) CachingOptions() resultscache.CachingOptions { + return resultscache.CachingOptions{} +} + type paramsInstantWrapper struct { *LokiInstantRequest } @@ -1840,6 +1854,10 @@ func (p paramsInstantWrapper) Shards() []string { return p.GetShards() } +func (p paramsInstantWrapper) CachingOptions() resultscache.CachingOptions { + return p.LokiInstantRequest.CachingOptions +} + type paramsSeriesWrapper struct { *LokiSeriesRequest } @@ -1876,6 +1894,10 @@ func (p paramsSeriesWrapper) GetStoreChunks() *logproto.ChunkRefGroup { return nil } +func (p paramsSeriesWrapper) CachingOptions() resultscache.CachingOptions { + return resultscache.CachingOptions{} +} + type paramsLabelWrapper struct { *LabelRequest } @@ -1912,6 +1934,10 @@ func (p paramsLabelWrapper) GetStoreChunks() *logproto.ChunkRefGroup { return nil } +func (p paramsLabelWrapper) CachingOptions() resultscache.CachingOptions { + return resultscache.CachingOptions{} +} + type paramsStatsWrapper struct { *logproto.IndexStatsRequest } @@ -1948,6 +1974,10 @@ func (p paramsStatsWrapper) GetStoreChunks() *logproto.ChunkRefGroup { return nil } +func (p paramsStatsWrapper) CachingOptions() resultscache.CachingOptions { + return resultscache.CachingOptions{} +} + type paramsDetectedFieldsWrapper struct { *DetectedFieldsRequest } @@ -2040,6 +2070,14 @@ func (p paramsDetectedFieldsWrapper) GetStoreChunks() *logproto.ChunkRefGroup { return nil } +func (p paramsDetectedLabelsWrapper) CachingOptions() resultscache.CachingOptions { + return resultscache.CachingOptions{} +} + +func (p paramsDetectedFieldsWrapper) CachingOptions() resultscache.CachingOptions { + return resultscache.CachingOptions{} +} + func httpResponseHeadersToPromResponseHeaders(httpHeaders http.Header) []queryrangebase.PrometheusResponseHeader { var promHeaders []queryrangebase.PrometheusResponseHeader for h, hv := range httpHeaders { diff --git a/pkg/querier/queryrange/codec_test.go b/pkg/querier/queryrange/codec_test.go index 2a67ac512bce..136379292238 100644 --- a/pkg/querier/queryrange/codec_test.go +++ b/pkg/querier/queryrange/codec_test.go @@ -219,6 +219,55 @@ func Test_codec_EncodeDecodeRequest(t *testing.T) { } } +func Test_codec_DecodeRequest_cacheHeader(t *testing.T) { + ctx := user.InjectOrgID(context.Background(), "1") + + tests := []struct { + name string + reqBuilder func() (*http.Request, error) + want queryrangebase.Request + }{ + { + "query_instant", + func() (*http.Request, error) { + req, err := http.NewRequest( + http.MethodGet, + fmt.Sprintf(`/v1/query?time=%d&query={foo="bar"}&limit=200&direction=FORWARD`, start.UnixNano()), + nil, + ) + if err == nil { + req.Header.Set(cacheControlHeader, noCacheVal) + } + return req, err + }, + &LokiInstantRequest{ + Query: `{foo="bar"}`, + Limit: 200, + Direction: logproto.FORWARD, + Path: "/v1/query", + TimeTs: start, + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`{foo="bar"}`), + }, + CachingOptions: queryrangebase.CachingOptions{ + Disabled: true, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req, err := tt.reqBuilder() + if err != nil { + t.Fatal(err) + } + got, err := DefaultCodec.DecodeRequest(ctx, req, nil) + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} + func Test_codec_DecodeResponse(t *testing.T) { tests := []struct { name string diff --git a/pkg/querier/queryrange/downstreamer.go b/pkg/querier/queryrange/downstreamer.go index cf1cfc36dc4a..9f946a3247e9 100644 --- a/pkg/querier/queryrange/downstreamer.go +++ b/pkg/querier/queryrange/downstreamer.go @@ -45,7 +45,8 @@ func ParamsToLokiRequest(params logql.Params) queryrangebase.Request { Plan: &plan.QueryPlan{ AST: params.GetExpression(), }, - StoreChunks: params.GetStoreChunks(), + StoreChunks: params.GetStoreChunks(), + CachingOptions: params.CachingOptions(), } } return &LokiRequest{ @@ -61,7 +62,8 @@ func ParamsToLokiRequest(params logql.Params) queryrangebase.Request { Plan: &plan.QueryPlan{ AST: params.GetExpression(), }, - StoreChunks: params.GetStoreChunks(), + StoreChunks: params.GetStoreChunks(), + CachingOptions: params.CachingOptions(), } } diff --git a/pkg/querier/queryrange/downstreamer_test.go b/pkg/querier/queryrange/downstreamer_test.go index 979cc5a04e00..a10913f22373 100644 --- a/pkg/querier/queryrange/downstreamer_test.go +++ b/pkg/querier/queryrange/downstreamer_test.go @@ -22,7 +22,9 @@ import ( "github.com/grafana/loki/v3/pkg/logql/syntax" "github.com/grafana/loki/v3/pkg/logqlmodel" "github.com/grafana/loki/v3/pkg/logqlmodel/stats" + "github.com/grafana/loki/v3/pkg/querier/plan" "github.com/grafana/loki/v3/pkg/querier/queryrange/queryrangebase" + "github.com/grafana/loki/v3/pkg/storage/chunk/cache/resultscache" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) @@ -328,6 +330,84 @@ func TestInstanceFor(t *testing.T) { ensureParallelism(t, in, in.parallelism) } +func TestParamsToLokiRequest(t *testing.T) { + // Usually, queryrangebase.Request converted into Params and passed to downstream engine + // And converted back to queryrangebase.Request from the params before executing those queries. + // This test makes sure, we don't loose `CachingOption` during this transformation. + + ts := time.Now() + qs := `sum(rate({foo="bar"}[2h] offset 1h))` + + cases := []struct { + name string + caching resultscache.CachingOptions + expReq queryrangebase.Request + }{ + { + "instant-query-cache-enabled", + resultscache.CachingOptions{ + Disabled: false, + }, + &LokiInstantRequest{ + Query: qs, + Limit: 1000, + TimeTs: ts, + Direction: logproto.BACKWARD, + Path: "/loki/api/v1/query", + Shards: nil, + StoreChunks: nil, + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(qs), + }, + CachingOptions: resultscache.CachingOptions{ + Disabled: false, + }, + }, + }, + { + "instant-query-cache-disabled", + resultscache.CachingOptions{ + Disabled: true, + }, + &LokiInstantRequest{ + Query: qs, + Limit: 1000, + TimeTs: ts, + Direction: logproto.BACKWARD, + Path: "/loki/api/v1/query", + Shards: nil, + StoreChunks: nil, + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(qs), + }, + CachingOptions: resultscache.CachingOptions{ + Disabled: true, + }, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + params, err := logql.NewLiteralParamsWithCaching( + `sum(rate({foo="bar"}[2h] offset 1h))`, + ts, + ts, + 0, + 0, + logproto.BACKWARD, + 1000, + nil, + nil, + tc.caching, + ) + require.NoError(t, err) + req := ParamsToLokiRequest(params) + require.Equal(t, tc.expReq, req) + }) + } +} + func TestInstanceDownstream(t *testing.T) { t.Run("Downstream simple query", func(t *testing.T) { ts := time.Unix(1, 0) diff --git a/pkg/querier/queryrange/queryrange.pb.go b/pkg/querier/queryrange/queryrange.pb.go index e295fac92c6d..ae2dddee539a 100644 --- a/pkg/querier/queryrange/queryrange.pb.go +++ b/pkg/querier/queryrange/queryrange.pb.go @@ -21,6 +21,7 @@ import ( queryrangebase "github.com/grafana/loki/v3/pkg/querier/queryrange/queryrangebase" _ "github.com/grafana/loki/v3/pkg/querier/queryrange/queryrangebase/definitions" github_com_grafana_loki_v3_pkg_querier_queryrange_queryrangebase_definitions "github.com/grafana/loki/v3/pkg/querier/queryrange/queryrangebase/definitions" + resultscache "github.com/grafana/loki/v3/pkg/storage/chunk/cache/resultscache" io "io" math "math" math_bits "math/bits" @@ -54,7 +55,8 @@ type LokiRequest struct { Plan *github_com_grafana_loki_v3_pkg_querier_plan.QueryPlan `protobuf:"bytes,10,opt,name=plan,proto3,customtype=github.com/grafana/loki/v3/pkg/querier/plan.QueryPlan" json:"plan,omitempty"` // If populated, these represent the chunk references that the querier should // use to fetch the data, plus any other chunks reported by ingesters. - StoreChunks *logproto.ChunkRefGroup `protobuf:"bytes,11,opt,name=storeChunks,proto3" json:"storeChunks"` + StoreChunks *logproto.ChunkRefGroup `protobuf:"bytes,11,opt,name=storeChunks,proto3" json:"storeChunks"` + CachingOptions resultscache.CachingOptions `protobuf:"bytes,12,opt,name=cachingOptions,proto3" json:"cachingOptions"` } func (m *LokiRequest) Reset() { *m = LokiRequest{} } @@ -159,6 +161,13 @@ func (m *LokiRequest) GetStoreChunks() *logproto.ChunkRefGroup { return nil } +func (m *LokiRequest) GetCachingOptions() resultscache.CachingOptions { + if m != nil { + return m.CachingOptions + } + return resultscache.CachingOptions{} +} + type LokiInstantRequest struct { Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` Limit uint32 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` @@ -169,7 +178,8 @@ type LokiInstantRequest struct { Plan *github_com_grafana_loki_v3_pkg_querier_plan.QueryPlan `protobuf:"bytes,7,opt,name=plan,proto3,customtype=github.com/grafana/loki/v3/pkg/querier/plan.QueryPlan" json:"plan,omitempty"` // If populated, these represent the chunk references that the querier should // use to fetch the data, plus any other chunks reported by ingesters. - StoreChunks *logproto.ChunkRefGroup `protobuf:"bytes,8,opt,name=storeChunks,proto3" json:"storeChunks"` + StoreChunks *logproto.ChunkRefGroup `protobuf:"bytes,8,opt,name=storeChunks,proto3" json:"storeChunks"` + CachingOptions resultscache.CachingOptions `protobuf:"bytes,9,opt,name=cachingOptions,proto3" json:"cachingOptions"` } func (m *LokiInstantRequest) Reset() { *m = LokiInstantRequest{} } @@ -253,6 +263,13 @@ func (m *LokiInstantRequest) GetStoreChunks() *logproto.ChunkRefGroup { return nil } +func (m *LokiInstantRequest) GetCachingOptions() resultscache.CachingOptions { + if m != nil { + return m.CachingOptions + } + return resultscache.CachingOptions{} +} + type Plan struct { Raw []byte `protobuf:"bytes,1,opt,name=raw,proto3" json:"raw,omitempty"` } @@ -1471,125 +1488,128 @@ func init() { } var fileDescriptor_51b9d53b40d11902 = []byte{ - // 1879 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x59, 0xcb, 0x6f, 0x1c, 0x49, - 0x19, 0x9f, 0x9e, 0xa7, 0xe7, 0xf3, 0x23, 0xa6, 0x62, 0xbc, 0x8d, 0x77, 0x77, 0x7a, 0x18, 0x89, - 0x5d, 0x83, 0x60, 0x86, 0xd8, 0xbb, 0x61, 0xd7, 0x84, 0x68, 0xd3, 0xeb, 0x04, 0x67, 0xc9, 0x42, - 0xb6, 0x6d, 0x71, 0xe0, 0x82, 0xca, 0x9e, 0xf2, 0x4c, 0xe3, 0x99, 0xee, 0x4e, 0x57, 0x8d, 0x13, - 0x4b, 0x08, 0xed, 0x3f, 0xb0, 0x62, 0x4f, 0xfc, 0x09, 0x88, 0x1b, 0x17, 0x4e, 0x9c, 0x38, 0x86, - 0x03, 0x52, 0x8e, 0xab, 0x91, 0x68, 0x88, 0x83, 0x10, 0xf2, 0x69, 0x25, 0xae, 0x20, 0xa1, 0x7a, - 0x74, 0x4f, 0xd5, 0xcc, 0x98, 0x8c, 0x13, 0x09, 0xc9, 0xc0, 0xc5, 0x53, 0x8f, 0xef, 0xf7, 0x75, - 0xd5, 0xef, 0x7b, 0xd4, 0x57, 0x65, 0x78, 0x33, 0x3a, 0xea, 0xb4, 0x1e, 0x0c, 0x48, 0xec, 0x93, - 0x58, 0xfc, 0x9e, 0xc4, 0x38, 0xe8, 0x10, 0xad, 0xd9, 0x8c, 0xe2, 0x90, 0x85, 0x08, 0x46, 0x23, - 0x6b, 0x1b, 0x1d, 0x9f, 0x75, 0x07, 0xfb, 0xcd, 0x83, 0xb0, 0xdf, 0xea, 0x84, 0x9d, 0xb0, 0xd5, - 0x09, 0xc3, 0x4e, 0x8f, 0xe0, 0xc8, 0xa7, 0xaa, 0xd9, 0x8a, 0xa3, 0x83, 0x16, 0x65, 0x98, 0x0d, - 0xa8, 0xc4, 0xaf, 0xad, 0x70, 0x41, 0xd1, 0x14, 0x10, 0x35, 0xea, 0x28, 0x71, 0xd1, 0xdb, 0x1f, - 0x1c, 0xb6, 0x98, 0xdf, 0x27, 0x94, 0xe1, 0x7e, 0x94, 0x0a, 0xf0, 0xf5, 0xf5, 0xc2, 0x8e, 0x44, - 0xfa, 0x41, 0x9b, 0x3c, 0xea, 0x60, 0x46, 0x1e, 0xe2, 0x13, 0x25, 0xf0, 0xaa, 0x21, 0x90, 0x36, - 0xd4, 0xe4, 0x9a, 0x31, 0x19, 0x61, 0xc6, 0x48, 0x1c, 0xa8, 0xb9, 0x2f, 0x19, 0x73, 0xf4, 0x88, - 0xb0, 0x83, 0xae, 0x9a, 0xaa, 0xab, 0xa9, 0x07, 0xbd, 0x7e, 0xd8, 0x26, 0x3d, 0xb1, 0x11, 0x2a, - 0xff, 0x2a, 0x89, 0xab, 0x5c, 0x22, 0x1a, 0xd0, 0xae, 0xf8, 0xa3, 0x06, 0xdf, 0x7f, 0x2e, 0x97, - 0xfb, 0x98, 0x92, 0x56, 0x9b, 0x1c, 0xfa, 0x81, 0xcf, 0xfc, 0x30, 0xa0, 0x7a, 0x5b, 0x29, 0xb9, - 0x3e, 0x9b, 0x92, 0x71, 0xfb, 0x34, 0xfe, 0x59, 0x80, 0xf9, 0x7b, 0xe1, 0x91, 0xef, 0x91, 0x07, - 0x03, 0x42, 0x19, 0x5a, 0x81, 0x92, 0x90, 0xb1, 0xad, 0xba, 0xb5, 0x5e, 0xf5, 0x64, 0x87, 0x8f, - 0xf6, 0xfc, 0xbe, 0xcf, 0xec, 0x7c, 0xdd, 0x5a, 0x5f, 0xf4, 0x64, 0x07, 0x21, 0x28, 0x52, 0x46, - 0x22, 0xbb, 0x50, 0xb7, 0xd6, 0x0b, 0x9e, 0x68, 0xa3, 0x35, 0x98, 0xf3, 0x03, 0x46, 0xe2, 0x63, - 0xdc, 0xb3, 0xab, 0x62, 0x3c, 0xeb, 0xa3, 0x9b, 0x50, 0xa1, 0x0c, 0xc7, 0x6c, 0x8f, 0xda, 0xc5, - 0xba, 0xb5, 0x3e, 0xbf, 0xb1, 0xd6, 0x94, 0x76, 0x6c, 0xa6, 0x76, 0x6c, 0xee, 0xa5, 0x76, 0x74, - 0xe7, 0x1e, 0x27, 0x4e, 0xee, 0xd3, 0x3f, 0x39, 0x96, 0x97, 0x82, 0xd0, 0x16, 0x94, 0x48, 0xd0, - 0xde, 0xa3, 0x76, 0xe9, 0x02, 0x68, 0x09, 0x41, 0xd7, 0xa0, 0xda, 0xf6, 0x63, 0x72, 0xc0, 0x39, - 0xb3, 0xcb, 0x75, 0x6b, 0x7d, 0x69, 0xe3, 0x6a, 0x33, 0x33, 0xfb, 0x76, 0x3a, 0xe5, 0x8d, 0xa4, - 0xf8, 0xf6, 0x22, 0xcc, 0xba, 0x76, 0x45, 0x30, 0x21, 0xda, 0xa8, 0x01, 0x65, 0xda, 0xc5, 0x71, - 0x9b, 0xda, 0x73, 0xf5, 0xc2, 0x7a, 0xd5, 0x85, 0xb3, 0xc4, 0x51, 0x23, 0x9e, 0xfa, 0x45, 0x3f, - 0x86, 0x62, 0xd4, 0xc3, 0x81, 0x0d, 0x62, 0x95, 0xcb, 0x4d, 0x8d, 0xf3, 0xfb, 0x3d, 0x1c, 0xb8, - 0xef, 0x0e, 0x13, 0xe7, 0x6d, 0x3d, 0x14, 0x62, 0x7c, 0x88, 0x03, 0xdc, 0xea, 0x85, 0x47, 0x7e, - 0xeb, 0x78, 0xb3, 0xa5, 0x5b, 0x92, 0x2b, 0x6a, 0x7e, 0xc4, 0x15, 0x70, 0xa8, 0x27, 0x14, 0xa3, - 0x0f, 0x60, 0x9e, 0xb2, 0x30, 0x26, 0xef, 0x77, 0x07, 0xc1, 0x11, 0xb5, 0xe7, 0xc5, 0x77, 0x5e, - 0x19, 0xed, 0x46, 0x8c, 0x7b, 0xe4, 0xf0, 0xbb, 0x71, 0x38, 0x88, 0xdc, 0x2b, 0x67, 0x89, 0xa3, - 0xcb, 0x7b, 0x7a, 0xa7, 0xf1, 0x8b, 0x02, 0x20, 0x6e, 0xff, 0xbb, 0x01, 0x65, 0x38, 0x60, 0x2f, - 0xe2, 0x06, 0x37, 0xa0, 0xcc, 0xc3, 0x6f, 0x8f, 0x0a, 0x47, 0x98, 0xd5, 0x2e, 0x0a, 0x63, 0x1a, - 0xa6, 0x78, 0x21, 0xc3, 0x94, 0xa6, 0x1a, 0xa6, 0xfc, 0x5c, 0xc3, 0x54, 0xfe, 0x43, 0x86, 0x99, - 0x7b, 0x19, 0xc3, 0xd8, 0x50, 0xe4, 0x9a, 0xd1, 0x32, 0x14, 0x62, 0xfc, 0x50, 0xd8, 0x61, 0xc1, - 0xe3, 0xcd, 0xc6, 0x69, 0x11, 0x16, 0x64, 0xc8, 0xd2, 0x28, 0x0c, 0x28, 0xe1, 0x7b, 0xdf, 0x15, - 0x39, 0x53, 0x5a, 0x4b, 0xed, 0x5d, 0x8c, 0x78, 0x6a, 0x06, 0xbd, 0x07, 0xc5, 0x6d, 0xcc, 0xb0, - 0xb0, 0xdc, 0xfc, 0xc6, 0x8a, 0xbe, 0x77, 0xae, 0x8b, 0xcf, 0xb9, 0xab, 0xdc, 0x38, 0x67, 0x89, - 0xb3, 0xd4, 0xc6, 0x0c, 0x7f, 0x3d, 0xec, 0xfb, 0x8c, 0xf4, 0x23, 0x76, 0xe2, 0x09, 0x24, 0x7a, - 0x1b, 0xaa, 0xb7, 0xe3, 0x38, 0x8c, 0xf7, 0x4e, 0x22, 0x22, 0x2c, 0x5d, 0x75, 0x5f, 0x39, 0x4b, - 0x9c, 0xab, 0x24, 0x1d, 0xd4, 0x10, 0x23, 0x49, 0xf4, 0x55, 0x28, 0x89, 0x8e, 0xb0, 0x6d, 0xd5, - 0xbd, 0x7a, 0x96, 0x38, 0x57, 0x04, 0x44, 0x13, 0x97, 0x12, 0xa6, 0x2b, 0x94, 0x66, 0x72, 0x85, - 0xcc, 0x23, 0xcb, 0xba, 0x47, 0xda, 0x50, 0x39, 0x26, 0x31, 0xe5, 0x6a, 0x2a, 0x62, 0x3c, 0xed, - 0xa2, 0x5b, 0x00, 0x9c, 0x18, 0x9f, 0x32, 0xff, 0x20, 0x35, 0xd0, 0x62, 0x53, 0xa6, 0x68, 0x8f, - 0xd0, 0x41, 0x8f, 0xb9, 0x48, 0xb1, 0xa0, 0x09, 0x7a, 0x5a, 0x1b, 0xfd, 0xda, 0x82, 0xca, 0x0e, - 0xc1, 0x6d, 0x12, 0x53, 0xbb, 0x5a, 0x2f, 0xac, 0xcf, 0x6f, 0x7c, 0xa5, 0xa9, 0xe7, 0xe3, 0xfb, - 0x71, 0xd8, 0x27, 0xac, 0x4b, 0x06, 0x34, 0x35, 0x90, 0x94, 0x76, 0x83, 0x61, 0xe2, 0x90, 0x19, - 0xdd, 0x6b, 0xa6, 0x63, 0xe0, 0xdc, 0x4f, 0x9d, 0x25, 0x8e, 0xf5, 0x0d, 0x2f, 0x5d, 0x25, 0xda, - 0x80, 0xb9, 0x87, 0x38, 0x0e, 0xfc, 0xa0, 0x43, 0x6d, 0x10, 0xd1, 0xb1, 0x7a, 0x96, 0x38, 0x28, - 0x1d, 0xd3, 0x0c, 0x91, 0xc9, 0x35, 0xfe, 0x68, 0xc1, 0x17, 0xb8, 0x63, 0xec, 0xf2, 0xf5, 0x50, - 0x2d, 0x2d, 0xf4, 0x31, 0x3b, 0xe8, 0xda, 0x16, 0x57, 0xe3, 0xc9, 0x8e, 0x9e, 0xd7, 0xf3, 0x2f, - 0x95, 0xd7, 0x0b, 0x17, 0xcf, 0xeb, 0x69, 0x2e, 0x28, 0x4e, 0xcd, 0x05, 0xa5, 0xf3, 0x72, 0x41, - 0xe3, 0xe7, 0x2a, 0xef, 0xa5, 0xfb, 0xbb, 0x40, 0x28, 0xdd, 0xc9, 0x42, 0xa9, 0x20, 0x56, 0x9b, - 0x79, 0xa8, 0xd4, 0x75, 0xb7, 0x4d, 0x02, 0xe6, 0x1f, 0xfa, 0x24, 0x7e, 0x4e, 0x40, 0x69, 0x5e, - 0x5a, 0x30, 0xbd, 0x54, 0x77, 0xb1, 0xe2, 0xa5, 0x70, 0x31, 0x33, 0xae, 0x4a, 0x2f, 0x10, 0x57, - 0x8d, 0xbf, 0xe7, 0x61, 0x95, 0x5b, 0xe4, 0x1e, 0xde, 0x27, 0xbd, 0xef, 0xe3, 0xfe, 0x05, 0xad, - 0xf2, 0x86, 0x66, 0x95, 0xaa, 0x8b, 0xfe, 0xcf, 0xfa, 0x6c, 0xac, 0xff, 0xd2, 0x82, 0xb9, 0xf4, - 0x00, 0x40, 0x4d, 0x00, 0x09, 0x13, 0x39, 0x5e, 0x72, 0xbd, 0xc4, 0xc1, 0x71, 0x36, 0xea, 0x69, - 0x12, 0xe8, 0x27, 0x50, 0x96, 0x3d, 0x15, 0x0b, 0xda, 0x51, 0xb7, 0xcb, 0x62, 0x82, 0xfb, 0xb7, - 0xda, 0x38, 0x62, 0x24, 0x76, 0xdf, 0xe5, 0xab, 0x18, 0x26, 0xce, 0x9b, 0xe7, 0xb1, 0x94, 0xd6, - 0xc5, 0x0a, 0xc7, 0xed, 0x2b, 0xbf, 0xe9, 0xa9, 0x2f, 0x34, 0x3e, 0xb1, 0x60, 0x99, 0x2f, 0x94, - 0x53, 0x93, 0x39, 0xc6, 0x36, 0xcc, 0xc5, 0xaa, 0x2d, 0x96, 0x3b, 0xbf, 0xd1, 0x68, 0x9a, 0xb4, - 0x4e, 0xa1, 0xd2, 0x2d, 0x3e, 0x4e, 0x1c, 0xcb, 0xcb, 0x90, 0x68, 0xd3, 0xa0, 0x31, 0x3f, 0x8d, - 0x46, 0x0e, 0xc9, 0x19, 0xc4, 0xfd, 0x2e, 0x0f, 0xe8, 0x2e, 0xbf, 0x57, 0x70, 0xff, 0x1b, 0xb9, - 0xea, 0xa3, 0x89, 0x15, 0xbd, 0x36, 0x22, 0x65, 0x52, 0xde, 0xbd, 0x39, 0x4c, 0x9c, 0xad, 0xe7, - 0xf8, 0xce, 0xbf, 0xc1, 0x6b, 0xbb, 0xd0, 0xdd, 0x37, 0x7f, 0x19, 0xdc, 0xb7, 0xf1, 0x9b, 0x3c, - 0x2c, 0xfd, 0x30, 0xec, 0x0d, 0xfa, 0x24, 0xa3, 0x2f, 0x9a, 0xa0, 0xcf, 0x1e, 0xd1, 0x67, 0xca, - 0xba, 0x5b, 0xc3, 0xc4, 0xb9, 0x3e, 0x2b, 0x75, 0x26, 0xf6, 0x52, 0xd3, 0xf6, 0xd7, 0x3c, 0xac, - 0xec, 0x85, 0xd1, 0xf7, 0x76, 0xc5, 0xdd, 0x53, 0x4b, 0x93, 0xdd, 0x09, 0xf2, 0x56, 0x46, 0xe4, - 0x71, 0xc4, 0x87, 0x98, 0xc5, 0xfe, 0x23, 0xf7, 0xfa, 0x30, 0x71, 0x36, 0x66, 0x25, 0x6e, 0x84, - 0xbb, 0xcc, 0xa4, 0x19, 0x35, 0x50, 0x61, 0xc6, 0x1a, 0xe8, 0x1f, 0x79, 0x58, 0xfd, 0x68, 0x80, - 0x03, 0xe6, 0xf7, 0x88, 0x24, 0x3b, 0xa3, 0xfa, 0xa7, 0x13, 0x54, 0xd7, 0x46, 0x54, 0x9b, 0x18, - 0x45, 0xfa, 0x7b, 0xc3, 0xc4, 0xb9, 0x31, 0x2b, 0xe9, 0xd3, 0x34, 0xfc, 0xcf, 0xd1, 0xff, 0xdb, - 0x3c, 0x2c, 0xed, 0xca, 0xaa, 0x2d, 0xdd, 0xf8, 0xf1, 0x14, 0xda, 0xf5, 0xc7, 0x9d, 0x68, 0xbf, - 0x69, 0x22, 0x2e, 0x96, 0x24, 0x4c, 0xec, 0xa5, 0x4e, 0x12, 0x7f, 0xc8, 0xc3, 0xea, 0x36, 0x61, - 0xe4, 0x80, 0x91, 0xf6, 0x1d, 0x9f, 0xf4, 0x34, 0x12, 0x3f, 0xb6, 0x26, 0x58, 0xac, 0x6b, 0xd7, - 0xac, 0xa9, 0x20, 0xd7, 0x1d, 0x26, 0xce, 0xcd, 0x59, 0x79, 0x9c, 0xae, 0xe3, 0x52, 0xf3, 0xf9, - 0xfb, 0x3c, 0x7c, 0x51, 0x5e, 0xf7, 0xe5, 0x6b, 0xe0, 0x88, 0xce, 0x9f, 0x4d, 0xb0, 0xe9, 0xe8, - 0xa9, 0x60, 0x0a, 0xc4, 0xbd, 0x35, 0x4c, 0x9c, 0xef, 0xcc, 0x9e, 0x0b, 0xa6, 0xa8, 0xf8, 0xaf, - 0xf1, 0x4d, 0x51, 0xed, 0x5f, 0xd4, 0x37, 0x4d, 0xd0, 0x8b, 0xf9, 0xa6, 0xa9, 0xe3, 0x52, 0xf3, - 0xf9, 0x97, 0x32, 0x2c, 0x0a, 0x2f, 0xc9, 0x68, 0xfc, 0x1a, 0xa8, 0xeb, 0x91, 0xe2, 0x10, 0xa5, - 0x57, 0xea, 0x38, 0x3a, 0x68, 0xee, 0xaa, 0x8b, 0x93, 0x94, 0x40, 0xef, 0x40, 0x99, 0x8a, 0x8b, - 0xab, 0xaa, 0x7c, 0x6b, 0xe3, 0x6f, 0x43, 0xe6, 0x15, 0x79, 0x27, 0xe7, 0x29, 0x79, 0x74, 0x03, - 0xca, 0x3d, 0xc1, 0xa2, 0xba, 0xb8, 0x37, 0xc6, 0x91, 0x93, 0x57, 0x39, 0x8e, 0x96, 0x18, 0x74, - 0x1d, 0x4a, 0xa2, 0xc4, 0x56, 0x6f, 0xc1, 0xc6, 0x67, 0x27, 0x0b, 0xdd, 0x9d, 0x9c, 0x27, 0xc5, - 0xd1, 0x06, 0x14, 0xa3, 0x38, 0xec, 0xab, 0xeb, 0xce, 0x6b, 0xe3, 0xdf, 0xd4, 0xef, 0x07, 0x3b, - 0x39, 0x4f, 0xc8, 0xa2, 0xb7, 0xa0, 0x42, 0xc5, 0xc5, 0x82, 0x8a, 0x87, 0x22, 0x5e, 0x55, 0x8e, - 0xc1, 0x34, 0x48, 0x2a, 0x8a, 0xde, 0x82, 0xf2, 0xb1, 0x28, 0x1b, 0xd5, 0x8b, 0xe1, 0x9a, 0x0e, - 0x32, 0x0b, 0x4a, 0xbe, 0x2f, 0x29, 0x8b, 0xee, 0xc0, 0x02, 0x0b, 0xa3, 0xa3, 0xb4, 0x3a, 0x53, - 0x8f, 0x4c, 0x75, 0x1d, 0x3b, 0xad, 0x7a, 0xdb, 0xc9, 0x79, 0x06, 0x0e, 0xdd, 0x87, 0xe5, 0x07, - 0x46, 0x19, 0x40, 0xa8, 0x78, 0x51, 0x1f, 0xe3, 0x79, 0x7a, 0x81, 0xb2, 0x93, 0xf3, 0x26, 0xd0, - 0x68, 0x1b, 0x96, 0xa8, 0x71, 0xc2, 0xa9, 0x27, 0x6a, 0x63, 0x5f, 0xe6, 0x19, 0xb8, 0x93, 0xf3, - 0xc6, 0x30, 0xe8, 0x1e, 0x2c, 0xb5, 0x8d, 0xfc, 0xae, 0x1e, 0xa0, 0x8d, 0x55, 0x4d, 0x3f, 0x01, - 0xb8, 0x36, 0x13, 0x8b, 0x7e, 0x00, 0xcb, 0xd1, 0x58, 0x6e, 0xb3, 0x17, 0x84, 0xbe, 0x2f, 0x9b, - 0xbb, 0x9c, 0x92, 0x04, 0xf9, 0x26, 0xc7, 0xc1, 0xfa, 0xf2, 0x64, 0x88, 0xdb, 0x8b, 0xe7, 0x2f, - 0xcf, 0x4c, 0x02, 0xfa, 0xf2, 0xe4, 0x8c, 0x0b, 0xa3, 0x74, 0xd4, 0xf8, 0xa4, 0x0c, 0x0b, 0x2a, - 0xcc, 0xe4, 0x6b, 0xd8, 0xb7, 0xb2, 0xc8, 0x91, 0x51, 0xf6, 0xfa, 0x79, 0x91, 0x23, 0xc4, 0xb5, - 0xc0, 0xf9, 0x66, 0x16, 0x38, 0x32, 0xe4, 0x56, 0x47, 0x29, 0x4e, 0x7c, 0x57, 0x43, 0xa8, 0x60, - 0xd9, 0x4c, 0x83, 0x45, 0x46, 0xda, 0xab, 0xd3, 0xef, 0x94, 0x29, 0x4a, 0x45, 0xca, 0x16, 0x54, - 0x7c, 0xf9, 0xac, 0x3f, 0x2d, 0xc6, 0x26, 0x5f, 0xfd, 0xb9, 0xef, 0x2b, 0x00, 0xda, 0x1c, 0x45, - 0x4c, 0x49, 0x3d, 0x63, 0x4f, 0x44, 0x4c, 0x06, 0x4a, 0x03, 0xe6, 0x5a, 0x16, 0x30, 0xe5, 0xf1, - 0xa7, 0xef, 0x34, 0x5c, 0xb2, 0x8d, 0xa9, 0x68, 0xb9, 0x0d, 0x8b, 0xa9, 0x7f, 0x89, 0x29, 0x15, - 0x2e, 0xaf, 0x9f, 0x57, 0xd6, 0xa5, 0x78, 0x13, 0x85, 0xee, 0x4e, 0x38, 0x65, 0x75, 0xfc, 0x28, - 0x1e, 0x77, 0xc9, 0x54, 0xd3, 0xb8, 0x47, 0x7e, 0x00, 0x57, 0x46, 0x4e, 0x25, 0xd7, 0x04, 0x93, - 0x15, 0xbe, 0xe1, 0x8e, 0xa9, 0xaa, 0x71, 0xa0, 0xbe, 0x2c, 0xe5, 0x8c, 0xf3, 0xe7, 0x2d, 0x2b, - 0x75, 0xc5, 0x89, 0x65, 0xc9, 0x09, 0xb4, 0x03, 0x73, 0x7d, 0xc2, 0x70, 0x1b, 0x33, 0x6c, 0x57, - 0xc4, 0xb1, 0xf4, 0xc6, 0x44, 0x80, 0x28, 0x74, 0xf3, 0x43, 0x25, 0x78, 0x3b, 0x60, 0xf1, 0x89, - 0x7a, 0xbb, 0xc8, 0xd0, 0x6b, 0xdf, 0x86, 0x45, 0x43, 0x00, 0x2d, 0x43, 0xe1, 0x88, 0xa4, 0xff, - 0xea, 0xe1, 0x4d, 0xb4, 0x02, 0xa5, 0x63, 0xdc, 0x1b, 0x10, 0xe1, 0x9f, 0x55, 0x4f, 0x76, 0xb6, - 0xf2, 0xef, 0x58, 0x6e, 0x15, 0x2a, 0xb1, 0xfc, 0x8a, 0xdb, 0x79, 0xf2, 0xb4, 0x96, 0xfb, 0xec, - 0x69, 0x2d, 0xf7, 0xf9, 0xd3, 0x9a, 0xf5, 0xf1, 0x69, 0xcd, 0xfa, 0xd5, 0x69, 0xcd, 0x7a, 0x7c, - 0x5a, 0xb3, 0x9e, 0x9c, 0xd6, 0xac, 0x3f, 0x9f, 0xd6, 0xac, 0xbf, 0x9d, 0xd6, 0x72, 0x9f, 0x9f, - 0xd6, 0xac, 0x4f, 0x9f, 0xd5, 0x72, 0x4f, 0x9e, 0xd5, 0x72, 0x9f, 0x3d, 0xab, 0xe5, 0x7e, 0x74, - 0xed, 0xc2, 0x27, 0xe4, 0x7e, 0x59, 0x30, 0xb5, 0xf9, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xd2, - 0x2b, 0x01, 0x83, 0x75, 0x1e, 0x00, 0x00, + // 1935 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x59, 0xcb, 0x6f, 0x23, 0x49, + 0x19, 0x77, 0xfb, 0x19, 0x7f, 0x79, 0x4c, 0xa8, 0x09, 0xd9, 0x26, 0x3b, 0xeb, 0x36, 0x96, 0xd8, + 0x0d, 0x08, 0xec, 0x1d, 0x67, 0x77, 0xd8, 0x0d, 0xc3, 0x68, 0xa7, 0x27, 0x33, 0x64, 0x86, 0x59, + 0x76, 0xb6, 0x13, 0x71, 0xe0, 0x82, 0x2a, 0x76, 0xc5, 0x6e, 0x62, 0x77, 0xf7, 0x74, 0x97, 0x33, + 0x13, 0x09, 0xa1, 0xfd, 0x07, 0x56, 0xec, 0x5f, 0x81, 0xb8, 0x71, 0xe1, 0xc4, 0x89, 0xe3, 0xee, + 0x01, 0x69, 0x8e, 0x2b, 0x4b, 0x34, 0x8c, 0x07, 0x21, 0x94, 0xd3, 0x4a, 0x5c, 0x39, 0xa0, 0x7a, + 0x74, 0xbb, 0xca, 0x76, 0x18, 0x27, 0x20, 0xa4, 0xc0, 0x5e, 0x92, 0x7a, 0x7c, 0xbf, 0xea, 0xaa, + 0xdf, 0xf7, 0xfb, 0xbe, 0x7a, 0x18, 0xde, 0x08, 0x8e, 0x3a, 0x8d, 0xc7, 0x03, 0x12, 0xba, 0x24, + 0xe4, 0xff, 0x4f, 0x42, 0xec, 0x75, 0x88, 0x52, 0xac, 0x07, 0xa1, 0x4f, 0x7d, 0x04, 0xe3, 0x96, + 0x8d, 0x66, 0xc7, 0xa5, 0xdd, 0xc1, 0x41, 0xbd, 0xe5, 0xf7, 0x1b, 0x1d, 0xbf, 0xe3, 0x37, 0x3a, + 0xbe, 0xdf, 0xe9, 0x11, 0x1c, 0xb8, 0x91, 0x2c, 0x36, 0xc2, 0xa0, 0xd5, 0x88, 0x28, 0xa6, 0x83, + 0x48, 0xe0, 0x37, 0xd6, 0x98, 0x21, 0x2f, 0x72, 0x88, 0x6c, 0xb5, 0xa4, 0x39, 0xaf, 0x1d, 0x0c, + 0x0e, 0x1b, 0xd4, 0xed, 0x93, 0x88, 0xe2, 0x7e, 0x90, 0x18, 0xb0, 0xf9, 0xf5, 0xfc, 0x8e, 0x40, + 0xba, 0x5e, 0x9b, 0x3c, 0xed, 0x60, 0x4a, 0x9e, 0xe0, 0x13, 0x69, 0xf0, 0xaa, 0x66, 0x90, 0x14, + 0x64, 0xe7, 0x86, 0xd6, 0x19, 0x60, 0x4a, 0x49, 0xe8, 0xc9, 0xbe, 0xaf, 0x69, 0x7d, 0xd1, 0x11, + 0xa1, 0xad, 0xae, 0xec, 0xaa, 0xca, 0xae, 0xc7, 0xbd, 0xbe, 0xdf, 0x26, 0x3d, 0xbe, 0x90, 0x48, + 0xfc, 0x95, 0x16, 0x57, 0x99, 0x45, 0x30, 0x88, 0xba, 0xfc, 0x8f, 0x6c, 0xbc, 0xf3, 0x52, 0x2e, + 0x0f, 0x70, 0x44, 0x1a, 0x6d, 0x72, 0xe8, 0x7a, 0x2e, 0x75, 0x7d, 0x2f, 0x52, 0xcb, 0x72, 0x90, + 0x1b, 0xf3, 0x0d, 0x32, 0xe9, 0x9f, 0x8d, 0x37, 0x19, 0x2e, 0xa2, 0x7e, 0x88, 0x3b, 0xa4, 0xd1, + 0xea, 0x0e, 0xbc, 0xa3, 0x46, 0x0b, 0xb7, 0xba, 0xa4, 0x11, 0x92, 0x68, 0xd0, 0xa3, 0x91, 0xa8, + 0xd0, 0x93, 0x80, 0xc8, 0x2f, 0xd5, 0x3e, 0xcb, 0xc3, 0xe2, 0x43, 0xff, 0xc8, 0x75, 0xc8, 0xe3, + 0x01, 0x89, 0x28, 0x5a, 0x83, 0x02, 0x1f, 0xd5, 0x34, 0xaa, 0xc6, 0x66, 0xd9, 0x11, 0x15, 0xd6, + 0xda, 0x73, 0xfb, 0x2e, 0x35, 0xb3, 0x55, 0x63, 0x73, 0xd9, 0x11, 0x15, 0x84, 0x20, 0x1f, 0x51, + 0x12, 0x98, 0xb9, 0xaa, 0xb1, 0x99, 0x73, 0x78, 0x19, 0x6d, 0xc0, 0x82, 0xeb, 0x51, 0x12, 0x1e, + 0xe3, 0x9e, 0x59, 0xe6, 0xed, 0x69, 0x1d, 0xdd, 0x82, 0x52, 0x44, 0x71, 0x48, 0xf7, 0x23, 0x33, + 0x5f, 0x35, 0x36, 0x17, 0x9b, 0x1b, 0x75, 0xe1, 0xf9, 0x7a, 0xe2, 0xf9, 0xfa, 0x7e, 0xe2, 0x79, + 0x7b, 0xe1, 0xd3, 0xd8, 0xca, 0x7c, 0xf2, 0x27, 0xcb, 0x70, 0x12, 0x10, 0xda, 0x86, 0x02, 0xf1, + 0xda, 0xfb, 0x91, 0x59, 0x38, 0x07, 0x5a, 0x40, 0xd0, 0x75, 0x28, 0xb7, 0xdd, 0x90, 0xb4, 0x18, + 0xcb, 0x66, 0xb1, 0x6a, 0x6c, 0xae, 0x34, 0xaf, 0xd6, 0x53, 0xa1, 0xec, 0x24, 0x5d, 0xce, 0xd8, + 0x8a, 0x2d, 0x2f, 0xc0, 0xb4, 0x6b, 0x96, 0x38, 0x13, 0xbc, 0x8c, 0x6a, 0x50, 0x8c, 0xba, 0x38, + 0x6c, 0x47, 0xe6, 0x42, 0x35, 0xb7, 0x59, 0xb6, 0xe1, 0x34, 0xb6, 0x64, 0x8b, 0x23, 0xff, 0xa3, + 0x9f, 0x42, 0x3e, 0xe8, 0x61, 0xcf, 0x04, 0x3e, 0xcb, 0xd5, 0xba, 0xe2, 0xa5, 0x47, 0x3d, 0xec, + 0xd9, 0xef, 0x0e, 0x63, 0xeb, 0x6d, 0x35, 0x78, 0x42, 0x7c, 0x88, 0x3d, 0xdc, 0xe8, 0xf9, 0x47, + 0x6e, 0xe3, 0x78, 0xab, 0xa1, 0xfa, 0x9e, 0x0d, 0x54, 0xff, 0x90, 0x0d, 0xc0, 0xa0, 0x0e, 0x1f, + 0x18, 0x3d, 0x80, 0x45, 0xe6, 0x63, 0x72, 0x87, 0x39, 0x38, 0x32, 0x17, 0xf9, 0x77, 0x5e, 0x19, + 0xaf, 0x86, 0xb7, 0x3b, 0xe4, 0xf0, 0x07, 0xa1, 0x3f, 0x08, 0xec, 0x2b, 0xa7, 0xb1, 0xa5, 0xda, + 0x3b, 0x6a, 0x05, 0x3d, 0x80, 0x15, 0x26, 0x0a, 0xd7, 0xeb, 0x7c, 0x10, 0x70, 0x05, 0x9a, 0x4b, + 0x7c, 0xb8, 0x6b, 0x75, 0x55, 0x32, 0xf5, 0x3b, 0x9a, 0x8d, 0x9d, 0x67, 0xf4, 0x3a, 0x13, 0xc8, + 0xda, 0x28, 0x07, 0x88, 0x69, 0xe9, 0xbe, 0x17, 0x51, 0xec, 0xd1, 0x8b, 0x48, 0xea, 0x26, 0x14, + 0x59, 0xf0, 0xef, 0x47, 0x5c, 0x54, 0xf3, 0xfa, 0x58, 0x62, 0x74, 0x27, 0xe7, 0xcf, 0xe5, 0xe4, + 0xc2, 0x4c, 0x27, 0x17, 0x5f, 0xea, 0xe4, 0xd2, 0x7f, 0xc9, 0xc9, 0x0b, 0xff, 0x59, 0x27, 0x97, + 0x2f, 0xec, 0x64, 0x13, 0xf2, 0x6c, 0x96, 0x68, 0x15, 0x72, 0x21, 0x7e, 0xc2, 0x7d, 0xba, 0xe4, + 0xb0, 0x62, 0x6d, 0x94, 0x87, 0x25, 0x91, 0x4a, 0xa2, 0xc0, 0xf7, 0x22, 0xc2, 0x78, 0xdc, 0xe3, + 0xd9, 0x5f, 0x78, 0x5e, 0xf2, 0xc8, 0x5b, 0x1c, 0xd9, 0x83, 0xde, 0x83, 0xfc, 0x0e, 0xa6, 0x98, + 0xab, 0x60, 0xb1, 0xb9, 0xa6, 0xf2, 0xc8, 0xc6, 0x62, 0x7d, 0xf6, 0x3a, 0x9b, 0xc8, 0x69, 0x6c, + 0xad, 0xb4, 0x31, 0xc5, 0xdf, 0xf6, 0xfb, 0x2e, 0x25, 0xfd, 0x80, 0x9e, 0x38, 0x1c, 0x89, 0xde, + 0x86, 0xf2, 0xdd, 0x30, 0xf4, 0xc3, 0xfd, 0x93, 0x80, 0x70, 0xd5, 0x94, 0xed, 0x57, 0x4e, 0x63, + 0xeb, 0x2a, 0x49, 0x1a, 0x15, 0xc4, 0xd8, 0x12, 0x7d, 0x13, 0x0a, 0xbc, 0xc2, 0x75, 0x52, 0xb6, + 0xaf, 0x9e, 0xc6, 0xd6, 0x15, 0x0e, 0x51, 0xcc, 0x85, 0x85, 0x2e, 0xab, 0xc2, 0x5c, 0xb2, 0x4a, + 0xd5, 0x5d, 0x54, 0xd5, 0x6d, 0x42, 0xe9, 0x98, 0x84, 0x11, 0x1b, 0xa6, 0xc4, 0xdb, 0x93, 0x2a, + 0xba, 0x0d, 0xc0, 0x88, 0x71, 0x23, 0xea, 0xb6, 0x12, 0x67, 0x2f, 0xd7, 0xc5, 0x66, 0xe3, 0x70, + 0x1f, 0xd9, 0x48, 0xb2, 0xa0, 0x18, 0x3a, 0x4a, 0x19, 0xfd, 0xc6, 0x80, 0xd2, 0x2e, 0xc1, 0x6d, + 0x12, 0x32, 0xf7, 0xe6, 0x36, 0x17, 0x9b, 0xdf, 0xa8, 0xab, 0x3b, 0xcb, 0xa3, 0xd0, 0xef, 0x13, + 0xda, 0x25, 0x83, 0x28, 0x71, 0x90, 0xb0, 0xb6, 0xbd, 0x61, 0x6c, 0x91, 0x39, 0xa5, 0x3a, 0xd7, + 0x86, 0x76, 0xe6, 0xa7, 0x4e, 0x63, 0xcb, 0xf8, 0x8e, 0x93, 0xcc, 0x12, 0x35, 0x61, 0xe1, 0x09, + 0x0e, 0x3d, 0xd7, 0xeb, 0x44, 0x26, 0xf0, 0x48, 0x5b, 0x3f, 0x8d, 0x2d, 0x94, 0xb4, 0x29, 0x8e, + 0x48, 0xed, 0x6a, 0x7f, 0x34, 0xe0, 0x2b, 0x4c, 0x18, 0x7b, 0x6c, 0x3e, 0x91, 0x92, 0x62, 0xfa, + 0x98, 0xb6, 0xba, 0xa6, 0xc1, 0x86, 0x71, 0x44, 0x45, 0xdd, 0x6f, 0xb2, 0xff, 0xd6, 0x7e, 0x93, + 0x3b, 0xff, 0x7e, 0x93, 0xe4, 0x95, 0xfc, 0xcc, 0xbc, 0x52, 0x38, 0x2b, 0xaf, 0xd4, 0x7e, 0x29, + 0x73, 0x68, 0xb2, 0xbe, 0x73, 0x84, 0xd2, 0xbd, 0x34, 0x94, 0x72, 0x7c, 0xb6, 0xa9, 0x42, 0xc5, + 0x58, 0xf7, 0xdb, 0xc4, 0xa3, 0xee, 0xa1, 0x4b, 0xc2, 0x97, 0x04, 0x94, 0xa2, 0xd2, 0x9c, 0xae, + 0x52, 0x55, 0x62, 0xf9, 0x4b, 0x21, 0x31, 0x3d, 0xae, 0x0a, 0x17, 0x88, 0xab, 0xda, 0xdf, 0xb3, + 0xb0, 0xce, 0x3c, 0xf2, 0x10, 0x1f, 0x90, 0xde, 0x8f, 0x70, 0xff, 0x9c, 0x5e, 0x79, 0x5d, 0xf1, + 0x4a, 0xd9, 0x46, 0x5f, 0xb2, 0x3e, 0x1f, 0xeb, 0xbf, 0x32, 0x60, 0x21, 0xd9, 0x00, 0x50, 0x1d, + 0x40, 0xc0, 0x78, 0x8e, 0x17, 0x5c, 0xaf, 0x30, 0x70, 0x98, 0xb6, 0x3a, 0x8a, 0x05, 0xfa, 0x19, + 0x14, 0x45, 0x4d, 0xc6, 0x82, 0xb2, 0x6d, 0xee, 0xd1, 0x90, 0xe0, 0xfe, 0xed, 0x36, 0x0e, 0x28, + 0x09, 0xed, 0x77, 0xd9, 0x2c, 0x86, 0xb1, 0xf5, 0xc6, 0x59, 0x2c, 0x25, 0x27, 0x7c, 0x89, 0x63, + 0xfe, 0x15, 0xdf, 0x74, 0xe4, 0x17, 0x6a, 0x1f, 0x1b, 0xb0, 0xca, 0x26, 0xca, 0xa8, 0x49, 0x85, + 0xb1, 0x03, 0x0b, 0xa1, 0x2c, 0xf3, 0xe9, 0x2e, 0x36, 0x6b, 0x75, 0x9d, 0xd6, 0x19, 0x54, 0xf2, + 0x0d, 0xd7, 0x70, 0x52, 0x24, 0xda, 0xd2, 0x68, 0xcc, 0xce, 0xa2, 0x51, 0xec, 0xd1, 0x2a, 0x71, + 0xbf, 0xcf, 0x02, 0xba, 0xcf, 0x6e, 0x48, 0x4c, 0x7f, 0x63, 0xa9, 0x3e, 0x9d, 0x9a, 0xd1, 0xb5, + 0x31, 0x29, 0xd3, 0xf6, 0xf6, 0xad, 0x61, 0x6c, 0x6d, 0xbf, 0x44, 0x3b, 0xff, 0x02, 0xaf, 0xac, + 0x42, 0x95, 0x6f, 0xf6, 0x32, 0xc8, 0xb7, 0xf6, 0xdb, 0x2c, 0xac, 0xfc, 0xd8, 0xef, 0x0d, 0xfa, + 0x24, 0xa5, 0x2f, 0x98, 0xa2, 0xcf, 0x1c, 0xd3, 0xa7, 0xdb, 0xda, 0xdb, 0xc3, 0xd8, 0xba, 0x31, + 0x2f, 0x75, 0x3a, 0xf6, 0x52, 0xd3, 0xf6, 0xd7, 0x2c, 0xac, 0xed, 0xfb, 0xc1, 0x0f, 0xf7, 0xf8, + 0x2d, 0x5a, 0x49, 0x93, 0xdd, 0x29, 0xf2, 0xd6, 0xc6, 0xe4, 0x31, 0xc4, 0xfb, 0x98, 0x86, 0xee, + 0x53, 0xfb, 0xc6, 0x30, 0xb6, 0x9a, 0xf3, 0x12, 0x37, 0xc6, 0x5d, 0x66, 0xd2, 0xb4, 0x33, 0x50, + 0x6e, 0xce, 0x33, 0xd0, 0x3f, 0xb2, 0xb0, 0xfe, 0xe1, 0x00, 0x7b, 0xd4, 0xed, 0x11, 0x41, 0x76, + 0x4a, 0xf5, 0xcf, 0xa7, 0xa8, 0xae, 0x8c, 0xa9, 0xd6, 0x31, 0x92, 0xf4, 0xf7, 0x86, 0xb1, 0x75, + 0x73, 0x5e, 0xd2, 0x67, 0x8d, 0xf0, 0x7f, 0x47, 0xff, 0xef, 0xb2, 0xb0, 0xb2, 0x27, 0x4e, 0x6d, + 0xc9, 0xc2, 0x8f, 0x67, 0xd0, 0xae, 0x3e, 0x53, 0x05, 0x07, 0x75, 0x1d, 0x71, 0xbe, 0x24, 0xa1, + 0x63, 0x2f, 0x75, 0x92, 0xf8, 0x43, 0x16, 0xd6, 0x77, 0x08, 0x25, 0x2d, 0x4a, 0xda, 0xf7, 0x5c, + 0xd2, 0x53, 0x48, 0xfc, 0xc8, 0x98, 0x62, 0xb1, 0xaa, 0x5c, 0xb3, 0x66, 0x82, 0x6c, 0x7b, 0x18, + 0x5b, 0xb7, 0xe6, 0xe5, 0x71, 0xf6, 0x18, 0x97, 0x9a, 0xcf, 0xcf, 0xb2, 0xf0, 0x55, 0xf1, 0x74, + 0x20, 0xde, 0x35, 0xc7, 0x74, 0xfe, 0x62, 0x8a, 0x4d, 0x4b, 0x4d, 0x05, 0x33, 0x20, 0xf6, 0xed, + 0x61, 0x6c, 0x7d, 0x7f, 0xfe, 0x5c, 0x30, 0x63, 0x88, 0xff, 0x19, 0x6d, 0xf2, 0xd3, 0xfe, 0x79, + 0xb5, 0xa9, 0x83, 0x2e, 0xa6, 0x4d, 0x7d, 0x8c, 0x4b, 0xcd, 0xe7, 0x5f, 0x8a, 0xb0, 0xcc, 0x55, + 0x92, 0xd2, 0xf8, 0x2d, 0x90, 0xd7, 0x23, 0xc9, 0x21, 0x4a, 0xae, 0xd4, 0x61, 0xd0, 0xaa, 0xef, + 0xc9, 0x8b, 0x93, 0xb0, 0x40, 0xef, 0x40, 0x31, 0xe2, 0x17, 0x57, 0x79, 0xf2, 0xad, 0x4c, 0xbe, + 0x0d, 0xe9, 0x57, 0xe4, 0xdd, 0x8c, 0x23, 0xed, 0xd1, 0x4d, 0x28, 0xf6, 0x38, 0x8b, 0xf2, 0xe2, + 0x5e, 0x9b, 0x44, 0x4e, 0x5f, 0xe5, 0x18, 0x5a, 0x60, 0xd0, 0x0d, 0x28, 0xf0, 0x23, 0xb6, 0x7c, + 0xa3, 0xd6, 0x3e, 0x3b, 0x7d, 0xd0, 0xdd, 0xcd, 0x38, 0xc2, 0x1c, 0x35, 0x21, 0x1f, 0x84, 0x7e, + 0x5f, 0x5e, 0x77, 0xae, 0x4d, 0x7e, 0x53, 0xbd, 0x1f, 0xec, 0x66, 0x1c, 0x6e, 0x8b, 0xde, 0x82, + 0x52, 0xc4, 0x2f, 0x16, 0x11, 0x7f, 0x28, 0x62, 0xa7, 0xca, 0x09, 0x98, 0x02, 0x49, 0x4c, 0xd1, + 0x5b, 0x50, 0x3c, 0xe6, 0xc7, 0x46, 0xf9, 0xfa, 0xb8, 0xa1, 0x82, 0xf4, 0x03, 0x25, 0x5b, 0x97, + 0xb0, 0x45, 0xf7, 0x60, 0x89, 0xfa, 0xc1, 0x51, 0x72, 0x3a, 0x93, 0x8f, 0x4c, 0x55, 0x15, 0x3b, + 0xeb, 0xf4, 0xb6, 0x9b, 0x71, 0x34, 0x1c, 0x7a, 0x04, 0xab, 0x8f, 0xb5, 0x63, 0x00, 0x49, 0x9e, + 0x13, 0x35, 0x9e, 0x67, 0x1f, 0x50, 0x76, 0x33, 0xce, 0x14, 0x1a, 0xed, 0xc0, 0x4a, 0xa4, 0xed, + 0x70, 0xf2, 0xe9, 0x5c, 0x5b, 0x97, 0xbe, 0x07, 0xee, 0x66, 0x9c, 0x09, 0x0c, 0x7a, 0x08, 0x2b, + 0x6d, 0x2d, 0xbf, 0xcb, 0x87, 0x71, 0x6d, 0x56, 0xb3, 0x77, 0x00, 0x36, 0x9a, 0x8e, 0x45, 0x1f, + 0xc0, 0x6a, 0x30, 0x91, 0xdb, 0xe4, 0xcb, 0xf8, 0xd7, 0xf5, 0x55, 0xce, 0x48, 0x82, 0x6c, 0x91, + 0x93, 0x60, 0x75, 0x7a, 0x22, 0xc4, 0xcd, 0xe5, 0xb3, 0xa7, 0xa7, 0x27, 0x01, 0x75, 0x7a, 0xa2, + 0xc7, 0x86, 0x71, 0x3a, 0xaa, 0x7d, 0x5c, 0x84, 0x25, 0x19, 0x66, 0xe2, 0x35, 0xec, 0xbb, 0x69, + 0xe4, 0x88, 0x28, 0x7b, 0xed, 0xac, 0xc8, 0xe1, 0xe6, 0x4a, 0xe0, 0xbc, 0x99, 0x06, 0x8e, 0x08, + 0xb9, 0xf5, 0x71, 0x8a, 0xe3, 0xdf, 0x55, 0x10, 0x32, 0x58, 0xb6, 0x92, 0x60, 0x11, 0x91, 0xf6, + 0xea, 0xec, 0x3b, 0x65, 0x82, 0x92, 0x91, 0xb2, 0x0d, 0x25, 0x57, 0xfc, 0x44, 0x30, 0x2b, 0xc6, + 0xa6, 0x7f, 0x41, 0x60, 0xda, 0x97, 0x00, 0xb4, 0x35, 0x8e, 0x98, 0x82, 0x7c, 0x12, 0x9f, 0x8a, + 0x98, 0x14, 0x94, 0x04, 0xcc, 0xf5, 0x34, 0x60, 0x8a, 0x93, 0xcf, 0xe8, 0x49, 0xb8, 0xa4, 0x0b, + 0x93, 0xd1, 0x72, 0x17, 0x96, 0x13, 0x7d, 0xf1, 0x2e, 0x19, 0x2e, 0xaf, 0x9d, 0x75, 0xac, 0x4b, + 0xf0, 0x3a, 0x0a, 0xdd, 0x9f, 0x12, 0x65, 0x79, 0x72, 0x2b, 0x9e, 0x94, 0x64, 0x32, 0xd2, 0xa4, + 0x22, 0x1f, 0xc0, 0x95, 0xb1, 0xa8, 0xc4, 0x9c, 0x60, 0xfa, 0x84, 0xaf, 0xc9, 0x31, 0x19, 0x6a, + 0x12, 0xa8, 0x4e, 0x4b, 0x8a, 0x71, 0xf1, 0xac, 0x69, 0x25, 0x52, 0x9c, 0x9a, 0x96, 0xe8, 0x40, + 0xbb, 0xb0, 0xd0, 0x27, 0x14, 0xb7, 0x31, 0xc5, 0x66, 0x89, 0x6f, 0x4b, 0xaf, 0x4f, 0x05, 0x88, + 0x44, 0xd7, 0xdf, 0x97, 0x86, 0x77, 0x3d, 0x1a, 0x9e, 0xc8, 0xb7, 0x8b, 0x14, 0xbd, 0xf1, 0x3d, + 0x58, 0xd6, 0x0c, 0xd0, 0x2a, 0xe4, 0x8e, 0x48, 0xf2, 0xb3, 0x11, 0x2b, 0xa2, 0x35, 0x28, 0x1c, + 0xe3, 0xde, 0x80, 0x70, 0x7d, 0x96, 0x1d, 0x51, 0xd9, 0xce, 0xbe, 0x63, 0xd8, 0x65, 0x28, 0x85, + 0xe2, 0x2b, 0x76, 0xe7, 0xd9, 0xf3, 0x4a, 0xe6, 0xf3, 0xe7, 0x95, 0xcc, 0x17, 0xcf, 0x2b, 0xc6, + 0x47, 0xa3, 0x8a, 0xf1, 0xeb, 0x51, 0xc5, 0xf8, 0x74, 0x54, 0x31, 0x9e, 0x8d, 0x2a, 0xc6, 0x9f, + 0x47, 0x15, 0xe3, 0x6f, 0xa3, 0x4a, 0xe6, 0x8b, 0x51, 0xc5, 0xf8, 0xe4, 0x45, 0x25, 0xf3, 0xec, + 0x45, 0x25, 0xf3, 0xf9, 0x8b, 0x4a, 0xe6, 0x27, 0xd7, 0xcf, 0xbd, 0x43, 0x1e, 0x14, 0x39, 0x53, + 0x5b, 0xff, 0x0c, 0x00, 0x00, 0xff, 0xff, 0xd4, 0x35, 0x32, 0x52, 0x3f, 0x1f, 0x00, 0x00, } func (this *LokiRequest) Equal(that interface{}) bool { @@ -1653,6 +1673,9 @@ func (this *LokiRequest) Equal(that interface{}) bool { if !this.StoreChunks.Equal(that1.StoreChunks) { return false } + if !this.CachingOptions.Equal(&that1.CachingOptions) { + return false + } return true } func (this *LokiInstantRequest) Equal(that interface{}) bool { @@ -1707,6 +1730,9 @@ func (this *LokiInstantRequest) Equal(that interface{}) bool { if !this.StoreChunks.Equal(that1.StoreChunks) { return false } + if !this.CachingOptions.Equal(&that1.CachingOptions) { + return false + } return true } func (this *Plan) Equal(that interface{}) bool { @@ -2898,7 +2924,7 @@ func (this *LokiRequest) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 15) + s := make([]string, 0, 16) s = append(s, "&queryrange.LokiRequest{") s = append(s, "Query: "+fmt.Sprintf("%#v", this.Query)+",\n") s = append(s, "Limit: "+fmt.Sprintf("%#v", this.Limit)+",\n") @@ -2913,6 +2939,7 @@ func (this *LokiRequest) GoString() string { if this.StoreChunks != nil { s = append(s, "StoreChunks: "+fmt.Sprintf("%#v", this.StoreChunks)+",\n") } + s = append(s, "CachingOptions: "+strings.Replace(this.CachingOptions.GoString(), `&`, ``, 1)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -2920,7 +2947,7 @@ func (this *LokiInstantRequest) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 12) + s := make([]string, 0, 13) s = append(s, "&queryrange.LokiInstantRequest{") s = append(s, "Query: "+fmt.Sprintf("%#v", this.Query)+",\n") s = append(s, "Limit: "+fmt.Sprintf("%#v", this.Limit)+",\n") @@ -2932,6 +2959,7 @@ func (this *LokiInstantRequest) GoString() string { if this.StoreChunks != nil { s = append(s, "StoreChunks: "+fmt.Sprintf("%#v", this.StoreChunks)+",\n") } + s = append(s, "CachingOptions: "+strings.Replace(this.CachingOptions.GoString(), `&`, ``, 1)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -3370,6 +3398,16 @@ func (m *LokiRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + { + size, err := m.CachingOptions.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQueryrange(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x62 if m.StoreChunks != nil { { size, err := m.StoreChunks.MarshalToSizedBuffer(dAtA[:i]) @@ -3420,21 +3458,21 @@ func (m *LokiRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x30 } - n3, err3 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.EndTs, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.EndTs):]) - if err3 != nil { - return 0, err3 - } - i -= n3 - i = encodeVarintQueryrange(dAtA, i, uint64(n3)) - i-- - dAtA[i] = 0x2a - n4, err4 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.StartTs, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.StartTs):]) + n4, err4 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.EndTs, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.EndTs):]) if err4 != nil { return 0, err4 } i -= n4 i = encodeVarintQueryrange(dAtA, i, uint64(n4)) i-- + dAtA[i] = 0x2a + n5, err5 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.StartTs, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.StartTs):]) + if err5 != nil { + return 0, err5 + } + i -= n5 + i = encodeVarintQueryrange(dAtA, i, uint64(n5)) + i-- dAtA[i] = 0x22 if m.Step != 0 { i = encodeVarintQueryrange(dAtA, i, uint64(m.Step)) @@ -3476,6 +3514,16 @@ func (m *LokiInstantRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + { + size, err := m.CachingOptions.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQueryrange(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x4a if m.StoreChunks != nil { { size, err := m.StoreChunks.MarshalToSizedBuffer(dAtA[:i]) @@ -3521,12 +3569,12 @@ func (m *LokiInstantRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x20 } - n7, err7 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.TimeTs, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.TimeTs):]) - if err7 != nil { - return 0, err7 + n9, err9 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.TimeTs, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.TimeTs):]) + if err9 != nil { + return 0, err9 } - i -= n7 - i = encodeVarintQueryrange(dAtA, i, uint64(n7)) + i -= n9 + i = encodeVarintQueryrange(dAtA, i, uint64(n9)) i-- dAtA[i] = 0x1a if m.Limit != 0 { @@ -3712,20 +3760,20 @@ func (m *LokiSeriesRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x22 } - n10, err10 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.EndTs, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.EndTs):]) - if err10 != nil { - return 0, err10 + n12, err12 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.EndTs, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.EndTs):]) + if err12 != nil { + return 0, err12 } - i -= n10 - i = encodeVarintQueryrange(dAtA, i, uint64(n10)) + i -= n12 + i = encodeVarintQueryrange(dAtA, i, uint64(n12)) i-- dAtA[i] = 0x1a - n11, err11 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.StartTs, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.StartTs):]) - if err11 != nil { - return 0, err11 + n13, err13 := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.StartTs, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdTime(m.StartTs):]) + if err13 != nil { + return 0, err13 } - i -= n11 - i = encodeVarintQueryrange(dAtA, i, uint64(n11)) + i -= n13 + i = encodeVarintQueryrange(dAtA, i, uint64(n13)) i-- dAtA[i] = 0x12 if len(m.Match) > 0 { @@ -4970,6 +5018,8 @@ func (m *LokiRequest) Size() (n int) { l = m.StoreChunks.Size() n += 1 + l + sovQueryrange(uint64(l)) } + l = m.CachingOptions.Size() + n += 1 + l + sovQueryrange(uint64(l)) return n } @@ -5009,6 +5059,8 @@ func (m *LokiInstantRequest) Size() (n int) { l = m.StoreChunks.Size() n += 1 + l + sovQueryrange(uint64(l)) } + l = m.CachingOptions.Size() + n += 1 + l + sovQueryrange(uint64(l)) return n } @@ -5681,6 +5733,7 @@ func (this *LokiRequest) String() string { `Interval:` + fmt.Sprintf("%v", this.Interval) + `,`, `Plan:` + fmt.Sprintf("%v", this.Plan) + `,`, `StoreChunks:` + strings.Replace(fmt.Sprintf("%v", this.StoreChunks), "ChunkRefGroup", "logproto.ChunkRefGroup", 1) + `,`, + `CachingOptions:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.CachingOptions), "CachingOptions", "resultscache.CachingOptions", 1), `&`, ``, 1) + `,`, `}`, }, "") return s @@ -5698,6 +5751,7 @@ func (this *LokiInstantRequest) String() string { `Shards:` + fmt.Sprintf("%v", this.Shards) + `,`, `Plan:` + fmt.Sprintf("%v", this.Plan) + `,`, `StoreChunks:` + strings.Replace(fmt.Sprintf("%v", this.StoreChunks), "ChunkRefGroup", "logproto.ChunkRefGroup", 1) + `,`, + `CachingOptions:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.CachingOptions), "CachingOptions", "resultscache.CachingOptions", 1), `&`, ``, 1) + `,`, `}`, }, "") return s @@ -6489,6 +6543,39 @@ func (m *LokiRequest) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 12: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CachingOptions", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQueryrange + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQueryrange + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthQueryrange + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.CachingOptions.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipQueryrange(dAtA[iNdEx:]) @@ -6781,6 +6868,39 @@ func (m *LokiInstantRequest) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 9: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CachingOptions", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQueryrange + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQueryrange + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthQueryrange + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.CachingOptions.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipQueryrange(dAtA[iNdEx:]) diff --git a/pkg/querier/queryrange/queryrange.proto b/pkg/querier/queryrange/queryrange.proto index 46513c9f4e1e..2169b2da3154 100644 --- a/pkg/querier/queryrange/queryrange.proto +++ b/pkg/querier/queryrange/queryrange.proto @@ -13,6 +13,7 @@ import "pkg/logqlmodel/stats/stats.proto"; import "pkg/push/push.proto"; import "pkg/querier/queryrange/queryrangebase/definitions/definitions.proto"; import "pkg/querier/queryrange/queryrangebase/queryrange.proto"; +import "pkg/storage/chunk/cache/resultscache/types.proto"; option go_package = "github.com/grafana/loki/v3/pkg/querier/queryrange"; option (gogoproto.marshaler_all) = true; @@ -39,6 +40,7 @@ message LokiRequest { // If populated, these represent the chunk references that the querier should // use to fetch the data, plus any other chunks reported by ingesters. logproto.ChunkRefGroup storeChunks = 11 [(gogoproto.jsontag) = "storeChunks"]; + resultscache.CachingOptions cachingOptions = 12 [(gogoproto.nullable) = false]; } message LokiInstantRequest { @@ -55,6 +57,7 @@ message LokiInstantRequest { // If populated, these represent the chunk references that the querier should // use to fetch the data, plus any other chunks reported by ingesters. logproto.ChunkRefGroup storeChunks = 8 [(gogoproto.jsontag) = "storeChunks"]; + resultscache.CachingOptions cachingOptions = 9 [(gogoproto.nullable) = false]; } message Plan { diff --git a/pkg/storage/chunk/cache/resultscache/cache_test.go b/pkg/storage/chunk/cache/resultscache/cache_test.go index 964a310f5951..0febe4802086 100644 --- a/pkg/storage/chunk/cache/resultscache/cache_test.go +++ b/pkg/storage/chunk/cache/resultscache/cache_test.go @@ -682,6 +682,105 @@ func Test_resultsCache_MissingData(t *testing.T) { require.False(t, hit) } +func Test_shouldCacheReq(t *testing.T) { + cfg := Config{ + CacheConfig: cache.Config{ + Cache: cache.NewMockCache(), + }, + } + c, err := cache.New(cfg.CacheConfig, nil, log.NewNopLogger(), stats.ResultCache, constants.Loki) + require.NoError(t, err) + rc := NewResultsCache( + log.NewNopLogger(), + c, + nil, + ConstSplitter(day), + mockLimits{}, + MockMerger{}, + MockExtractor{}, + nil, + nil, + func(_ context.Context, tenantIDs []string, r Request) int { + return 10 + }, + nil, + false, + false, + ) + require.NoError(t, err) + + // create cache with handler + ctx := user.InjectOrgID(context.Background(), "1") + + // create request with start end within the key extents + req := parsedRequest.WithStartEndForCache(time.UnixMilli(50), time.UnixMilli(120)) + + // fill cache + key := ConstSplitter(day).GenerateCacheKey(context.Background(), "1", req) + rc.put(ctx, key, []Extent{mkExtent(50, 120)}) + + // Asserts (when `shouldLookupCache` is non-nil and set to return false (should not cache), resultcache should get result from upstream handler (mockHandler)) + // 1. With `shouldLookupCache` non-nil and set `noCacheReq`, should get result from `next` handler + // 2. With `shouldLookupCache` non-nil and set `cacheReq`, should get result from cache + // 3. With `shouldLookupCache` nil, should get result from cache + + cases := []struct { + name string + shouldLookupCache ShouldCacheReqFn + // expected number of times, upstream `next` handler is called inside results cache + expCount int + }{ + { + name: "don't lookup cache", + shouldLookupCache: noLookupCache, + expCount: 1, + }, + { + name: "lookup cache", + shouldLookupCache: lookupCache, + expCount: 0, + }, + { + name: "nil", + shouldLookupCache: nil, + expCount: 0, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mh := &mockHandler{} + rc.next = mh + rc.shouldCacheReq = tc.shouldLookupCache + + _, err = rc.Do(ctx, req) + require.NoError(t, err) + require.Equal(t, tc.expCount, mh.called) + + }) + } +} + +type mockHandler struct { + called int + res Response +} + +func (mh *mockHandler) Do(_ context.Context, _ Request) (Response, error) { + mh.called++ + return mh.res, nil +} + +// noLookupCache is of type `ShouldCacheReq` that always returns false (do not cache) +func noLookupCache(_ context.Context, _ Request) bool { + return false +} + +// lookupCache is of type `ShouldCacheReq` that always returns true (cache the result) +func lookupCache(_ context.Context, _ Request) bool { + return true +} + func mkAPIResponse(start, end, step int64) *MockResponse { var samples []*MockSample for i := start; i <= end; i += step { From 8978ecf0c85dfbe18b52632112e5be20eff411cf Mon Sep 17 00:00:00 2001 From: Salva Corts Date: Mon, 20 May 2024 12:36:22 +0200 Subject: [PATCH 02/41] feat: Boilerplate for new bloom build planner and worker components. (#12989) --- docs/sources/shared/configuration.md | 10 ++++++ pkg/bloombuild/builder/builder.go | 50 ++++++++++++++++++++++++++++ pkg/bloombuild/builder/config.go | 21 ++++++++++++ pkg/bloombuild/builder/metrics.go | 26 +++++++++++++++ pkg/bloombuild/config.go | 40 ++++++++++++++++++++++ pkg/bloombuild/planner/config.go | 21 ++++++++++++ pkg/bloombuild/planner/metrics.go | 26 +++++++++++++++ pkg/bloombuild/planner/planner.go | 50 ++++++++++++++++++++++++++++ pkg/loki/loki.go | 7 ++++ pkg/loki/modules.go | 34 ++++++++++++++++++- 10 files changed, 284 insertions(+), 1 deletion(-) create mode 100644 pkg/bloombuild/builder/builder.go create mode 100644 pkg/bloombuild/builder/config.go create mode 100644 pkg/bloombuild/builder/metrics.go create mode 100644 pkg/bloombuild/config.go create mode 100644 pkg/bloombuild/planner/config.go create mode 100644 pkg/bloombuild/planner/metrics.go create mode 100644 pkg/bloombuild/planner/planner.go diff --git a/docs/sources/shared/configuration.md b/docs/sources/shared/configuration.md index fe57f40daa58..f59f6501c94e 100644 --- a/docs/sources/shared/configuration.md +++ b/docs/sources/shared/configuration.md @@ -326,6 +326,16 @@ pattern_ingester: # merging them as bloom blocks. [bloom_compactor: ] +bloom_build: + # Flag to enable or disable the usage of the bloom-planner and bloom-builder + # components. + # CLI flag: -bloom-build.enabled + [enabled: | default = false] + + planner: + + builder: + # Experimental: The bloom_gateway block configures the Loki bloom gateway # server, responsible for serving queries for filtering chunks based on filter # expressions. diff --git a/pkg/bloombuild/builder/builder.go b/pkg/bloombuild/builder/builder.go new file mode 100644 index 000000000000..098e7d6d83f0 --- /dev/null +++ b/pkg/bloombuild/builder/builder.go @@ -0,0 +1,50 @@ +package builder + +import ( + "context" + + "github.com/go-kit/log" + "github.com/grafana/dskit/services" + "github.com/prometheus/client_golang/prometheus" + + utillog "github.com/grafana/loki/v3/pkg/util/log" +) + +type Worker struct { + services.Service + + cfg Config + metrics *Metrics + logger log.Logger +} + +func New( + cfg Config, + logger log.Logger, + r prometheus.Registerer, +) (*Worker, error) { + utillog.WarnExperimentalUse("Bloom Builder", logger) + + w := &Worker{ + cfg: cfg, + metrics: NewMetrics(r), + logger: logger, + } + + w.Service = services.NewBasicService(w.starting, w.running, w.stopping) + return w, nil +} + +func (w *Worker) starting(_ context.Context) (err error) { + w.metrics.running.Set(1) + return err +} + +func (w *Worker) stopping(_ error) error { + w.metrics.running.Set(0) + return nil +} + +func (w *Worker) running(_ context.Context) error { + return nil +} diff --git a/pkg/bloombuild/builder/config.go b/pkg/bloombuild/builder/config.go new file mode 100644 index 000000000000..ac282ccf95eb --- /dev/null +++ b/pkg/bloombuild/builder/config.go @@ -0,0 +1,21 @@ +package builder + +import "flag" + +// Config configures the bloom-builder component. +type Config struct { + // TODO: Add config +} + +// RegisterFlagsWithPrefix registers flags for the bloom-planner configuration. +func (cfg *Config) RegisterFlagsWithPrefix(_ string, _ *flag.FlagSet) { + // TODO: Register flags with flagsPrefix +} + +func (cfg *Config) Validate() error { + return nil +} + +type Limits interface { + // TODO: Add limits +} diff --git a/pkg/bloombuild/builder/metrics.go b/pkg/bloombuild/builder/metrics.go new file mode 100644 index 000000000000..e8f46fa02508 --- /dev/null +++ b/pkg/bloombuild/builder/metrics.go @@ -0,0 +1,26 @@ +package builder + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +const ( + metricsNamespace = "loki" + metricsSubsystem = "bloombuilder" +) + +type Metrics struct { + running prometheus.Gauge +} + +func NewMetrics(r prometheus.Registerer) *Metrics { + return &Metrics{ + running: promauto.With(r).NewGauge(prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "running", + Help: "Value will be 1 if the bloom builder is currently running on this instance", + }), + } +} diff --git a/pkg/bloombuild/config.go b/pkg/bloombuild/config.go new file mode 100644 index 000000000000..c69c605607f5 --- /dev/null +++ b/pkg/bloombuild/config.go @@ -0,0 +1,40 @@ +package bloombuild + +import ( + "flag" + "fmt" + + "github.com/grafana/loki/v3/pkg/bloombuild/builder" + "github.com/grafana/loki/v3/pkg/bloombuild/planner" +) + +// Config configures the bloom-planner component. +type Config struct { + Enabled bool `yaml:"enabled"` + + Planner planner.Config `yaml:"planner"` + Builder builder.Config `yaml:"builder"` +} + +// RegisterFlags registers flags for the bloom building configuration. +func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + f.BoolVar(&cfg.Enabled, "bloom-build.enabled", false, "Flag to enable or disable the usage of the bloom-planner and bloom-builder components.") + cfg.Planner.RegisterFlagsWithPrefix("bloom-build.planner", f) + cfg.Builder.RegisterFlagsWithPrefix("bloom-build.builder", f) +} + +func (cfg *Config) Validate() error { + if !cfg.Enabled { + return nil + } + + if err := cfg.Planner.Validate(); err != nil { + return fmt.Errorf("invalid bloom planner configuration: %w", err) + } + + if err := cfg.Builder.Validate(); err != nil { + return fmt.Errorf("invalid bloom builder configuration: %w", err) + } + + return nil +} diff --git a/pkg/bloombuild/planner/config.go b/pkg/bloombuild/planner/config.go new file mode 100644 index 000000000000..dd8cb315d934 --- /dev/null +++ b/pkg/bloombuild/planner/config.go @@ -0,0 +1,21 @@ +package planner + +import "flag" + +// Config configures the bloom-planner component. +type Config struct { + // TODO: Add config +} + +// RegisterFlagsWithPrefix registers flags for the bloom-planner configuration. +func (cfg *Config) RegisterFlagsWithPrefix(_ string, _ *flag.FlagSet) { + // TODO: Register flags with flagsPrefix +} + +func (cfg *Config) Validate() error { + return nil +} + +type Limits interface { + // TODO: Add limits +} diff --git a/pkg/bloombuild/planner/metrics.go b/pkg/bloombuild/planner/metrics.go new file mode 100644 index 000000000000..e9a9035e14df --- /dev/null +++ b/pkg/bloombuild/planner/metrics.go @@ -0,0 +1,26 @@ +package planner + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +const ( + metricsNamespace = "loki" + metricsSubsystem = "bloomplanner" +) + +type Metrics struct { + running prometheus.Gauge +} + +func NewMetrics(r prometheus.Registerer) *Metrics { + return &Metrics{ + running: promauto.With(r).NewGauge(prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "running", + Help: "Value will be 1 if bloom planner is currently running on this instance", + }), + } +} diff --git a/pkg/bloombuild/planner/planner.go b/pkg/bloombuild/planner/planner.go new file mode 100644 index 000000000000..7732d180b0bb --- /dev/null +++ b/pkg/bloombuild/planner/planner.go @@ -0,0 +1,50 @@ +package planner + +import ( + "context" + + "github.com/go-kit/log" + "github.com/grafana/dskit/services" + "github.com/prometheus/client_golang/prometheus" + + utillog "github.com/grafana/loki/v3/pkg/util/log" +) + +type Planner struct { + services.Service + + cfg Config + metrics *Metrics + logger log.Logger +} + +func New( + cfg Config, + logger log.Logger, + r prometheus.Registerer, +) (*Planner, error) { + utillog.WarnExperimentalUse("Bloom Planner", logger) + + p := &Planner{ + cfg: cfg, + metrics: NewMetrics(r), + logger: logger, + } + + p.Service = services.NewBasicService(p.starting, p.running, p.stopping) + return p, nil +} + +func (p *Planner) starting(_ context.Context) (err error) { + p.metrics.running.Set(1) + return err +} + +func (p *Planner) stopping(_ error) error { + p.metrics.running.Set(0) + return nil +} + +func (p *Planner) running(_ context.Context) error { + return nil +} diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index b682c4bfaa65..9446b351aab8 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -31,6 +31,7 @@ import ( "google.golang.org/grpc/health/grpc_health_v1" "github.com/grafana/loki/v3/pkg/analytics" + "github.com/grafana/loki/v3/pkg/bloombuild" "github.com/grafana/loki/v3/pkg/bloomcompactor" "github.com/grafana/loki/v3/pkg/bloomgateway" "github.com/grafana/loki/v3/pkg/compactor" @@ -90,6 +91,7 @@ type Config struct { Pattern pattern.Config `yaml:"pattern_ingester,omitempty"` IndexGateway indexgateway.Config `yaml:"index_gateway"` BloomCompactor bloomcompactor.Config `yaml:"bloom_compactor,omitempty" category:"experimental"` + BloomBuild bloombuild.Config `yaml:"bloom_build,omitempty" category:"experimental"` BloomGateway bloomgateway.Config `yaml:"bloom_gateway,omitempty" category:"experimental"` StorageConfig storage.Config `yaml:"storage_config,omitempty"` ChunkStoreConfig config.ChunkStoreConfig `yaml:"chunk_store_config,omitempty"` @@ -173,6 +175,7 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) { c.Tracing.RegisterFlags(f) c.CompactorConfig.RegisterFlags(f) c.BloomCompactor.RegisterFlags(f) + c.BloomBuild.RegisterFlags(f) c.QueryScheduler.RegisterFlags(f) c.Analytics.RegisterFlags(f) c.OperationalConfig.RegisterFlags(f) @@ -649,6 +652,8 @@ func (t *Loki) setupModuleManager() error { mm.RegisterModule(BloomStore, t.initBloomStore) mm.RegisterModule(BloomCompactor, t.initBloomCompactor) mm.RegisterModule(BloomCompactorRing, t.initBloomCompactorRing, modules.UserInvisibleModule) + mm.RegisterModule(BloomPlanner, t.initBloomPlanner) + mm.RegisterModule(BloomBuilder, t.initBloomBuilder) mm.RegisterModule(IndexGateway, t.initIndexGateway) mm.RegisterModule(IndexGatewayRing, t.initIndexGatewayRing, modules.UserInvisibleModule) mm.RegisterModule(IndexGatewayInterceptors, t.initIndexGatewayInterceptors, modules.UserInvisibleModule) @@ -686,6 +691,8 @@ func (t *Loki) setupModuleManager() error { IndexGateway: {Server, Store, BloomStore, IndexGatewayRing, IndexGatewayInterceptors, Analytics}, BloomGateway: {Server, BloomStore, Analytics}, BloomCompactor: {Server, BloomStore, BloomCompactorRing, Analytics, Store}, + BloomPlanner: {Server, BloomStore, Analytics, Store}, + BloomBuilder: {Server, BloomStore, Analytics, Store}, PatternIngester: {Server, MemberlistKV, Analytics}, PatternRingClient: {Server, MemberlistKV, Analytics}, IngesterQuerier: {Ring}, diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go index 0280bd514d3c..a563e80f789f 100644 --- a/pkg/loki/modules.go +++ b/pkg/loki/modules.go @@ -38,6 +38,8 @@ import ( "github.com/grafana/loki/v3/pkg/storage/types" "github.com/grafana/loki/v3/pkg/analytics" + "github.com/grafana/loki/v3/pkg/bloombuild/builder" + "github.com/grafana/loki/v3/pkg/bloombuild/planner" "github.com/grafana/loki/v3/pkg/bloomgateway" "github.com/grafana/loki/v3/pkg/compactor" compactorclient "github.com/grafana/loki/v3/pkg/compactor/client" @@ -122,6 +124,8 @@ const ( QuerySchedulerRing string = "query-scheduler-ring" BloomCompactor string = "bloom-compactor" BloomCompactorRing string = "bloom-compactor-ring" + BloomPlanner string = "bloom-planner" + BloomBuilder string = "bloom-builder" BloomStore string = "bloom-store" All string = "all" Read string = "read" @@ -803,7 +807,7 @@ func (t *Loki) updateConfigForShipperStore() { t.Cfg.StorageConfig.TSDBShipperConfig.Mode = indexshipper.ModeWriteOnly t.Cfg.StorageConfig.TSDBShipperConfig.IngesterDBRetainPeriod = shipperQuerierIndexUpdateDelay(t.Cfg.StorageConfig.IndexCacheValidity, t.Cfg.StorageConfig.TSDBShipperConfig.ResyncInterval) - case t.Cfg.isTarget(Querier), t.Cfg.isTarget(Ruler), t.Cfg.isTarget(Read), t.Cfg.isTarget(Backend), t.isModuleActive(IndexGateway), t.Cfg.isTarget(BloomCompactor): + case t.Cfg.isTarget(Querier), t.Cfg.isTarget(Ruler), t.Cfg.isTarget(Read), t.Cfg.isTarget(Backend), t.isModuleActive(IndexGateway), t.Cfg.isTarget(BloomCompactor), t.Cfg.isTarget(BloomPlanner), t.Cfg.isTarget(BloomBuilder): // We do not want query to do any updates to index t.Cfg.StorageConfig.BoltDBShipperConfig.Mode = indexshipper.ModeReadOnly t.Cfg.StorageConfig.TSDBShipperConfig.Mode = indexshipper.ModeReadOnly @@ -1553,6 +1557,34 @@ func (t *Loki) initBloomCompactorRing() (services.Service, error) { return t.bloomCompactorRingManager, nil } +func (t *Loki) initBloomPlanner() (services.Service, error) { + if !t.Cfg.BloomBuild.Enabled { + return nil, nil + } + + logger := log.With(util_log.Logger, "component", "bloom-planner") + + return planner.New( + t.Cfg.BloomBuild.Planner, + logger, + prometheus.DefaultRegisterer, + ) +} + +func (t *Loki) initBloomBuilder() (services.Service, error) { + if !t.Cfg.BloomBuild.Enabled { + return nil, nil + } + + logger := log.With(util_log.Logger, "component", "bloom-worker") + + return builder.New( + t.Cfg.BloomBuild.Builder, + logger, + prometheus.DefaultRegisterer, + ) +} + func (t *Loki) initQueryScheduler() (services.Service, error) { s, err := scheduler.NewScheduler(t.Cfg.QueryScheduler, t.Overrides, util_log.Logger, t.querySchedulerRingManager, prometheus.DefaultRegisterer, t.Cfg.MetricsNamespace) if err != nil { From 31a13146ed5f631374b7d71d22b219286e3144db Mon Sep 17 00:00:00 2001 From: choeffer Date: Mon, 20 May 2024 16:39:25 +0200 Subject: [PATCH 03/41] docs(install-monolithic): add quotation marks (#12982) Co-authored-by: Michel Hollands <42814411+MichelHollands@users.noreply.github.com> --- docs/sources/setup/install/helm/install-monolithic/_index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sources/setup/install/helm/install-monolithic/_index.md b/docs/sources/setup/install/helm/install-monolithic/_index.md index 86344a9488d7..fd52c11c2003 100644 --- a/docs/sources/setup/install/helm/install-monolithic/_index.md +++ b/docs/sources/setup/install/helm/install-monolithic/_index.md @@ -47,7 +47,7 @@ If you set the `singleBinary.replicas` value to 2 or more, this chart configures type: 'filesystem' schemaConfig: configs: - - from: 2024-01-01 + - from: "2024-01-01" store: tsdb index: prefix: loki_index_ @@ -72,7 +72,7 @@ If you set the `singleBinary.replicas` value to 2 or more, this chart configures replication_factor: 3 schemaConfig: configs: - - from: 2024-01-01 + - from: "2024-01-01" store: tsdb index: prefix: loki_index_ From 94d610e5e0220da1c0bb65bdc9b46ea793dc7387 Mon Sep 17 00:00:00 2001 From: Yarden Shoham Date: Mon, 20 May 2024 18:05:50 +0300 Subject: [PATCH 04/41] docs: Fix broken link in the release notes (#12990) Co-authored-by: J Stickler --- docs/sources/release-notes/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/release-notes/_index.md b/docs/sources/release-notes/_index.md index db74b50c16d7..ebd9afda5afb 100644 --- a/docs/sources/release-notes/_index.md +++ b/docs/sources/release-notes/_index.md @@ -8,7 +8,7 @@ weight: 100 Release notes for Loki are in the CHANGELOG for the release and listed here by version number. -- [V3.0 release notes](https://grafana.com/docs/loki//release-notes/v3.0/) +- [V3.0 release notes](https://grafana.com/docs/loki//release-notes/v3-0/) - [V2.9 release notes](https://grafana.com/docs/loki//release-notes/v2-9/) - [V2.8 release notes](https://grafana.com/docs/loki//release-notes/v2-8/) - [V2.7 release notes](https://grafana.com/docs/loki//release-notes/v2-7/) From 75ccf2160bfe647b1cb3daffb98869e9c1c44130 Mon Sep 17 00:00:00 2001 From: Christian Haudum Date: Mon, 20 May 2024 17:14:40 +0200 Subject: [PATCH 05/41] feat(blooms): Separate page buffer pools for series pages and bloom pages (#12992) Series pages are much smaller than bloom pages and therefore can make use of a separate buffer pool with different buckets. The second commit fixes a possible panic. Signed-off-by: Christian Haudum --- pkg/storage/bloom/v1/bloom.go | 14 +++++++------- pkg/storage/bloom/v1/index.go | 6 +++--- pkg/storage/bloom/v1/util.go | 17 ++++++++++++++--- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/pkg/storage/bloom/v1/bloom.go b/pkg/storage/bloom/v1/bloom.go index 2f195e36df2b..aa51762d4e4e 100644 --- a/pkg/storage/bloom/v1/bloom.go +++ b/pkg/storage/bloom/v1/bloom.go @@ -24,7 +24,7 @@ type Bloom struct { func (b *Bloom) Encode(enc *encoding.Encbuf) error { // divide by 8 b/c bloom capacity is measured in bits, but we want bytes - buf := bytes.NewBuffer(BlockPool.Get(int(b.Capacity() / 8))) + buf := bytes.NewBuffer(BloomPagePool.Get(int(b.Capacity() / 8))) // TODO(owen-d): have encoder implement writer directly so we don't need // to indirect via a buffer @@ -36,7 +36,7 @@ func (b *Bloom) Encode(enc *encoding.Encbuf) error { data := buf.Bytes() enc.PutUvarint(len(data)) // length of bloom filter enc.PutBytes(data) - BlockPool.Put(data[:0]) // release to pool + BloomPagePool.Put(data[:0]) // release to pool return nil } @@ -65,8 +65,8 @@ func (b *Bloom) Decode(dec *encoding.Decbuf) error { } func LazyDecodeBloomPage(r io.Reader, pool chunkenc.ReaderPool, page BloomPageHeader) (*BloomPageDecoder, error) { - data := BlockPool.Get(page.Len)[:page.Len] - defer BlockPool.Put(data) + data := BloomPagePool.Get(page.Len)[:page.Len] + defer BloomPagePool.Put(data) _, err := io.ReadFull(r, data) if err != nil { @@ -84,7 +84,7 @@ func LazyDecodeBloomPage(r io.Reader, pool chunkenc.ReaderPool, page BloomPageHe } defer pool.PutReader(decompressor) - b := BlockPool.Get(page.DecompressedLen)[:page.DecompressedLen] + b := BloomPagePool.Get(page.DecompressedLen)[:page.DecompressedLen] if _, err = io.ReadFull(decompressor, b); err != nil { return nil, errors.Wrap(err, "decompressing bloom page") @@ -101,7 +101,7 @@ func LazyDecodeBloomPageNoCompression(r io.Reader, page BloomPageHeader) (*Bloom if page.Len != page.DecompressedLen+4 { return nil, errors.New("the Len and DecompressedLen of the page do not match") } - data := BlockPool.Get(page.Len)[:page.Len] + data := BloomPagePool.Get(page.Len)[:page.Len] _, err := io.ReadFull(r, data) if err != nil { @@ -163,7 +163,7 @@ func (d *BloomPageDecoder) Relinquish() { d.data = nil if cap(data) > 0 { - BlockPool.Put(data) + BloomPagePool.Put(data) } } diff --git a/pkg/storage/bloom/v1/index.go b/pkg/storage/bloom/v1/index.go index 674a1a883dfb..a4bcd450650c 100644 --- a/pkg/storage/bloom/v1/index.go +++ b/pkg/storage/bloom/v1/index.go @@ -155,7 +155,7 @@ func (b *BlockIndex) NewSeriesPageDecoder(r io.ReadSeeker, header SeriesPageHead defer func() { if err != nil { metrics.pagesSkipped.WithLabelValues(pageTypeSeries, skipReasonErr).Inc() - metrics.bytesSkipped.WithLabelValues(pageTypeSeries).Add(float64(header.DecompressedLen)) + metrics.bytesSkipped.WithLabelValues(pageTypeSeries, skipReasonErr).Add(float64(header.DecompressedLen)) } else { metrics.pagesRead.WithLabelValues(pageTypeSeries).Inc() metrics.bytesRead.WithLabelValues(pageTypeSeries).Add(float64(header.DecompressedLen)) @@ -166,8 +166,8 @@ func (b *BlockIndex) NewSeriesPageDecoder(r io.ReadSeeker, header SeriesPageHead return nil, errors.Wrap(err, "seeking to series page") } - data := BlockPool.Get(header.Len)[:header.Len] - defer BlockPool.Put(data) + data := SeriesPagePool.Get(header.Len)[:header.Len] + defer SeriesPagePool.Put(data) _, err = io.ReadFull(r, data) if err != nil { return nil, errors.Wrap(err, "reading series page") diff --git a/pkg/storage/bloom/v1/util.go b/pkg/storage/bloom/v1/util.go index 06cf1f6add22..22fb47e43e79 100644 --- a/pkg/storage/bloom/v1/util.go +++ b/pkg/storage/bloom/v1/util.go @@ -32,10 +32,21 @@ var ( }, } - // 4KB -> 128MB - BlockPool = BytePool{ + // buffer pool for series pages + // 1KB 2KB 4KB 8KB 16KB 32KB 64KB 128KB + SeriesPagePool = BytePool{ pool: pool.New( - 4<<10, 128<<20, 2, + 1<<10, 128<<10, 2, + func(size int) interface{} { + return make([]byte, size) + }), + } + + // buffer pool for bloom pages + // 128KB 256KB 512KB 1MB 2MB 4MB 8MB 16MB 32MB 64MB 128MB + BloomPagePool = BytePool{ + pool: pool.New( + 128<<10, 128<<20, 2, func(size int) interface{} { return make([]byte, size) }), From 8442dca9d2341471996a73a011f206630c67e857 Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Mon, 20 May 2024 17:52:17 -0400 Subject: [PATCH 06/41] feat: Added getting started video (#12975) --- docs/sources/get-started/_index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/sources/get-started/_index.md b/docs/sources/get-started/_index.md index f82d5f9f089c..c85f383345fb 100644 --- a/docs/sources/get-started/_index.md +++ b/docs/sources/get-started/_index.md @@ -7,6 +7,8 @@ description: Provides an overview of the steps for implementing Grafana Loki to # Get started with Grafana Loki +{{< youtube id="1uk8LtQqsZQ" >}} + Loki is a horizontally-scalable, highly-available, multi-tenant log aggregation system inspired by Prometheus. It is designed to be very cost effective and easy to operate. It does not index the contents of the logs, but rather a set of labels for each log stream. Because all Loki implementations are unique, the installation process is From 1f5291a4a3bd3c98c190d9a5dda32bbd78f18c3b Mon Sep 17 00:00:00 2001 From: Ashwanth Date: Tue, 21 May 2024 12:38:02 +0530 Subject: [PATCH 07/41] fix(indexstats): do not collect stats from "IndexStats" lookups for other query types (#12978) --- pkg/ingester/tailer.go | 2 ++ pkg/querier/queryrange/roundtrip.go | 25 ++----------------------- 2 files changed, 4 insertions(+), 23 deletions(-) diff --git a/pkg/ingester/tailer.go b/pkg/ingester/tailer.go index b39f42957360..b2d6d9874ee2 100644 --- a/pkg/ingester/tailer.go +++ b/pkg/ingester/tailer.go @@ -6,6 +6,8 @@ import ( "sync" "time" + "go.uber.org/atomic" + "github.com/go-kit/log/level" "github.com/prometheus/prometheus/model/labels" "go.uber.org/atomic" diff --git a/pkg/querier/queryrange/roundtrip.go b/pkg/querier/queryrange/roundtrip.go index 61da06929fe1..35806abb38d4 100644 --- a/pkg/querier/queryrange/roundtrip.go +++ b/pkg/querier/queryrange/roundtrip.go @@ -274,7 +274,7 @@ func NewMiddleware( seriesRT = seriesTripperware.Wrap(next) labelsRT = labelsTripperware.Wrap(next) instantRT = instantMetricTripperware.Wrap(next) - statsRT = indexStatsTripperware.Wrap(next) + statsRT = base.MergeMiddlewares(StatsCollectorMiddleware(), indexStatsTripperware).Wrap(next) seriesVolumeRT = seriesVolumeTripperware.Wrap(next) detectedFieldsRT = detectedFieldsTripperware.Wrap(next) detectedLabelsRT = detectedLabelsTripperware.Wrap(next) @@ -1055,22 +1055,6 @@ func NewVolumeTripperware(cfg Config, log log.Logger, limits Limits, schema conf ), nil } -func statsTripperware(nextTW base.Middleware) base.Middleware { - return base.MiddlewareFunc(func(next base.Handler) base.Handler { - return base.HandlerFunc(func(ctx context.Context, r base.Request) (base.Response, error) { - cacheMiddlewares := []base.Middleware{ - StatsCollectorMiddleware(), - nextTW, - } - - // wrap nextRT with our new middleware - return base.MergeMiddlewares( - cacheMiddlewares..., - ).Wrap(next).Do(ctx, r) - }) - }) -} - func volumeRangeTripperware(nextTW base.Middleware) base.Middleware { return base.MiddlewareFunc(func(next base.Handler) base.Handler { return base.HandlerFunc(func(ctx context.Context, r base.Request) (base.Response, error) { @@ -1141,7 +1125,7 @@ func NewIndexStatsTripperware(cfg Config, log log.Logger, limits Limits, schema } } - tw, err := sharedIndexTripperware( + return sharedIndexTripperware( cacheMiddleware, cfg, merger, @@ -1152,11 +1136,6 @@ func NewIndexStatsTripperware(cfg Config, log log.Logger, limits Limits, schema schema, metricsNamespace, ) - if err != nil { - return nil, err - } - - return statsTripperware(tw), nil } func sharedIndexTripperware( From bf8a27887979b2337263c55bd23aead9eed6ea0f Mon Sep 17 00:00:00 2001 From: Ashwanth Date: Tue, 21 May 2024 12:56:07 +0530 Subject: [PATCH 08/41] chore: remove duplicate imports (#13001) --- pkg/ingester/tailer.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/ingester/tailer.go b/pkg/ingester/tailer.go index b2d6d9874ee2..b39f42957360 100644 --- a/pkg/ingester/tailer.go +++ b/pkg/ingester/tailer.go @@ -6,8 +6,6 @@ import ( "sync" "time" - "go.uber.org/atomic" - "github.com/go-kit/log/level" "github.com/prometheus/prometheus/model/labels" "go.uber.org/atomic" From 7a3338ead82e4c577652ab86e9a55faf200ac05a Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Tue, 21 May 2024 10:41:42 +0100 Subject: [PATCH 09/41] feat: loki/main.go: Log which config file path is used on startup (#12985) Co-authored-by: Michel Hollands <42814411+MichelHollands@users.noreply.github.com> --- cmd/loki/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/loki/main.go b/cmd/loki/main.go index d9f461397787..401085b3aab1 100644 --- a/cmd/loki/main.go +++ b/cmd/loki/main.go @@ -118,6 +118,7 @@ func main() { } level.Info(util_log.Logger).Log("msg", "Starting Loki", "version", version.Info()) + level.Info(util_log.Logger).Log("msg", "Loading configuration file", "filename", config.ConfigFile) err = t.Run(loki.RunOpts{StartTime: startTime}) util_log.CheckFatal("running loki", err, util_log.Logger) From 319503643589163edce0b939de0beac074006a9f Mon Sep 17 00:00:00 2001 From: Salva Corts Date: Tue, 21 May 2024 13:12:24 +0200 Subject: [PATCH 10/41] refactor(bloom planner): Compute gaps and build tasks from metas and TSDBs (#12994) --- docs/sources/shared/configuration.md | 27 ++ pkg/bloombuild/planner/config.go | 29 +- pkg/bloombuild/planner/metrics.go | 36 +++ pkg/bloombuild/planner/planner.go | 407 +++++++++++++++++++++++- pkg/bloombuild/planner/planner_test.go | 321 +++++++++++++++++++ pkg/bloombuild/planner/tableIterator.go | 50 +++ pkg/bloombuild/planner/task.go | 22 ++ pkg/bloombuild/planner/tsdb.go | 261 +++++++++++++++ pkg/bloombuild/planner/tsdb_test.go | 105 ++++++ pkg/bloombuild/planner/util.go | 125 ++++++++ pkg/bloombuild/planner/util_test.go | 172 ++++++++++ pkg/loki/modules.go | 5 + pkg/util/limiter/combined_limits.go | 4 + pkg/validation/limits.go | 14 + 14 files changed, 1568 insertions(+), 10 deletions(-) create mode 100644 pkg/bloombuild/planner/planner_test.go create mode 100644 pkg/bloombuild/planner/tableIterator.go create mode 100644 pkg/bloombuild/planner/task.go create mode 100644 pkg/bloombuild/planner/tsdb.go create mode 100644 pkg/bloombuild/planner/tsdb_test.go create mode 100644 pkg/bloombuild/planner/util.go create mode 100644 pkg/bloombuild/planner/util_test.go diff --git a/docs/sources/shared/configuration.md b/docs/sources/shared/configuration.md index f59f6501c94e..d30dce2f7775 100644 --- a/docs/sources/shared/configuration.md +++ b/docs/sources/shared/configuration.md @@ -333,6 +333,23 @@ bloom_build: [enabled: | default = false] planner: + # Interval at which to re-run the bloom creation planning. + # CLI flag: -bloom-build.planner.interval + [planning_interval: | default = 8h] + + # Newest day-table offset (from today, inclusive) to build blooms for. + # Increase to lower cost by not re-writing data to object storage too + # frequently since recent data changes more often at the cost of not having + # blooms available as quickly. + # CLI flag: -bloom-build.planner.min-table-offset + [min_table_offset: | default = 1] + + # Oldest day-table offset (from today, inclusive) to compact. This can be + # used to lower cost by not trying to compact older data which doesn't + # change. This can be optimized by aligning it with the maximum + # `reject_old_samples_max_age` setting of any tenant. + # CLI flag: -bloom-build.planner.max-table-offset + [max_table_offset: | default = 2] builder: @@ -3382,6 +3399,16 @@ shard_streams: # CLI flag: -bloom-compactor.max-bloom-size [bloom_compactor_max_bloom_size: | default = 128MB] +# Experimental. Whether to create blooms for the tenant. +# CLI flag: -bloom-build.enable +[bloom_creation_enabled: | default = false] + +# Experimental. Number of splits to create for the series keyspace when building +# blooms. The series keyspace is split into this many parts to parallelize bloom +# creation. +# CLI flag: -bloom-build.split-keyspace-by +[bloom_split_series_keyspace_by: | default = 256] + # Experimental. Length of the n-grams created when computing blooms from log # lines. # CLI flag: -bloom-compactor.ngram-length diff --git a/pkg/bloombuild/planner/config.go b/pkg/bloombuild/planner/config.go index dd8cb315d934..47b01c0b286e 100644 --- a/pkg/bloombuild/planner/config.go +++ b/pkg/bloombuild/planner/config.go @@ -1,21 +1,40 @@ package planner -import "flag" +import ( + "flag" + "fmt" + "time" +) // Config configures the bloom-planner component. type Config struct { - // TODO: Add config + PlanningInterval time.Duration `yaml:"planning_interval"` + MinTableOffset int `yaml:"min_table_offset"` + MaxTableOffset int `yaml:"max_table_offset"` } // RegisterFlagsWithPrefix registers flags for the bloom-planner configuration. -func (cfg *Config) RegisterFlagsWithPrefix(_ string, _ *flag.FlagSet) { - // TODO: Register flags with flagsPrefix +func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + f.DurationVar(&cfg.PlanningInterval, prefix+".interval", 8*time.Hour, "Interval at which to re-run the bloom creation planning.") + f.IntVar(&cfg.MinTableOffset, prefix+".min-table-offset", 1, "Newest day-table offset (from today, inclusive) to build blooms for. Increase to lower cost by not re-writing data to object storage too frequently since recent data changes more often at the cost of not having blooms available as quickly.") + // TODO(owen-d): ideally we'd set this per tenant based on their `reject_old_samples_max_age` setting, + // but due to how we need to discover tenants, we can't do that yet. Tenant+Period discovery is done by + // iterating the table periods in object storage and looking for tenants within that period. + // In order to have this done dynamically, we'd need to account for tenant specific overrides, which are also + // dynamically reloaded. + // I'm doing it the simple way for now. + f.IntVar(&cfg.MaxTableOffset, prefix+".max-table-offset", 2, "Oldest day-table offset (from today, inclusive) to compact. This can be used to lower cost by not trying to compact older data which doesn't change. This can be optimized by aligning it with the maximum `reject_old_samples_max_age` setting of any tenant.") } func (cfg *Config) Validate() error { + if cfg.MinTableOffset > cfg.MaxTableOffset { + return fmt.Errorf("min-table-offset (%d) must be less than or equal to max-table-offset (%d)", cfg.MinTableOffset, cfg.MaxTableOffset) + } + return nil } type Limits interface { - // TODO: Add limits + BloomCreationEnabled(tenantID string) bool + BloomSplitSeriesKeyspaceBy(tenantID string) int } diff --git a/pkg/bloombuild/planner/metrics.go b/pkg/bloombuild/planner/metrics.go index e9a9035e14df..c0028237d9b1 100644 --- a/pkg/bloombuild/planner/metrics.go +++ b/pkg/bloombuild/planner/metrics.go @@ -8,10 +8,19 @@ import ( const ( metricsNamespace = "loki" metricsSubsystem = "bloomplanner" + + statusSuccess = "success" + statusFailure = "failure" ) type Metrics struct { running prometheus.Gauge + + buildStarted prometheus.Counter + buildCompleted *prometheus.CounterVec + buildTime *prometheus.HistogramVec + + tenantsDiscovered prometheus.Counter } func NewMetrics(r prometheus.Registerer) *Metrics { @@ -22,5 +31,32 @@ func NewMetrics(r prometheus.Registerer) *Metrics { Name: "running", Help: "Value will be 1 if bloom planner is currently running on this instance", }), + + buildStarted: promauto.With(r).NewCounter(prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "build_started_total", + Help: "Total number of builds started", + }), + buildCompleted: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "build_completed_total", + Help: "Total number of builds completed", + }, []string{"status"}), + buildTime: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "build_time_seconds", + Help: "Time spent during a builds cycle.", + Buckets: prometheus.DefBuckets, + }, []string{"status"}), + + tenantsDiscovered: promauto.With(r).NewCounter(prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "tenants_discovered_total", + Help: "Number of tenants discovered during the current build iteration", + }), } } diff --git a/pkg/bloombuild/planner/planner.go b/pkg/bloombuild/planner/planner.go index 7732d180b0bb..0be853a2f604 100644 --- a/pkg/bloombuild/planner/planner.go +++ b/pkg/bloombuild/planner/planner.go @@ -2,33 +2,63 @@ package planner import ( "context" + "fmt" + "sort" + "time" "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/grafana/dskit/services" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/model" + "github.com/grafana/loki/v3/pkg/storage" + v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1" + "github.com/grafana/loki/v3/pkg/storage/config" + "github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper" + "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb" utillog "github.com/grafana/loki/v3/pkg/util/log" ) type Planner struct { services.Service - cfg Config + cfg Config + limits Limits + schemaCfg config.SchemaConfig + + tsdbStore TSDBStore + bloomStore bloomshipper.Store + metrics *Metrics logger log.Logger } func New( cfg Config, + limits Limits, + schemaCfg config.SchemaConfig, + storeCfg storage.Config, + storageMetrics storage.ClientMetrics, + bloomStore bloomshipper.Store, logger log.Logger, r prometheus.Registerer, ) (*Planner, error) { utillog.WarnExperimentalUse("Bloom Planner", logger) + tsdbStore, err := NewTSDBStores(schemaCfg, storeCfg, storageMetrics, logger) + if err != nil { + return nil, fmt.Errorf("error creating TSDB store: %w", err) + } + p := &Planner{ - cfg: cfg, - metrics: NewMetrics(r), - logger: logger, + cfg: cfg, + limits: limits, + schemaCfg: schemaCfg, + tsdbStore: tsdbStore, + bloomStore: bloomStore, + metrics: NewMetrics(r), + logger: logger, } p.Service = services.NewBasicService(p.starting, p.running, p.stopping) @@ -45,6 +75,373 @@ func (p *Planner) stopping(_ error) error { return nil } -func (p *Planner) running(_ context.Context) error { +func (p *Planner) running(ctx context.Context) error { + // run once at beginning + if err := p.runOne(ctx); err != nil { + level.Error(p.logger).Log("msg", "bloom build iteration failed for the first time", "err", err) + } + + ticker := time.NewTicker(p.cfg.PlanningInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + err := ctx.Err() + level.Debug(p.logger).Log("msg", "planner context done", "err", err) + return err + + case <-ticker.C: + if err := p.runOne(ctx); err != nil { + level.Error(p.logger).Log("msg", "bloom build iteration failed", "err", err) + } + } + } +} + +func (p *Planner) runOne(ctx context.Context) error { + var ( + start = time.Now() + status = statusFailure + ) + defer func() { + p.metrics.buildCompleted.WithLabelValues(status).Inc() + p.metrics.buildTime.WithLabelValues(status).Observe(time.Since(start).Seconds()) + }() + + p.metrics.buildStarted.Inc() + level.Info(p.logger).Log("msg", "running bloom build planning") + + tables := p.tables(time.Now()) + level.Debug(p.logger).Log("msg", "loaded tables", "tables", tables.TotalDays()) + + work, err := p.loadWork(ctx, tables) + if err != nil { + level.Error(p.logger).Log("msg", "error loading work", "err", err) + return fmt.Errorf("error loading work: %w", err) + } + + // TODO: Enqueue instead of buffering here + // This is just a placeholder for now + var tasks []Task + + for _, w := range work { + gaps, err := p.findGapsForBounds(ctx, w.tenant, w.table, w.ownershipRange) + if err != nil { + level.Error(p.logger).Log("msg", "error finding gaps", "err", err, "tenant", w.tenant, "table", w.table, "ownership", w.ownershipRange.String()) + return fmt.Errorf("error finding gaps for tenant (%s) in table (%s) for bounds (%s): %w", w.tenant, w.table, w.ownershipRange, err) + } + + for _, gap := range gaps { + tasks = append(tasks, Task{ + table: w.table.Addr(), + tenant: w.tenant, + OwnershipBounds: w.ownershipRange, + tsdb: gap.tsdb, + gaps: gap.gaps, + }) + } + } + + status = statusSuccess + level.Info(p.logger).Log( + "msg", "bloom build iteration completed", + "duration", time.Since(start).Seconds(), + "tasks", len(tasks), + ) return nil } + +func (p *Planner) tables(ts time.Time) *dayRangeIterator { + // adjust the minimum by one to make it inclusive, which is more intuitive + // for a configuration variable + adjustedMin := p.cfg.MinTableOffset - 1 + minCompactionDelta := time.Duration(adjustedMin) * config.ObjectStorageIndexRequiredPeriod + maxCompactionDelta := time.Duration(p.cfg.MaxTableOffset) * config.ObjectStorageIndexRequiredPeriod + + from := ts.Add(-maxCompactionDelta).UnixNano() / int64(config.ObjectStorageIndexRequiredPeriod) * int64(config.ObjectStorageIndexRequiredPeriod) + through := ts.Add(-minCompactionDelta).UnixNano() / int64(config.ObjectStorageIndexRequiredPeriod) * int64(config.ObjectStorageIndexRequiredPeriod) + + fromDay := config.NewDayTime(model.TimeFromUnixNano(from)) + throughDay := config.NewDayTime(model.TimeFromUnixNano(through)) + level.Debug(p.logger).Log("msg", "loaded tables for compaction", "from", fromDay, "through", throughDay) + return newDayRangeIterator(fromDay, throughDay, p.schemaCfg) +} + +type tenantTableRange struct { + tenant string + table config.DayTable + ownershipRange v1.FingerprintBounds + + // TODO: Add tracking + //finished bool + //queueTime, startTime, endTime time.Time +} + +func (p *Planner) loadWork( + ctx context.Context, + tables *dayRangeIterator, +) ([]tenantTableRange, error) { + var work []tenantTableRange + + for tables.Next() && tables.Err() == nil && ctx.Err() == nil { + table := tables.At() + level.Debug(p.logger).Log("msg", "loading work for table", "table", table) + + tenants, err := p.tenants(ctx, table) + if err != nil { + return nil, fmt.Errorf("error loading tenants: %w", err) + } + level.Debug(p.logger).Log("msg", "loaded tenants", "table", table, "tenants", tenants.Len()) + + for tenants.Next() && tenants.Err() == nil && ctx.Err() == nil { + p.metrics.tenantsDiscovered.Inc() + tenant := tenants.At() + + if !p.limits.BloomCreationEnabled(tenant) { + continue + } + + splitFactor := p.limits.BloomSplitSeriesKeyspaceBy(tenant) + bounds := SplitFingerprintKeyspaceByFactor(splitFactor) + + for _, bounds := range bounds { + work = append(work, tenantTableRange{ + tenant: tenant, + table: table, + ownershipRange: bounds, + }) + } + + level.Debug(p.logger).Log("msg", "loading work for tenant", "table", table, "tenant", tenant, "splitFactor", splitFactor) + } + if err := tenants.Err(); err != nil { + level.Error(p.logger).Log("msg", "error iterating tenants", "err", err) + return nil, fmt.Errorf("error iterating tenants: %w", err) + } + + } + if err := tables.Err(); err != nil { + level.Error(p.logger).Log("msg", "error iterating tables", "err", err) + return nil, fmt.Errorf("error iterating tables: %w", err) + } + + return work, ctx.Err() +} + +func (p *Planner) tenants(ctx context.Context, table config.DayTable) (*v1.SliceIter[string], error) { + tenants, err := p.tsdbStore.UsersForPeriod(ctx, table) + if err != nil { + return nil, fmt.Errorf("error loading tenants for table (%s): %w", table, err) + } + + return v1.NewSliceIter(tenants), nil +} + +/* +Planning works as follows, split across many functions for clarity: + 1. Fetch all meta.jsons for the given tenant and table which overlap the ownership range of this compactor. + 2. Load current TSDBs for this tenant/table. + 3. For each live TSDB (there should be only 1, but this works with multiple), find any gaps + (fingerprint ranges) which are not up-to-date, determined by checking other meta.json files and comparing + the TSDBs they were generated from as well as their ownership ranges. +*/ +func (p *Planner) findGapsForBounds( + ctx context.Context, + tenant string, + table config.DayTable, + ownershipRange v1.FingerprintBounds, +) ([]blockPlan, error) { + logger := log.With(p.logger, "org_id", tenant, "table", table.Addr(), "ownership", ownershipRange.String()) + + // Fetch source metas to be used in both build and cleanup of out-of-date metas+blooms + metas, err := p.bloomStore.FetchMetas( + ctx, + bloomshipper.MetaSearchParams{ + TenantID: tenant, + Interval: bloomshipper.NewInterval(table.Bounds()), + Keyspace: ownershipRange, + }, + ) + if err != nil { + level.Error(logger).Log("msg", "failed to get metas", "err", err) + return nil, fmt.Errorf("failed to get metas: %w", err) + } + + level.Debug(logger).Log("msg", "found relevant metas", "metas", len(metas)) + + // Find gaps in the TSDBs for this tenant/table + gaps, err := p.findOutdatedGaps(ctx, tenant, table, ownershipRange, metas, logger) + if err != nil { + return nil, fmt.Errorf("failed to find outdated gaps: %w", err) + } + + return gaps, nil +} + +// blockPlan is a plan for all the work needed to build a meta.json +// It includes: +// - the tsdb (source of truth) which contains all the series+chunks +// we need to ensure are indexed in bloom blocks +// - a list of gaps that are out of date and need to be checked+built +// - within each gap, a list of block refs which overlap the gap are included +// so we can use them to accelerate bloom generation. They likely contain many +// of the same chunks we need to ensure are indexed, just from previous tsdb iterations. +// This is a performance optimization to avoid expensive re-reindexing +type blockPlan struct { + tsdb tsdb.SingleTenantTSDBIdentifier + gaps []GapWithBlocks +} + +func (p *Planner) findOutdatedGaps( + ctx context.Context, + tenant string, + table config.DayTable, + ownershipRange v1.FingerprintBounds, + metas []bloomshipper.Meta, + logger log.Logger, +) ([]blockPlan, error) { + // Resolve TSDBs + tsdbs, err := p.tsdbStore.ResolveTSDBs(ctx, table, tenant) + if err != nil { + level.Error(logger).Log("msg", "failed to resolve tsdbs", "err", err) + return nil, fmt.Errorf("failed to resolve tsdbs: %w", err) + } + + if len(tsdbs) == 0 { + return nil, nil + } + + // Determine which TSDBs have gaps in the ownership range and need to + // be processed. + tsdbsWithGaps, err := gapsBetweenTSDBsAndMetas(ownershipRange, tsdbs, metas) + if err != nil { + level.Error(logger).Log("msg", "failed to find gaps", "err", err) + return nil, fmt.Errorf("failed to find gaps: %w", err) + } + + if len(tsdbsWithGaps) == 0 { + level.Debug(logger).Log("msg", "blooms exist for all tsdbs") + return nil, nil + } + + work, err := blockPlansForGaps(tsdbsWithGaps, metas) + if err != nil { + level.Error(logger).Log("msg", "failed to create plan", "err", err) + return nil, fmt.Errorf("failed to create plan: %w", err) + } + + return work, nil +} + +// Used to signal the gaps that need to be populated for a tsdb +type tsdbGaps struct { + tsdb tsdb.SingleTenantTSDBIdentifier + gaps []v1.FingerprintBounds +} + +// gapsBetweenTSDBsAndMetas returns if the metas are up-to-date with the TSDBs. This is determined by asserting +// that for each TSDB, there are metas covering the entire ownership range which were generated from that specific TSDB. +func gapsBetweenTSDBsAndMetas( + ownershipRange v1.FingerprintBounds, + tsdbs []tsdb.SingleTenantTSDBIdentifier, + metas []bloomshipper.Meta, +) (res []tsdbGaps, err error) { + for _, db := range tsdbs { + id := db.Name() + + relevantMetas := make([]v1.FingerprintBounds, 0, len(metas)) + for _, meta := range metas { + for _, s := range meta.Sources { + if s.Name() == id { + relevantMetas = append(relevantMetas, meta.Bounds) + } + } + } + + gaps, err := FindGapsInFingerprintBounds(ownershipRange, relevantMetas) + if err != nil { + return nil, err + } + + if len(gaps) > 0 { + res = append(res, tsdbGaps{ + tsdb: db, + gaps: gaps, + }) + } + } + + return res, err +} + +// blockPlansForGaps groups tsdb gaps we wish to fill with overlapping but out of date blocks. +// This allows us to expedite bloom generation by using existing blocks to fill in the gaps +// since many will contain the same chunks. +func blockPlansForGaps(tsdbs []tsdbGaps, metas []bloomshipper.Meta) ([]blockPlan, error) { + plans := make([]blockPlan, 0, len(tsdbs)) + + for _, idx := range tsdbs { + plan := blockPlan{ + tsdb: idx.tsdb, + gaps: make([]GapWithBlocks, 0, len(idx.gaps)), + } + + for _, gap := range idx.gaps { + planGap := GapWithBlocks{ + bounds: gap, + } + + for _, meta := range metas { + + if meta.Bounds.Intersection(gap) == nil { + // this meta doesn't overlap the gap, skip + continue + } + + for _, block := range meta.Blocks { + if block.Bounds.Intersection(gap) == nil { + // this block doesn't overlap the gap, skip + continue + } + // this block overlaps the gap, add it to the plan + // for this gap + planGap.blocks = append(planGap.blocks, block) + } + } + + // ensure we sort blocks so deduping iterator works as expected + sort.Slice(planGap.blocks, func(i, j int) bool { + return planGap.blocks[i].Bounds.Less(planGap.blocks[j].Bounds) + }) + + peekingBlocks := v1.NewPeekingIter[bloomshipper.BlockRef]( + v1.NewSliceIter[bloomshipper.BlockRef]( + planGap.blocks, + ), + ) + // dedupe blocks which could be in multiple metas + itr := v1.NewDedupingIter[bloomshipper.BlockRef, bloomshipper.BlockRef]( + func(a, b bloomshipper.BlockRef) bool { + return a == b + }, + v1.Identity[bloomshipper.BlockRef], + func(a, _ bloomshipper.BlockRef) bloomshipper.BlockRef { + return a + }, + peekingBlocks, + ) + + deduped, err := v1.Collect[bloomshipper.BlockRef](itr) + if err != nil { + return nil, fmt.Errorf("failed to dedupe blocks: %w", err) + } + planGap.blocks = deduped + + plan.gaps = append(plan.gaps, planGap) + } + + plans = append(plans, plan) + } + + return plans, nil +} diff --git a/pkg/bloombuild/planner/planner_test.go b/pkg/bloombuild/planner/planner_test.go new file mode 100644 index 000000000000..346bd145ab8d --- /dev/null +++ b/pkg/bloombuild/planner/planner_test.go @@ -0,0 +1,321 @@ +package planner + +import ( + "testing" + "time" + + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + + v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1" + "github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper" + "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb" +) + +func tsdbID(n int) tsdb.SingleTenantTSDBIdentifier { + return tsdb.SingleTenantTSDBIdentifier{ + TS: time.Unix(int64(n), 0), + } +} + +func genMeta(min, max model.Fingerprint, sources []int, blocks []bloomshipper.BlockRef) bloomshipper.Meta { + m := bloomshipper.Meta{ + MetaRef: bloomshipper.MetaRef{ + Ref: bloomshipper.Ref{ + Bounds: v1.NewBounds(min, max), + }, + }, + Blocks: blocks, + } + for _, source := range sources { + m.Sources = append(m.Sources, tsdbID(source)) + } + return m +} + +func Test_gapsBetweenTSDBsAndMetas(t *testing.T) { + + for _, tc := range []struct { + desc string + err bool + exp []tsdbGaps + ownershipRange v1.FingerprintBounds + tsdbs []tsdb.SingleTenantTSDBIdentifier + metas []bloomshipper.Meta + }{ + { + desc: "non-overlapping tsdbs and metas", + err: true, + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0)}, + metas: []bloomshipper.Meta{ + genMeta(11, 20, []int{0}, nil), + }, + }, + { + desc: "single tsdb", + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0)}, + metas: []bloomshipper.Meta{ + genMeta(4, 8, []int{0}, nil), + }, + exp: []tsdbGaps{ + { + tsdb: tsdbID(0), + gaps: []v1.FingerprintBounds{ + v1.NewBounds(0, 3), + v1.NewBounds(9, 10), + }, + }, + }, + }, + { + desc: "multiple tsdbs with separate blocks", + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0), tsdbID(1)}, + metas: []bloomshipper.Meta{ + genMeta(0, 5, []int{0}, nil), + genMeta(6, 10, []int{1}, nil), + }, + exp: []tsdbGaps{ + { + tsdb: tsdbID(0), + gaps: []v1.FingerprintBounds{ + v1.NewBounds(6, 10), + }, + }, + { + tsdb: tsdbID(1), + gaps: []v1.FingerprintBounds{ + v1.NewBounds(0, 5), + }, + }, + }, + }, + { + desc: "multiple tsdbs with the same blocks", + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0), tsdbID(1)}, + metas: []bloomshipper.Meta{ + genMeta(0, 5, []int{0, 1}, nil), + genMeta(6, 8, []int{1}, nil), + }, + exp: []tsdbGaps{ + { + tsdb: tsdbID(0), + gaps: []v1.FingerprintBounds{ + v1.NewBounds(6, 10), + }, + }, + { + tsdb: tsdbID(1), + gaps: []v1.FingerprintBounds{ + v1.NewBounds(9, 10), + }, + }, + }, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + gaps, err := gapsBetweenTSDBsAndMetas(tc.ownershipRange, tc.tsdbs, tc.metas) + if tc.err { + require.Error(t, err) + return + } + require.Equal(t, tc.exp, gaps) + }) + } +} + +func genBlockRef(min, max model.Fingerprint) bloomshipper.BlockRef { + bounds := v1.NewBounds(min, max) + return bloomshipper.BlockRef{ + Ref: bloomshipper.Ref{ + Bounds: bounds, + }, + } +} + +func Test_blockPlansForGaps(t *testing.T) { + for _, tc := range []struct { + desc string + ownershipRange v1.FingerprintBounds + tsdbs []tsdb.SingleTenantTSDBIdentifier + metas []bloomshipper.Meta + err bool + exp []blockPlan + }{ + { + desc: "single overlapping meta+no overlapping block", + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0)}, + metas: []bloomshipper.Meta{ + genMeta(5, 20, []int{1}, []bloomshipper.BlockRef{genBlockRef(11, 20)}), + }, + exp: []blockPlan{ + { + tsdb: tsdbID(0), + gaps: []GapWithBlocks{ + { + bounds: v1.NewBounds(0, 10), + }, + }, + }, + }, + }, + { + desc: "single overlapping meta+one overlapping block", + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0)}, + metas: []bloomshipper.Meta{ + genMeta(5, 20, []int{1}, []bloomshipper.BlockRef{genBlockRef(9, 20)}), + }, + exp: []blockPlan{ + { + tsdb: tsdbID(0), + gaps: []GapWithBlocks{ + { + bounds: v1.NewBounds(0, 10), + blocks: []bloomshipper.BlockRef{genBlockRef(9, 20)}, + }, + }, + }, + }, + }, + { + // the range which needs to be generated doesn't overlap with existing blocks + // from other tsdb versions since theres an up to date tsdb version block, + // but we can trim the range needing generation + desc: "trims up to date area", + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0)}, + metas: []bloomshipper.Meta{ + genMeta(9, 20, []int{0}, []bloomshipper.BlockRef{genBlockRef(9, 20)}), // block for same tsdb + genMeta(9, 20, []int{1}, []bloomshipper.BlockRef{genBlockRef(9, 20)}), // block for different tsdb + }, + exp: []blockPlan{ + { + tsdb: tsdbID(0), + gaps: []GapWithBlocks{ + { + bounds: v1.NewBounds(0, 8), + }, + }, + }, + }, + }, + { + desc: "uses old block for overlapping range", + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0)}, + metas: []bloomshipper.Meta{ + genMeta(9, 20, []int{0}, []bloomshipper.BlockRef{genBlockRef(9, 20)}), // block for same tsdb + genMeta(5, 20, []int{1}, []bloomshipper.BlockRef{genBlockRef(5, 20)}), // block for different tsdb + }, + exp: []blockPlan{ + { + tsdb: tsdbID(0), + gaps: []GapWithBlocks{ + { + bounds: v1.NewBounds(0, 8), + blocks: []bloomshipper.BlockRef{genBlockRef(5, 20)}, + }, + }, + }, + }, + }, + { + desc: "multi case", + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0), tsdbID(1)}, // generate for both tsdbs + metas: []bloomshipper.Meta{ + genMeta(0, 2, []int{0}, []bloomshipper.BlockRef{ + genBlockRef(0, 1), + genBlockRef(1, 2), + }), // tsdb_0 + genMeta(6, 8, []int{0}, []bloomshipper.BlockRef{genBlockRef(6, 8)}), // tsdb_0 + + genMeta(3, 5, []int{1}, []bloomshipper.BlockRef{genBlockRef(3, 5)}), // tsdb_1 + genMeta(8, 10, []int{1}, []bloomshipper.BlockRef{genBlockRef(8, 10)}), // tsdb_1 + }, + exp: []blockPlan{ + { + tsdb: tsdbID(0), + gaps: []GapWithBlocks{ + // tsdb (id=0) can source chunks from the blocks built from tsdb (id=1) + { + bounds: v1.NewBounds(3, 5), + blocks: []bloomshipper.BlockRef{genBlockRef(3, 5)}, + }, + { + bounds: v1.NewBounds(9, 10), + blocks: []bloomshipper.BlockRef{genBlockRef(8, 10)}, + }, + }, + }, + // tsdb (id=1) can source chunks from the blocks built from tsdb (id=0) + { + tsdb: tsdbID(1), + gaps: []GapWithBlocks{ + { + bounds: v1.NewBounds(0, 2), + blocks: []bloomshipper.BlockRef{ + genBlockRef(0, 1), + genBlockRef(1, 2), + }, + }, + { + bounds: v1.NewBounds(6, 7), + blocks: []bloomshipper.BlockRef{genBlockRef(6, 8)}, + }, + }, + }, + }, + }, + { + desc: "dedupes block refs", + ownershipRange: v1.NewBounds(0, 10), + tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0)}, + metas: []bloomshipper.Meta{ + genMeta(9, 20, []int{1}, []bloomshipper.BlockRef{ + genBlockRef(1, 4), + genBlockRef(9, 20), + }), // blocks for first diff tsdb + genMeta(5, 20, []int{2}, []bloomshipper.BlockRef{ + genBlockRef(5, 10), + genBlockRef(9, 20), // same block references in prior meta (will be deduped) + }), // block for second diff tsdb + }, + exp: []blockPlan{ + { + tsdb: tsdbID(0), + gaps: []GapWithBlocks{ + { + bounds: v1.NewBounds(0, 10), + blocks: []bloomshipper.BlockRef{ + genBlockRef(1, 4), + genBlockRef(5, 10), + genBlockRef(9, 20), + }, + }, + }, + }, + }, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + // we reuse the gapsBetweenTSDBsAndMetas function to generate the gaps as this function is tested + // separately and it's used to generate input in our regular code path (easier to write tests this way). + gaps, err := gapsBetweenTSDBsAndMetas(tc.ownershipRange, tc.tsdbs, tc.metas) + require.NoError(t, err) + + plans, err := blockPlansForGaps(gaps, tc.metas) + if tc.err { + require.Error(t, err) + return + } + require.Equal(t, tc.exp, plans) + + }) + } +} diff --git a/pkg/bloombuild/planner/tableIterator.go b/pkg/bloombuild/planner/tableIterator.go new file mode 100644 index 000000000000..c17458a04806 --- /dev/null +++ b/pkg/bloombuild/planner/tableIterator.go @@ -0,0 +1,50 @@ +package planner + +import ( + "fmt" + + "github.com/grafana/loki/v3/pkg/storage/config" +) + +type dayRangeIterator struct { + min, max, cur config.DayTime + curPeriod config.PeriodConfig + schemaCfg config.SchemaConfig + err error +} + +func newDayRangeIterator(min, max config.DayTime, schemaCfg config.SchemaConfig) *dayRangeIterator { + return &dayRangeIterator{min: min, max: max, cur: min.Dec(), schemaCfg: schemaCfg} +} + +func (r *dayRangeIterator) TotalDays() int { + offset := r.cur + if r.cur.Before(r.min) { + offset = r.min + } + return int(r.max.Sub(offset.Time) / config.ObjectStorageIndexRequiredPeriod) +} + +func (r *dayRangeIterator) Next() bool { + r.cur = r.cur.Inc() + if !r.cur.Before(r.max) { + return false + } + + period, err := r.schemaCfg.SchemaForTime(r.cur.ModelTime()) + if err != nil { + r.err = fmt.Errorf("getting schema for time (%s): %w", r.cur, err) + return false + } + r.curPeriod = period + + return true +} + +func (r *dayRangeIterator) At() config.DayTable { + return config.NewDayTable(r.cur, r.curPeriod.IndexTables.Prefix) +} + +func (r *dayRangeIterator) Err() error { + return nil +} diff --git a/pkg/bloombuild/planner/task.go b/pkg/bloombuild/planner/task.go new file mode 100644 index 000000000000..80f730c4fb6d --- /dev/null +++ b/pkg/bloombuild/planner/task.go @@ -0,0 +1,22 @@ +package planner + +import ( + v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1" + "github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper" + "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb" +) + +// TODO: Extract this definiton to a proto file at pkg/bloombuild/protos/protos.proto + +type GapWithBlocks struct { + bounds v1.FingerprintBounds + blocks []bloomshipper.BlockRef +} + +type Task struct { + table string + tenant string + OwnershipBounds v1.FingerprintBounds + tsdb tsdb.SingleTenantTSDBIdentifier + gaps []GapWithBlocks +} diff --git a/pkg/bloombuild/planner/tsdb.go b/pkg/bloombuild/planner/tsdb.go new file mode 100644 index 000000000000..7c15c43306db --- /dev/null +++ b/pkg/bloombuild/planner/tsdb.go @@ -0,0 +1,261 @@ +package planner + +import ( + "context" + "fmt" + "io" + "math" + "path" + "strings" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/pkg/errors" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/labels" + + "github.com/grafana/loki/v3/pkg/chunkenc" + baseStore "github.com/grafana/loki/v3/pkg/storage" + v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1" + "github.com/grafana/loki/v3/pkg/storage/config" + "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/storage" + "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb" + "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb/index" + "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb/sharding" + "github.com/grafana/loki/v3/pkg/storage/types" +) + +const ( + gzipExtension = ".gz" +) + +type TSDBStore interface { + UsersForPeriod(ctx context.Context, table config.DayTable) ([]string, error) + ResolveTSDBs(ctx context.Context, table config.DayTable, tenant string) ([]tsdb.SingleTenantTSDBIdentifier, error) + LoadTSDB( + ctx context.Context, + table config.DayTable, + tenant string, + id tsdb.Identifier, + bounds v1.FingerprintBounds, + ) (v1.Iterator[*v1.Series], error) +} + +// BloomTSDBStore is a wrapper around the storage.Client interface which +// implements the TSDBStore interface for this pkg. +type BloomTSDBStore struct { + storage storage.Client + logger log.Logger +} + +func NewBloomTSDBStore(storage storage.Client, logger log.Logger) *BloomTSDBStore { + return &BloomTSDBStore{ + storage: storage, + logger: logger, + } +} + +func (b *BloomTSDBStore) UsersForPeriod(ctx context.Context, table config.DayTable) ([]string, error) { + _, users, err := b.storage.ListFiles(ctx, table.Addr(), true) // bypass cache for ease of testing + return users, err +} + +func (b *BloomTSDBStore) ResolveTSDBs(ctx context.Context, table config.DayTable, tenant string) ([]tsdb.SingleTenantTSDBIdentifier, error) { + indices, err := b.storage.ListUserFiles(ctx, table.Addr(), tenant, true) // bypass cache for ease of testing + if err != nil { + return nil, errors.Wrap(err, "failed to list user files") + } + + ids := make([]tsdb.SingleTenantTSDBIdentifier, 0, len(indices)) + for _, index := range indices { + key := index.Name + if decompress := storage.IsCompressedFile(index.Name); decompress { + key = strings.TrimSuffix(key, gzipExtension) + } + + id, ok := tsdb.ParseSingleTenantTSDBPath(path.Base(key)) + if !ok { + return nil, errors.Errorf("failed to parse single tenant tsdb path: %s", key) + } + + ids = append(ids, id) + + } + return ids, nil +} + +func (b *BloomTSDBStore) LoadTSDB( + ctx context.Context, + table config.DayTable, + tenant string, + id tsdb.Identifier, + bounds v1.FingerprintBounds, +) (v1.Iterator[*v1.Series], error) { + withCompression := id.Name() + gzipExtension + + data, err := b.storage.GetUserFile(ctx, table.Addr(), tenant, withCompression) + if err != nil { + return nil, errors.Wrap(err, "failed to get file") + } + defer data.Close() + + decompressorPool := chunkenc.GetReaderPool(chunkenc.EncGZIP) + decompressor, err := decompressorPool.GetReader(data) + if err != nil { + return nil, errors.Wrap(err, "failed to get decompressor") + } + defer decompressorPool.PutReader(decompressor) + + buf, err := io.ReadAll(decompressor) + if err != nil { + return nil, errors.Wrap(err, "failed to read file") + } + + reader, err := index.NewReader(index.RealByteSlice(buf)) + if err != nil { + return nil, errors.Wrap(err, "failed to create index reader") + } + + idx := tsdb.NewTSDBIndex(reader) + defer func() { + if err := idx.Close(); err != nil { + level.Error(b.logger).Log("msg", "failed to close index", "err", err) + } + }() + + return NewTSDBSeriesIter(ctx, tenant, idx, bounds) +} + +func NewTSDBSeriesIter(ctx context.Context, user string, f sharding.ForSeries, bounds v1.FingerprintBounds) (v1.Iterator[*v1.Series], error) { + // TODO(salvacorts): Create a pool + series := make([]*v1.Series, 0, 100) + + if err := f.ForSeries( + ctx, + user, + bounds, + 0, math.MaxInt64, + func(_ labels.Labels, fp model.Fingerprint, chks []index.ChunkMeta) (stop bool) { + select { + case <-ctx.Done(): + return true + default: + res := &v1.Series{ + Fingerprint: fp, + Chunks: make(v1.ChunkRefs, 0, len(chks)), + } + for _, chk := range chks { + res.Chunks = append(res.Chunks, v1.ChunkRef{ + From: model.Time(chk.MinTime), + Through: model.Time(chk.MaxTime), + Checksum: chk.Checksum, + }) + } + + series = append(series, res) + return false + } + }, + labels.MustNewMatcher(labels.MatchEqual, "", ""), + ); err != nil { + return nil, err + } + + select { + case <-ctx.Done(): + return v1.NewEmptyIter[*v1.Series](), ctx.Err() + default: + return v1.NewCancelableIter[*v1.Series](ctx, v1.NewSliceIter[*v1.Series](series)), nil + } +} + +type TSDBStores struct { + schemaCfg config.SchemaConfig + stores []TSDBStore +} + +func NewTSDBStores( + schemaCfg config.SchemaConfig, + storeCfg baseStore.Config, + clientMetrics baseStore.ClientMetrics, + logger log.Logger, +) (*TSDBStores, error) { + res := &TSDBStores{ + schemaCfg: schemaCfg, + stores: make([]TSDBStore, len(schemaCfg.Configs)), + } + + for i, cfg := range schemaCfg.Configs { + if cfg.IndexType == types.TSDBType { + + c, err := baseStore.NewObjectClient(cfg.ObjectType, storeCfg, clientMetrics) + if err != nil { + return nil, errors.Wrap(err, "failed to create object client") + } + res.stores[i] = NewBloomTSDBStore(storage.NewIndexStorageClient(c, cfg.IndexTables.PathPrefix), logger) + } + } + + return res, nil +} + +func (s *TSDBStores) storeForPeriod(table config.DayTime) (TSDBStore, error) { + for i := len(s.schemaCfg.Configs) - 1; i >= 0; i-- { + period := s.schemaCfg.Configs[i] + + if !table.Before(period.From) { + // we have the desired period config + + if s.stores[i] != nil { + // valid: it's of tsdb type + return s.stores[i], nil + } + + // invalid + return nil, errors.Errorf( + "store for period is not of TSDB type (%s) while looking up store for (%v)", + period.IndexType, + table, + ) + } + + } + + return nil, fmt.Errorf( + "there is no store matching no matching period found for table (%v) -- too early", + table, + ) +} + +func (s *TSDBStores) UsersForPeriod(ctx context.Context, table config.DayTable) ([]string, error) { + store, err := s.storeForPeriod(table.DayTime) + if err != nil { + return nil, err + } + + return store.UsersForPeriod(ctx, table) +} + +func (s *TSDBStores) ResolveTSDBs(ctx context.Context, table config.DayTable, tenant string) ([]tsdb.SingleTenantTSDBIdentifier, error) { + store, err := s.storeForPeriod(table.DayTime) + if err != nil { + return nil, err + } + + return store.ResolveTSDBs(ctx, table, tenant) +} + +func (s *TSDBStores) LoadTSDB( + ctx context.Context, + table config.DayTable, + tenant string, + id tsdb.Identifier, + bounds v1.FingerprintBounds, +) (v1.Iterator[*v1.Series], error) { + store, err := s.storeForPeriod(table.DayTime) + if err != nil { + return nil, err + } + + return store.LoadTSDB(ctx, table, tenant, id, bounds) +} diff --git a/pkg/bloombuild/planner/tsdb_test.go b/pkg/bloombuild/planner/tsdb_test.go new file mode 100644 index 000000000000..f47c193c2cd1 --- /dev/null +++ b/pkg/bloombuild/planner/tsdb_test.go @@ -0,0 +1,105 @@ +package planner + +import ( + "context" + "math" + "testing" + + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/labels" + "github.com/stretchr/testify/require" + + v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1" + "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb/index" +) + +type forSeriesTestImpl []*v1.Series + +func (f forSeriesTestImpl) ForSeries( + _ context.Context, + _ string, + _ index.FingerprintFilter, + _ model.Time, + _ model.Time, + fn func(labels.Labels, model.Fingerprint, []index.ChunkMeta) bool, + _ ...*labels.Matcher, +) error { + for i := range f { + unmapped := make([]index.ChunkMeta, 0, len(f[i].Chunks)) + for _, c := range f[i].Chunks { + unmapped = append(unmapped, index.ChunkMeta{ + MinTime: int64(c.From), + MaxTime: int64(c.Through), + Checksum: c.Checksum, + }) + } + + fn(nil, f[i].Fingerprint, unmapped) + } + return nil +} + +func (f forSeriesTestImpl) Close() error { + return nil +} + +func TestTSDBSeriesIter(t *testing.T) { + input := []*v1.Series{ + { + Fingerprint: 1, + Chunks: []v1.ChunkRef{ + { + From: 0, + Through: 1, + Checksum: 2, + }, + { + From: 3, + Through: 4, + Checksum: 5, + }, + }, + }, + } + srcItr := v1.NewSliceIter(input) + itr, err := NewTSDBSeriesIter(context.Background(), "", forSeriesTestImpl(input), v1.NewBounds(0, math.MaxUint64)) + require.NoError(t, err) + + v1.EqualIterators[*v1.Series]( + t, + func(a, b *v1.Series) { + require.Equal(t, a, b) + }, + itr, + srcItr, + ) +} + +func TestTSDBSeriesIter_Expiry(t *testing.T) { + t.Run("expires on creation", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + itr, err := NewTSDBSeriesIter(ctx, "", forSeriesTestImpl{ + {}, // a single entry + }, v1.NewBounds(0, math.MaxUint64)) + require.Error(t, err) + require.False(t, itr.Next()) + }) + + t.Run("expires during consumption", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + itr, err := NewTSDBSeriesIter(ctx, "", forSeriesTestImpl{ + {}, + {}, + }, v1.NewBounds(0, math.MaxUint64)) + require.NoError(t, err) + + require.True(t, itr.Next()) + require.NoError(t, itr.Err()) + + cancel() + require.False(t, itr.Next()) + require.Error(t, itr.Err()) + }) + +} diff --git a/pkg/bloombuild/planner/util.go b/pkg/bloombuild/planner/util.go new file mode 100644 index 000000000000..f9a97587f802 --- /dev/null +++ b/pkg/bloombuild/planner/util.go @@ -0,0 +1,125 @@ +package planner + +import ( + "fmt" + "math" + + "github.com/prometheus/common/model" + + v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1" +) + +// SplitFingerprintKeyspaceByFactor splits the keyspace covered by model.Fingerprint into contiguous non-overlapping ranges. +func SplitFingerprintKeyspaceByFactor(factor int) []v1.FingerprintBounds { + if factor <= 0 { + return nil + } + + bounds := make([]v1.FingerprintBounds, 0, factor) + + // The keyspace of a Fingerprint is from 0 to max uint64. + keyspaceSize := uint64(math.MaxUint64) + + // Calculate the size of each range. + rangeSize := keyspaceSize / uint64(factor) + + for i := 0; i < factor; i++ { + // Calculate the start and end of the range. + start := uint64(i) * rangeSize + end := start + rangeSize - 1 + + // For the last range, make sure it ends at the end of the keyspace. + if i == factor-1 { + end = keyspaceSize + } + + // Create a FingerprintBounds for the range and add it to the slice. + bounds = append(bounds, v1.FingerprintBounds{ + Min: model.Fingerprint(start), + Max: model.Fingerprint(end), + }) + } + + return bounds +} + +func FindGapsInFingerprintBounds(ownershipRange v1.FingerprintBounds, metas []v1.FingerprintBounds) (gaps []v1.FingerprintBounds, err error) { + if len(metas) == 0 { + return []v1.FingerprintBounds{ownershipRange}, nil + } + + // turn the available metas into a list of non-overlapping metas + // for easier processing + var nonOverlapping []v1.FingerprintBounds + // First, we reduce the metas into a smaller set by combining overlaps. They must be sorted. + var cur *v1.FingerprintBounds + for i := 0; i < len(metas); i++ { + j := i + 1 + + // first iteration (i == 0), set the current meta + if cur == nil { + cur = &metas[i] + } + + if j >= len(metas) { + // We've reached the end of the list. Add the last meta to the non-overlapping set. + nonOverlapping = append(nonOverlapping, *cur) + break + } + + combined := cur.Union(metas[j]) + if len(combined) == 1 { + // There was an overlap between the two tested ranges. Combine them and keep going. + cur = &combined[0] + continue + } + + // There was no overlap between the two tested ranges. Add the first to the non-overlapping set. + // and keep the second for the next iteration. + nonOverlapping = append(nonOverlapping, combined[0]) + cur = &combined[1] + } + + // Now, detect gaps between the non-overlapping metas and the ownership range. + // The left bound of the ownership range will be adjusted as we go. + leftBound := ownershipRange.Min + for _, meta := range nonOverlapping { + + clippedMeta := meta.Intersection(ownershipRange) + // should never happen as long as we are only combining metas + // that intersect with the ownership range + if clippedMeta == nil { + return nil, fmt.Errorf("meta is not within ownership range: %v", meta) + } + + searchRange := ownershipRange.Slice(leftBound, clippedMeta.Max) + // update the left bound for the next iteration + // We do the max to prevent the max bound to overflow from MaxUInt64 to 0 + leftBound = min( + max(clippedMeta.Max+1, clippedMeta.Max), + max(ownershipRange.Max+1, ownershipRange.Max), + ) + + // since we've already ensured that the meta is within the ownership range, + // we know the xor will be of length zero (when the meta is equal to the ownership range) + // or 1 (when the meta is a subset of the ownership range) + xors := searchRange.Unless(*clippedMeta) + if len(xors) == 0 { + // meta is equal to the ownership range. This means the meta + // covers this entire section of the ownership range. + continue + } + + gaps = append(gaps, xors[0]) + } + + // If the leftBound is less than the ownership range max, and it's smaller than MaxUInt64, + // There is a gap between the last meta and the end of the ownership range. + // Note: we check `leftBound < math.MaxUint64` since in the loop above we clamp the + // leftBound to MaxUint64 to prevent an overflow to 0: `max(clippedMeta.Max+1, clippedMeta.Max)` + if leftBound < math.MaxUint64 && leftBound <= ownershipRange.Max { + gaps = append(gaps, v1.NewBounds(leftBound, ownershipRange.Max)) + } + + return gaps, nil +} diff --git a/pkg/bloombuild/planner/util_test.go b/pkg/bloombuild/planner/util_test.go new file mode 100644 index 000000000000..6755478ef729 --- /dev/null +++ b/pkg/bloombuild/planner/util_test.go @@ -0,0 +1,172 @@ +package planner + +import ( + "math" + "testing" + + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + + v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1" +) + +func TestSplitFingerprintKeyspaceByFactor(t *testing.T) { + for _, tt := range []struct { + name string + factor int + }{ + { + name: "Factor is 0", + factor: 0, + }, + { + name: "Factor is 1", + factor: 1, + }, + { + name: "Factor is 256", + factor: 256, + }, + } { + t.Run(tt.name, func(t *testing.T) { + got := SplitFingerprintKeyspaceByFactor(tt.factor) + + if tt.factor == 0 { + require.Empty(t, got) + return + } + + // Check overall min and max values of the ranges. + require.Equal(t, model.Fingerprint(math.MaxUint64), got[len(got)-1].Max) + require.Equal(t, model.Fingerprint(0), got[0].Min) + + // For each range, check that the max value of the previous range is one less than the min value of the current range. + for i := 1; i < len(got); i++ { + require.Equal(t, got[i-1].Max+1, got[i].Min) + } + }) + } +} + +func Test_FindGapsInFingerprintBounds(t *testing.T) { + for _, tc := range []struct { + desc string + err bool + exp []v1.FingerprintBounds + ownershipRange v1.FingerprintBounds + metas []v1.FingerprintBounds + }{ + { + desc: "error nonoverlapping metas", + err: true, + exp: nil, + ownershipRange: v1.NewBounds(0, 10), + metas: []v1.FingerprintBounds{v1.NewBounds(11, 20)}, + }, + { + desc: "one meta with entire ownership range", + err: false, + exp: nil, + ownershipRange: v1.NewBounds(0, 10), + metas: []v1.FingerprintBounds{v1.NewBounds(0, 10)}, + }, + { + desc: "two non-overlapping metas with entire ownership range", + err: false, + exp: nil, + ownershipRange: v1.NewBounds(0, 10), + metas: []v1.FingerprintBounds{ + v1.NewBounds(0, 5), + v1.NewBounds(6, 10), + }, + }, + { + desc: "two overlapping metas with entire ownership range", + err: false, + exp: nil, + ownershipRange: v1.NewBounds(0, 10), + metas: []v1.FingerprintBounds{ + v1.NewBounds(0, 6), + v1.NewBounds(4, 10), + }, + }, + { + desc: "one meta with partial ownership range", + err: false, + exp: []v1.FingerprintBounds{ + v1.NewBounds(6, 10), + }, + ownershipRange: v1.NewBounds(0, 10), + metas: []v1.FingerprintBounds{ + v1.NewBounds(0, 5), + }, + }, + { + desc: "smaller subsequent meta with partial ownership range", + err: false, + exp: []v1.FingerprintBounds{ + v1.NewBounds(8, 10), + }, + ownershipRange: v1.NewBounds(0, 10), + metas: []v1.FingerprintBounds{ + v1.NewBounds(0, 7), + v1.NewBounds(3, 4), + }, + }, + { + desc: "hole in the middle", + err: false, + exp: []v1.FingerprintBounds{ + v1.NewBounds(4, 5), + }, + ownershipRange: v1.NewBounds(0, 10), + metas: []v1.FingerprintBounds{ + v1.NewBounds(0, 3), + v1.NewBounds(6, 10), + }, + }, + { + desc: "holes on either end", + err: false, + exp: []v1.FingerprintBounds{ + v1.NewBounds(0, 2), + v1.NewBounds(8, 10), + }, + ownershipRange: v1.NewBounds(0, 10), + metas: []v1.FingerprintBounds{ + v1.NewBounds(3, 5), + v1.NewBounds(6, 7), + }, + }, + { + desc: "full ownership range with single meta", + err: false, + exp: nil, + ownershipRange: v1.NewBounds(0, math.MaxUint64), + metas: []v1.FingerprintBounds{ + v1.NewBounds(0, math.MaxUint64), + }, + }, + { + desc: "full ownership range with multiple metas", + err: false, + exp: nil, + ownershipRange: v1.NewBounds(0, math.MaxUint64), + // Three metas covering the whole 0 - MaxUint64 + metas: []v1.FingerprintBounds{ + v1.NewBounds(0, math.MaxUint64/3), + v1.NewBounds(math.MaxUint64/3+1, math.MaxUint64/2), + v1.NewBounds(math.MaxUint64/2+1, math.MaxUint64), + }, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + gaps, err := FindGapsInFingerprintBounds(tc.ownershipRange, tc.metas) + if tc.err { + require.Error(t, err) + return + } + require.Equal(t, tc.exp, gaps) + }) + } +} diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go index a563e80f789f..e73369aca2d7 100644 --- a/pkg/loki/modules.go +++ b/pkg/loki/modules.go @@ -1566,6 +1566,11 @@ func (t *Loki) initBloomPlanner() (services.Service, error) { return planner.New( t.Cfg.BloomBuild.Planner, + t.Overrides, + t.Cfg.SchemaConfig, + t.Cfg.StorageConfig, + t.ClientMetrics, + t.BloomStore, logger, prometheus.DefaultRegisterer, ) diff --git a/pkg/util/limiter/combined_limits.go b/pkg/util/limiter/combined_limits.go index 39684c7b43e8..92caf2c19d68 100644 --- a/pkg/util/limiter/combined_limits.go +++ b/pkg/util/limiter/combined_limits.go @@ -1,6 +1,8 @@ package limiter import ( + bloombuilder "github.com/grafana/loki/v3/pkg/bloombuild/builder" + bloomplanner "github.com/grafana/loki/v3/pkg/bloombuild/planner" "github.com/grafana/loki/v3/pkg/bloomcompactor" "github.com/grafana/loki/v3/pkg/bloomgateway" "github.com/grafana/loki/v3/pkg/compactor" @@ -26,4 +28,6 @@ type CombinedLimits interface { indexgateway.Limits bloomgateway.Limits bloomcompactor.Limits + bloomplanner.Limits + bloombuilder.Limits } diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index ca33d1f4bf42..b0660686f5c1 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -205,6 +205,9 @@ type Limits struct { BloomCompactorMaxBlockSize flagext.ByteSize `yaml:"bloom_compactor_max_block_size" json:"bloom_compactor_max_block_size" category:"experimental"` BloomCompactorMaxBloomSize flagext.ByteSize `yaml:"bloom_compactor_max_bloom_size" json:"bloom_compactor_max_bloom_size" category:"experimental"` + BloomCreationEnabled bool `yaml:"bloom_creation_enabled" json:"bloom_creation_enabled" category:"experimental"` + BloomSplitSeriesKeyspaceBy int `yaml:"bloom_split_series_keyspace_by" json:"bloom_split_series_keyspace_by" category:"experimental"` + BloomNGramLength int `yaml:"bloom_ngram_length" json:"bloom_ngram_length" category:"experimental"` BloomNGramSkip int `yaml:"bloom_ngram_skip" json:"bloom_ngram_skip" category:"experimental"` BloomFalsePositiveRate float64 `yaml:"bloom_false_positive_rate" json:"bloom_false_positive_rate" category:"experimental"` @@ -380,6 +383,9 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { ), ) + f.BoolVar(&l.BloomCreationEnabled, "bloom-build.enable", false, "Experimental. Whether to create blooms for the tenant.") + f.IntVar(&l.BloomSplitSeriesKeyspaceBy, "bloom-build.split-keyspace-by", 256, "Experimental. Number of splits to create for the series keyspace when building blooms. The series keyspace is split into this many parts to parallelize bloom creation.") + _ = l.BloomCompactorMaxBloomSize.Set(defaultBloomCompactorMaxBloomSize) f.Var(&l.BloomCompactorMaxBloomSize, "bloom-compactor.max-bloom-size", fmt.Sprintf( @@ -973,6 +979,14 @@ func (o *Overrides) BloomCompactorEnabled(userID string) bool { return o.getOverridesForUser(userID).BloomCompactorEnabled } +func (o *Overrides) BloomCreationEnabled(userID string) bool { + return o.getOverridesForUser(userID).BloomCreationEnabled +} + +func (o *Overrides) BloomSplitSeriesKeyspaceBy(userID string) int { + return o.getOverridesForUser(userID).BloomSplitSeriesKeyspaceBy +} + func (o *Overrides) BloomNGramLength(userID string) int { return o.getOverridesForUser(userID).BloomNGramLength } From d6f29fc789760318b048d005e14c91eba748b45e Mon Sep 17 00:00:00 2001 From: Vitor Gomes <41302394+vitoorgomes@users.noreply.github.com> Date: Wed, 22 May 2024 04:34:42 +1200 Subject: [PATCH 11/41] docs: update otlp ingestion with correct endpoint and add endpoint to reference api docs (#12996) --- docs/sources/reference/loki-http-api.md | 11 +++++++++++ docs/sources/send-data/otel/_index.md | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/sources/reference/loki-http-api.md b/docs/sources/reference/loki-http-api.md index 1f85a4b48bbc..e72a03ba51b0 100644 --- a/docs/sources/reference/loki-http-api.md +++ b/docs/sources/reference/loki-http-api.md @@ -24,6 +24,7 @@ Authorization needs to be done separately, for example, using an open-source loa These endpoints are exposed by the `distributor`, `write`, and `all` components: - [`POST /loki/api/v1/push`](#ingest-logs) +- [`POST /otlp/v1/logs`](#ingest-logs-using-otlp) A [list of clients]({{< relref "../send-data" >}}) can be found in the clients documentation. @@ -260,6 +261,16 @@ curl -H "Content-Type: application/json" \ --data-raw '{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}' ``` +## Ingest logs using OTLP + +```bash +POST /otlp/v1/logs +``` + +`/otlp/v1/logs` lets the OpenTelemetry Collector send logs to Loki using `otlphttp` procotol. + +For information on how to configure Loki, refer to the [OTel Collector topic](https://grafana.com/docs/loki//send-data/otel/). + ## Query logs at a single point in time ```bash diff --git a/docs/sources/send-data/otel/_index.md b/docs/sources/send-data/otel/_index.md index 27d092a81c09..b7a67fcb14d0 100644 --- a/docs/sources/send-data/otel/_index.md +++ b/docs/sources/send-data/otel/_index.md @@ -30,7 +30,7 @@ You need to make the following changes to the [OpenTelemetry Collector config](h ```yaml exporters: otlphttp: - endpoint: http://:3100/otlp + endpoint: http://:3100/otlp/v1/logs ``` And enable it in `service.pipelines`: @@ -57,7 +57,7 @@ exporters: otlphttp: auth: authenticator: basicauth/otlp - endpoint: http://:3100/otlp + endpoint: http://:3100/otlp/v1/logs service: extensions: [basicauth/otlp] From efd8f5dc1b3bb3313de1ed6b26750d5bd5632b16 Mon Sep 17 00:00:00 2001 From: Salva Corts Date: Wed, 22 May 2024 10:43:32 +0200 Subject: [PATCH 12/41] refactor(blooms): Add queue to bloom planner and enqueue tasks (#13005) --- docs/sources/shared/configuration.md | 9 ++++ pkg/bloombuild/planner/config.go | 32 ++++++++++-- pkg/bloombuild/planner/metrics.go | 40 ++++++++++++++- pkg/bloombuild/planner/planner.go | 73 +++++++++++++++++++++------- pkg/bloombuild/planner/task.go | 7 +++ pkg/validation/limits.go | 6 +++ 6 files changed, 145 insertions(+), 22 deletions(-) diff --git a/docs/sources/shared/configuration.md b/docs/sources/shared/configuration.md index d30dce2f7775..acf11102be51 100644 --- a/docs/sources/shared/configuration.md +++ b/docs/sources/shared/configuration.md @@ -351,6 +351,10 @@ bloom_build: # CLI flag: -bloom-build.planner.max-table-offset [max_table_offset: | default = 2] + # Maximum number of tasks to queue per tenant. + # CLI flag: -bloom-build.planner.max-tasks-per-tenant + [max_queued_tasks_per_tenant: | default = 30000] + builder: # Experimental: The bloom_gateway block configures the Loki bloom gateway @@ -3409,6 +3413,11 @@ shard_streams: # CLI flag: -bloom-build.split-keyspace-by [bloom_split_series_keyspace_by: | default = 256] +# Experimental. Maximum number of builders to use when building blooms. 0 allows +# unlimited builders. +# CLI flag: -bloom-build.max-builders +[bloom_build_max_builders: | default = 0] + # Experimental. Length of the n-grams created when computing blooms from log # lines. # CLI flag: -bloom-compactor.ngram-length diff --git a/pkg/bloombuild/planner/config.go b/pkg/bloombuild/planner/config.go index 47b01c0b286e..aff25873b12f 100644 --- a/pkg/bloombuild/planner/config.go +++ b/pkg/bloombuild/planner/config.go @@ -8,9 +8,10 @@ import ( // Config configures the bloom-planner component. type Config struct { - PlanningInterval time.Duration `yaml:"planning_interval"` - MinTableOffset int `yaml:"min_table_offset"` - MaxTableOffset int `yaml:"max_table_offset"` + PlanningInterval time.Duration `yaml:"planning_interval"` + MinTableOffset int `yaml:"min_table_offset"` + MaxTableOffset int `yaml:"max_table_offset"` + MaxQueuedTasksPerTenant int `yaml:"max_queued_tasks_per_tenant"` } // RegisterFlagsWithPrefix registers flags for the bloom-planner configuration. @@ -24,6 +25,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { // dynamically reloaded. // I'm doing it the simple way for now. f.IntVar(&cfg.MaxTableOffset, prefix+".max-table-offset", 2, "Oldest day-table offset (from today, inclusive) to compact. This can be used to lower cost by not trying to compact older data which doesn't change. This can be optimized by aligning it with the maximum `reject_old_samples_max_age` setting of any tenant.") + f.IntVar(&cfg.MaxQueuedTasksPerTenant, prefix+".max-tasks-per-tenant", 30000, "Maximum number of tasks to queue per tenant.") } func (cfg *Config) Validate() error { @@ -37,4 +39,28 @@ func (cfg *Config) Validate() error { type Limits interface { BloomCreationEnabled(tenantID string) bool BloomSplitSeriesKeyspaceBy(tenantID string) int + BloomBuildMaxBuilders(tenantID string) int +} + +type QueueLimits struct { + limits Limits +} + +func NewQueueLimits(limits Limits) *QueueLimits { + return &QueueLimits{limits: limits} +} + +// MaxConsumers is used to compute how many of the available builders are allowed to handle tasks for a given tenant. +// 0 is returned when neither limits are applied. 0 means all builders can be used. +func (c *QueueLimits) MaxConsumers(tenantID string, allConsumers int) int { + if c == nil || c.limits == nil { + return 0 + } + + maxBuilders := c.limits.BloomBuildMaxBuilders(tenantID) + if maxBuilders == 0 { + return 0 + } + + return min(allConsumers, maxBuilders) } diff --git a/pkg/bloombuild/planner/metrics.go b/pkg/bloombuild/planner/metrics.go index c0028237d9b1..347af1926617 100644 --- a/pkg/bloombuild/planner/metrics.go +++ b/pkg/bloombuild/planner/metrics.go @@ -1,8 +1,12 @@ package planner import ( + "time" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + + "github.com/grafana/loki/v3/pkg/queue" ) const ( @@ -16,6 +20,11 @@ const ( type Metrics struct { running prometheus.Gauge + // Extra Queue metrics + connectedBuilders prometheus.GaugeFunc + queueDuration prometheus.Histogram + inflightRequests prometheus.Summary + buildStarted prometheus.Counter buildCompleted *prometheus.CounterVec buildTime *prometheus.HistogramVec @@ -23,7 +32,10 @@ type Metrics struct { tenantsDiscovered prometheus.Counter } -func NewMetrics(r prometheus.Registerer) *Metrics { +func NewMetrics( + r prometheus.Registerer, + getConnectedBuilders func() float64, +) *Metrics { return &Metrics{ running: promauto.With(r).NewGauge(prometheus.GaugeOpts{ Namespace: metricsNamespace, @@ -31,6 +43,28 @@ func NewMetrics(r prometheus.Registerer) *Metrics { Name: "running", Help: "Value will be 1 if bloom planner is currently running on this instance", }), + connectedBuilders: promauto.With(r).NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "connected_builders", + Help: "Number of builders currently connected to the planner.", + }, getConnectedBuilders), + queueDuration: promauto.With(r).NewHistogram(prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "queue_duration_seconds", + Help: "Time spend by tasks in queue before getting picked up by a builder.", + Buckets: prometheus.DefBuckets, + }), + inflightRequests: promauto.With(r).NewSummary(prometheus.SummaryOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "inflight_tasks", + Help: "Number of inflight tasks (either queued or processing) sampled at a regular interval. Quantile buckets keep track of inflight tasks over the last 60s.", + Objectives: map[float64]float64{0.5: 0.05, 0.75: 0.02, 0.8: 0.02, 0.9: 0.01, 0.95: 0.01, 0.99: 0.001}, + MaxAge: time.Minute, + AgeBuckets: 6, + }), buildStarted: promauto.With(r).NewCounter(prometheus.CounterOpts{ Namespace: metricsNamespace, @@ -60,3 +94,7 @@ func NewMetrics(r prometheus.Registerer) *Metrics { }), } } + +func NewQueueMetrics(r prometheus.Registerer) *queue.Metrics { + return queue.NewMetrics(r, metricsNamespace, metricsSubsystem) +} diff --git a/pkg/bloombuild/planner/planner.go b/pkg/bloombuild/planner/planner.go index 0be853a2f604..9a5b9f6dc238 100644 --- a/pkg/bloombuild/planner/planner.go +++ b/pkg/bloombuild/planner/planner.go @@ -12,16 +12,21 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/grafana/loki/v3/pkg/queue" "github.com/grafana/loki/v3/pkg/storage" v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1" "github.com/grafana/loki/v3/pkg/storage/config" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb" + "github.com/grafana/loki/v3/pkg/util" utillog "github.com/grafana/loki/v3/pkg/util/log" ) type Planner struct { services.Service + // Subservices manager. + subservices *services.Manager + subservicesWatcher *services.FailureWatcher cfg Config limits Limits @@ -30,6 +35,9 @@ type Planner struct { tsdbStore TSDBStore bloomStore bloomshipper.Store + tasksQueue *queue.RequestQueue + activeUsers *util.ActiveUsersCleanupService + metrics *Metrics logger log.Logger } @@ -51,15 +59,34 @@ func New( return nil, fmt.Errorf("error creating TSDB store: %w", err) } + // Queue to manage tasks + queueMetrics := NewQueueMetrics(r) + tasksQueue := queue.NewRequestQueue(cfg.MaxQueuedTasksPerTenant, 0, NewQueueLimits(limits), queueMetrics) + + // Clean metrics for inactive users: do not have added tasks to the queue in the last 1 hour + activeUsers := util.NewActiveUsersCleanupService(5*time.Minute, 1*time.Hour, func(user string) { + queueMetrics.Cleanup(user) + }) + p := &Planner{ - cfg: cfg, - limits: limits, - schemaCfg: schemaCfg, - tsdbStore: tsdbStore, - bloomStore: bloomStore, - metrics: NewMetrics(r), - logger: logger, + cfg: cfg, + limits: limits, + schemaCfg: schemaCfg, + tsdbStore: tsdbStore, + bloomStore: bloomStore, + tasksQueue: tasksQueue, + activeUsers: activeUsers, + metrics: NewMetrics(r, tasksQueue.GetConnectedConsumersMetric), + logger: logger, + } + + svcs := []services.Service{p.tasksQueue, p.activeUsers} + p.subservices, err = services.NewManager(svcs...) + if err != nil { + return nil, fmt.Errorf("error creating subservices manager: %w", err) } + p.subservicesWatcher = services.NewFailureWatcher() + p.subservicesWatcher.WatchManager(p.subservices) p.Service = services.NewBasicService(p.starting, p.running, p.stopping) return p, nil @@ -91,6 +118,7 @@ func (p *Planner) running(ctx context.Context) error { return err case <-ticker.C: + level.Info(p.logger).Log("msg", "starting bloom build iteration") if err := p.runOne(ctx); err != nil { level.Error(p.logger).Log("msg", "bloom build iteration failed", "err", err) } @@ -109,44 +137,53 @@ func (p *Planner) runOne(ctx context.Context) error { }() p.metrics.buildStarted.Inc() - level.Info(p.logger).Log("msg", "running bloom build planning") tables := p.tables(time.Now()) level.Debug(p.logger).Log("msg", "loaded tables", "tables", tables.TotalDays()) work, err := p.loadWork(ctx, tables) if err != nil { - level.Error(p.logger).Log("msg", "error loading work", "err", err) return fmt.Errorf("error loading work: %w", err) } - // TODO: Enqueue instead of buffering here - // This is just a placeholder for now - var tasks []Task - + var totalTasks int for _, w := range work { + logger := log.With(p.logger, "tenant", w.tenant, "table", w.table.Addr(), "ownership", w.ownershipRange.String()) + gaps, err := p.findGapsForBounds(ctx, w.tenant, w.table, w.ownershipRange) if err != nil { - level.Error(p.logger).Log("msg", "error finding gaps", "err", err, "tenant", w.tenant, "table", w.table, "ownership", w.ownershipRange.String()) - return fmt.Errorf("error finding gaps for tenant (%s) in table (%s) for bounds (%s): %w", w.tenant, w.table, w.ownershipRange, err) + level.Error(logger).Log("msg", "error finding gaps", "err", err) + continue } + now := time.Now() for _, gap := range gaps { - tasks = append(tasks, Task{ + totalTasks++ + task := Task{ table: w.table.Addr(), tenant: w.tenant, OwnershipBounds: w.ownershipRange, tsdb: gap.tsdb, gaps: gap.gaps, - }) + + queueTime: now, + ctx: ctx, + } + + p.activeUsers.UpdateUserTimestamp(task.tenant, now) + if err := p.tasksQueue.Enqueue(task.tenant, nil, task, nil); err != nil { + level.Error(logger).Log("msg", "error enqueuing task", "err", err) + continue + } } } + level.Debug(p.logger).Log("msg", "planning completed", "tasks", totalTasks) + status = statusSuccess level.Info(p.logger).Log( "msg", "bloom build iteration completed", "duration", time.Since(start).Seconds(), - "tasks", len(tasks), ) return nil } diff --git a/pkg/bloombuild/planner/task.go b/pkg/bloombuild/planner/task.go index 80f730c4fb6d..bff459fe1764 100644 --- a/pkg/bloombuild/planner/task.go +++ b/pkg/bloombuild/planner/task.go @@ -1,6 +1,9 @@ package planner import ( + "context" + "time" + v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb" @@ -19,4 +22,8 @@ type Task struct { OwnershipBounds v1.FingerprintBounds tsdb tsdb.SingleTenantTSDBIdentifier gaps []GapWithBlocks + + // Tracking + queueTime time.Time + ctx context.Context } diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index b0660686f5c1..148205b306c1 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -207,6 +207,7 @@ type Limits struct { BloomCreationEnabled bool `yaml:"bloom_creation_enabled" json:"bloom_creation_enabled" category:"experimental"` BloomSplitSeriesKeyspaceBy int `yaml:"bloom_split_series_keyspace_by" json:"bloom_split_series_keyspace_by" category:"experimental"` + BloomBuildMaxBuilders int `yaml:"bloom_build_max_builders" json:"bloom_build_max_builders" category:"experimental"` BloomNGramLength int `yaml:"bloom_ngram_length" json:"bloom_ngram_length" category:"experimental"` BloomNGramSkip int `yaml:"bloom_ngram_skip" json:"bloom_ngram_skip" category:"experimental"` @@ -385,6 +386,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&l.BloomCreationEnabled, "bloom-build.enable", false, "Experimental. Whether to create blooms for the tenant.") f.IntVar(&l.BloomSplitSeriesKeyspaceBy, "bloom-build.split-keyspace-by", 256, "Experimental. Number of splits to create for the series keyspace when building blooms. The series keyspace is split into this many parts to parallelize bloom creation.") + f.IntVar(&l.BloomBuildMaxBuilders, "bloom-build.max-builders", 0, "Experimental. Maximum number of builders to use when building blooms. 0 allows unlimited builders.") _ = l.BloomCompactorMaxBloomSize.Set(defaultBloomCompactorMaxBloomSize) f.Var(&l.BloomCompactorMaxBloomSize, "bloom-compactor.max-bloom-size", @@ -987,6 +989,10 @@ func (o *Overrides) BloomSplitSeriesKeyspaceBy(userID string) int { return o.getOverridesForUser(userID).BloomSplitSeriesKeyspaceBy } +func (o *Overrides) BloomBuildMaxBuilders(userID string) int { + return o.getOverridesForUser(userID).BloomBuildMaxBuilders +} + func (o *Overrides) BloomNGramLength(userID string) int { return o.getOverridesForUser(userID).BloomNGramLength } From 1948899999107e7f27f4b9faace64942abcdb41f Mon Sep 17 00:00:00 2001 From: Quentin Bisson Date: Wed, 22 May 2024 15:16:29 +0200 Subject: [PATCH 13/41] fix: Mixins - Add missing log datasource on loki-deletion (#13011) --- .../dashboards/loki-deletion.json | 10 ++++++++++ .../loki-mixin-compiled/dashboards/loki-deletion.json | 10 ++++++++++ .../loki-mixin/dashboards/loki-deletion.libsonnet | 1 + 3 files changed, 21 insertions(+) diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-deletion.json b/production/loki-mixin-compiled-ssd/dashboards/loki-deletion.json index 8ce9529906aa..0718507d941f 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-deletion.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-deletion.json @@ -701,6 +701,16 @@ "tagsQuery": "", "type": "query", "useTags": false + }, + { + "hide": 0, + "label": null, + "name": "loki_datasource", + "options": [ ], + "query": "loki", + "refresh": 1, + "regex": "", + "type": "datasource" } ] }, diff --git a/production/loki-mixin-compiled/dashboards/loki-deletion.json b/production/loki-mixin-compiled/dashboards/loki-deletion.json index 2db2b7cb3658..7b048f729e2b 100644 --- a/production/loki-mixin-compiled/dashboards/loki-deletion.json +++ b/production/loki-mixin-compiled/dashboards/loki-deletion.json @@ -701,6 +701,16 @@ "tagsQuery": "", "type": "query", "useTags": false + }, + { + "hide": 0, + "label": null, + "name": "loki_datasource", + "options": [ ], + "query": "loki", + "refresh": 1, + "regex": "", + "type": "datasource" } ] }, diff --git a/production/loki-mixin/dashboards/loki-deletion.libsonnet b/production/loki-mixin/dashboards/loki-deletion.libsonnet index 0ddb39fea695..a1c50ecfa691 100644 --- a/production/loki-mixin/dashboards/loki-deletion.libsonnet +++ b/production/loki-mixin/dashboards/loki-deletion.libsonnet @@ -12,6 +12,7 @@ local utils = import 'mixin-utils/utils.libsonnet'; .addCluster() .addNamespace() .addTag() + .addLog() .addRow( ($.row('Headlines') + { From 8d9fb68ae5d4f26ddc2ae184a1cb6a3b2a2c2127 Mon Sep 17 00:00:00 2001 From: Quentin Bisson Date: Wed, 22 May 2024 22:00:08 +0200 Subject: [PATCH 14/41] fix: remove unneccessary disk panels for ssd read path (#13014) Signed-off-by: QuentinBisson --- .../dashboards/loki-reads-resources.json | 147 +----------------- .../dashboards/loki-reads-resources.libsonnet | 20 ++- 2 files changed, 21 insertions(+), 146 deletions(-) diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-reads-resources.json b/production/loki-mixin-compiled-ssd/dashboards/loki-reads-resources.json index d2114319a5cc..ed5fc3dd245d 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-reads-resources.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-reads-resources.json @@ -277,147 +277,6 @@ "sort": 2 }, "type": "timeseries" - }, - { - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 100, - "lineWidth": 0, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ ] - }, - "unit": "Bps" - }, - "overrides": [ ] - }, - "gridPos": { }, - "id": 4, - "links": [ ], - "options": { - "legend": { - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-read.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", - "format": "time_series", - "legendFormat": "{{pod}} - {{device}}", - "legendLink": null - } - ], - "title": "Disk Writes", - "type": "timeseries" - }, - { - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 100, - "lineWidth": 0, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ ] - }, - "unit": "Bps" - }, - "overrides": [ ] - }, - "gridPos": { }, - "id": 5, - "links": [ ], - "options": { - "legend": { - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-read.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", - "format": "time_series", - "legendFormat": "{{pod}} - {{device}}", - "legendLink": null - } - ], - "title": "Disk Reads", - "type": "timeseries" - }, - { - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ ] - }, - "unit": "percentunit" - }, - "overrides": [ ] - }, - "gridPos": { }, - "id": 6, - "links": [ ], - "options": { - "legend": { - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"(loki.*|enterprise-logs)-read.*\"})", - "format": "time_series", - "legendFormat": "{{persistentvolumeclaim}}", - "legendLink": null - } - ], - "title": "Disk Space Utilization", - "type": "timeseries" } ], "repeat": null, @@ -495,7 +354,7 @@ } ] }, - "id": 7, + "id": 4, "links": [ ], "options": { "legend": { @@ -596,7 +455,7 @@ } ] }, - "id": 8, + "id": 5, "links": [ ], "options": { "legend": { @@ -658,7 +517,7 @@ }, "overrides": [ ] }, - "id": 9, + "id": 6, "links": [ ], "options": { "legend": { diff --git a/production/loki-mixin/dashboards/loki-reads-resources.libsonnet b/production/loki-mixin/dashboards/loki-reads-resources.libsonnet index 21db04ea2cf8..0ec22e131edb 100644 --- a/production/loki-mixin/dashboards/loki-reads-resources.libsonnet +++ b/production/loki-mixin/dashboards/loki-reads-resources.libsonnet @@ -94,8 +94,24 @@ local utils = import 'mixin-utils/utils.libsonnet'; $.containerDiskSpaceUtilizationPanel('Disk Space Utilization', 'querier'), ) ) - .addRow( - grafana.row.new(if $._config.ssd.enabled then 'Read path' else 'Index Gateway') + // Add the read path for single scalable deployment only. The read path should not display disk utilization as the index gateway is present in the backend pods. + .addRowIf( + $._config.ssd.enabled, + grafana.row.new('Read path') + .addPanel( + $.CPUUsagePanel('CPU', index_gateway_pod_matcher), + ) + .addPanel( + $.memoryWorkingSetPanel('Memory (workingset)', index_gateway_pod_matcher), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', index_gateway_job_matcher), + ) + ) + // Otherwise we add the index gateway information + .addRowIf( + !$._config.ssd.enabled, + grafana.row.new('Index Gateway') .addPanel( $.CPUUsagePanel('CPU', index_gateway_pod_matcher), ) From d3c9cec22891b45ed1cb93a9eacc5dad6a117fc5 Mon Sep 17 00:00:00 2001 From: Quentin Bisson Date: Thu, 23 May 2024 01:59:28 +0200 Subject: [PATCH 15/41] fix: upgrade old plugin for the loki-operational dashboard. (#13016) Signed-off-by: QuentinBisson --- .../dashboards/loki-operational.json | 176 ++++++++++------- .../dashboards/loki-operational.json | 176 ++++++++++------- .../dashboard-loki-operational.json | 178 +++++++++++------- 3 files changed, 322 insertions(+), 208 deletions(-) diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-operational.json b/production/loki-mixin-compiled-ssd/dashboards/loki-operational.json index d9f668ed88a3..911e9f7267d7 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-operational.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-operational.json @@ -1657,9 +1657,93 @@ } }, { - "columns": [ ], "datasource": "$datasource", - "fontSize": "100%", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "right", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "decimals": 2, + "displayName": "", + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "tenant" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "reason" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + } + ] + }, "gridPos": { "h": 8, "w": 12, @@ -1667,71 +1751,20 @@ "y": 27 }, "id": 113, - "pageSize": null, - "panels": [ ], - "showHeader": true, - "sort": { - "col": 3, - "desc": true - }, - "styles": [ - { - "alias": "Time", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "tenant", - "thresholds": [ ], - "type": "string", - "unit": "short" - }, - { - "alias": "", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "reason", - "thresholds": [ ], - "type": "number", - "unit": "short" + "show": false }, - { - "alias": "", - "align": "right", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ ], - "type": "number", - "unit": "short" - } - ], + "showHeader": true + }, + "panels": [ ], + "pluginVersion": "10.4.0", "targets": [ { "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])[$__range:$__rate_interval])))", @@ -1742,11 +1775,16 @@ "refId": "A" } ], - "timeFrom": null, - "timeShift": null, "title": "Discarded Lines Per Interval", - "transform": "table", - "type": "table-old" + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [ ] + } + } + ], + "type": "table" } ], "targets": [ ], diff --git a/production/loki-mixin-compiled/dashboards/loki-operational.json b/production/loki-mixin-compiled/dashboards/loki-operational.json index d677775d52ef..48684c77e5d9 100644 --- a/production/loki-mixin-compiled/dashboards/loki-operational.json +++ b/production/loki-mixin-compiled/dashboards/loki-operational.json @@ -1754,9 +1754,93 @@ } }, { - "columns": [ ], "datasource": "$datasource", - "fontSize": "100%", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "right", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "decimals": 2, + "displayName": "", + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "tenant" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "reason" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + } + ] + }, "gridPos": { "h": 8, "w": 12, @@ -1764,71 +1848,20 @@ "y": 27 }, "id": 113, - "pageSize": null, - "panels": [ ], - "showHeader": true, - "sort": { - "col": 3, - "desc": true - }, - "styles": [ - { - "alias": "Time", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "tenant", - "thresholds": [ ], - "type": "string", - "unit": "short" - }, - { - "alias": "", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "reason", - "thresholds": [ ], - "type": "number", - "unit": "short" + "show": false }, - { - "alias": "", - "align": "right", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ ], - "type": "number", - "unit": "short" - } - ], + "showHeader": true + }, + "panels": [ ], + "pluginVersion": "10.4.0", "targets": [ { "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])[$__range:$__rate_interval])))", @@ -1839,11 +1872,16 @@ "refId": "A" } ], - "timeFrom": null, - "timeShift": null, "title": "Discarded Lines Per Interval", - "transform": "table", - "type": "table-old" + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [ ] + } + } + ], + "type": "table" } ], "targets": [ ], diff --git a/production/loki-mixin/dashboards/dashboard-loki-operational.json b/production/loki-mixin/dashboards/dashboard-loki-operational.json index 3f215c2e9083..da51f8612ed8 100644 --- a/production/loki-mixin/dashboards/dashboard-loki-operational.json +++ b/production/loki-mixin/dashboards/dashboard-loki-operational.json @@ -1741,9 +1741,93 @@ } }, { - "columns": [], "datasource": "$datasource", - "fontSize": "100%", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "right", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "decimals": 2, + "displayName": "", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "tenant" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "reason" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + } + ] + }, "gridPos": { "h": 8, "w": 12, @@ -1751,70 +1835,20 @@ "y": 27 }, "id": 113, - "pageSize": null, - "showHeader": true, - "sort": { - "col": 3, - "desc": true - }, - "styles": [ - { - "alias": "Time", - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "tenant", - "thresholds": [], - "type": "string", - "unit": "short" - }, - { - "alias": "", - "align": "auto", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "reason", - "thresholds": [], - "type": "number", - "unit": "short" + "show": false }, - { - "alias": "", - "align": "right", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], + "showHeader": true + }, + "panels": [], + "pluginVersion": "10.4.0", "targets": [ { "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])[$__range:$__rate_interval])))", @@ -1825,11 +1859,16 @@ "refId": "A" } ], - "timeFrom": null, - "timeShift": null, "title": "Discarded Lines Per Interval", - "transform": "table", - "type": "table-old" + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" } ], "title": "Limits", @@ -1852,7 +1891,6 @@ "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 1, "fillGradient": 0, "gridPos": { @@ -2244,7 +2282,7 @@ "dashes": false, "datasource": "$datasource", "fieldConfig": { - "defaults": { + "defaults": { "unit": "binBps" } }, From 987e551f9e21b9a612dd0b6a3e60503ce6fe13a8 Mon Sep 17 00:00:00 2001 From: Quentin Bisson Date: Thu, 23 May 2024 02:15:52 +0200 Subject: [PATCH 16/41] fix: allow cluster label override in bloom dashboards (#13012) Signed-off-by: QuentinBisson --- .../dashboards/loki-bloom-compactor.json | 74 +++++++++++++++++++ .../dashboards/loki-bloom-gateway.json | 57 ++++++++++++++ .../dashboards/loki-bloom-compactor.json | 74 +++++++++++++++++++ .../dashboards/loki-bloom-gateway.json | 57 ++++++++++++++ .../dashboards/loki-bloom-compactor.libsonnet | 65 ++++++++++++++-- .../dashboards/loki-bloom-gateway.libsonnet | 65 ++++++++++++++-- 6 files changed, 380 insertions(+), 12 deletions(-) diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-compactor.json b/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-compactor.json index c667d0c01ecc..c365fab0a7e5 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-compactor.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-compactor.json @@ -32,6 +32,7 @@ }, "id": 111, "panels": [ ], + "targets": [ ], "title": "Overview", "type": "row" }, @@ -52,7 +53,9 @@ "content": "## About the Bloom Compactor\nThe compactor iterates through chunks and creates blooms out of them.\nThe size of the resulting blooms depends on the bloom filter settings, the tokenizer settings, the number of ring tokens per compactor and the total number opf compactors.\n\nCompactors are horizontally scalable and uses a ring to:\n- Shard tenants\n- Shard series fingerprints within a tenant subring.\n\nThe blooms for the series are grouped together in blocks which are flushed to object store.", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "transparent": true, "type": "text" }, @@ -140,6 +143,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -290,6 +294,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -371,6 +376,7 @@ "sortOrder": "Descending", "wrapLogMessage": false }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -467,6 +473,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -576,6 +583,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -645,6 +653,7 @@ "showThresholdMarkers": false, "sizing": "auto" }, + "panels": [ ], "pluginVersion": "11.0.0-68102", "targets": [ { @@ -665,6 +674,7 @@ "type": "gauge" } ], + "targets": [ ], "title": "Progress per pod", "type": "row" }, @@ -696,7 +706,9 @@ "content": "", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "title": "We use tenant sharding so each compactor will process a subset of the tenants.", "transparent": true, "type": "text" @@ -777,6 +789,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -893,6 +906,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -1000,6 +1014,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -1149,6 +1164,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -1273,6 +1289,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -1448,6 +1465,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -1576,6 +1594,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -1678,6 +1697,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -1697,6 +1717,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Resource Usage", "type": "row" }, @@ -1791,6 +1812,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -1904,6 +1926,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -2004,6 +2027,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -2130,6 +2154,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -2150,6 +2175,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Bloom building", "type": "row" }, @@ -2238,6 +2264,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2332,6 +2359,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2426,6 +2454,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2445,6 +2474,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Blocks building", "type": "row" }, @@ -2533,6 +2563,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -2628,6 +2659,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -2648,6 +2680,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Metas building", "type": "row" }, @@ -2679,7 +2712,9 @@ "content": "", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "title": "We use tenant sharding so each compactor will process a subset of the tenants.", "transparent": true, "type": "text" @@ -2759,6 +2794,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2878,6 +2914,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2914,7 +2951,9 @@ "content": "", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "title": "Number of tenant tables processed. ", "transparent": true, "type": "text" @@ -2994,6 +3033,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3113,6 +3153,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3149,7 +3190,9 @@ "content": "", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "title": "Series per compaction (includes series copied from other blocks)", "transparent": true, "type": "text" @@ -3230,6 +3273,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3349,6 +3393,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3385,7 +3430,9 @@ "content": "", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "title": "Number of bytes from chunks added to blocks during each compaction.", "transparent": true, "type": "text" @@ -3466,6 +3513,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3586,6 +3634,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3605,6 +3654,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Data processed", "type": "row" }, @@ -3640,7 +3690,9 @@ "content": "\nCompactors write blocks to the attached PVs before flushing them into the object store.\nIt also download chunks and index files.\n\nAfter compacting a given tenant, all the downloaded index files and chunks, as well as the already flushed blocks are deleted.", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-69747", + "targets": [ ], "title": "", "transparent": true, "type": "text" @@ -3726,6 +3778,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3850,6 +3903,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3946,6 +4000,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4068,6 +4123,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4164,6 +4220,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4286,6 +4343,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4304,6 +4362,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Disk Usage", "type": "row" }, @@ -4339,7 +4398,9 @@ "content": "Once all blocks and metas are built locally, the compactor flushes them to the object store.\n\nAfter each iteration, the compactor deletes the metas and blocks marked for deletion in the tombstones.", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-69747", + "targets": [ ], "title": "", "transparent": true, "type": "text" @@ -4366,7 +4427,9 @@ "content": "---\n#### GCS\n", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-69747", + "targets": [ ], "title": "", "transparent": true, "type": "text" @@ -4447,6 +4510,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4541,6 +4605,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4607,7 +4672,9 @@ "content": "---\n#### S3\n", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-69747", + "targets": [ ], "title": "", "transparent": true, "type": "text" @@ -4688,6 +4755,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4782,6 +4850,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4848,7 +4917,9 @@ "content": "---\n#### Azure\nBlob Storage", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-69747", + "targets": [ ], "title": "", "transparent": true, "type": "text" @@ -4929,6 +5000,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -5023,6 +5095,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -5068,6 +5141,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Object Store", "type": "row" } diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-gateway.json b/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-gateway.json index 27a058ae800e..2d5e16a9d7e0 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-gateway.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-gateway.json @@ -33,6 +33,7 @@ }, "id": 73, "panels": [ ], + "targets": [ ], "title": "Overview", "type": "row" }, @@ -131,6 +132,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -223,6 +225,7 @@ "showThresholdMarkers": true, "sizing": "auto" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", "targets": [ { @@ -395,6 +398,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -536,6 +540,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -577,6 +582,7 @@ "sortOrder": "Descending", "wrapLogMessage": true }, + "panels": [ ], "targets": [ { "datasource": { @@ -730,6 +736,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -847,6 +854,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -970,6 +978,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -1094,6 +1103,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -1234,6 +1244,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -1336,6 +1347,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -1437,6 +1449,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -1547,6 +1560,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -1647,6 +1661,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -1666,6 +1681,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Resource usage", "type": "row" }, @@ -1799,6 +1815,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -1937,6 +1954,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2032,6 +2050,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2152,6 +2171,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2171,6 +2191,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "QPS and Latency", "type": "row" }, @@ -2260,6 +2281,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2380,6 +2402,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2513,6 +2536,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2532,6 +2556,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Task Queue", "type": "row" }, @@ -2625,6 +2650,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2748,6 +2774,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2870,6 +2897,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2901,6 +2929,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Processing", "type": "row" }, @@ -2936,7 +2965,9 @@ "content": "", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "title": "We cache bloom blocks in memory to prevent the gateway from hitting the object store too often", "transparent": true, "type": "text" @@ -3021,6 +3052,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3178,6 +3210,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3347,6 +3380,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3458,6 +3492,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3581,6 +3616,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3678,6 +3714,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Blocks Cache", "type": "row" }, @@ -3713,7 +3750,9 @@ "content": "", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "title": "The gateway download bloom meta files and blocks from the object store.", "transparent": true, "type": "text" @@ -3740,7 +3779,9 @@ "content": "---\n#### GCS\n", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "transparent": true, "type": "text" }, @@ -3820,6 +3861,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3913,6 +3955,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3978,7 +4021,9 @@ "content": "---\n#### S3\n", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "transparent": true, "type": "text" }, @@ -4058,6 +4103,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4151,6 +4197,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4216,7 +4263,9 @@ "content": "---\n#### Azure\nBlob Storage\n\n", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "transparent": true, "type": "text" }, @@ -4296,6 +4345,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4389,6 +4439,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4433,6 +4484,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Object Store", "type": "row" }, @@ -4528,6 +4580,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4652,6 +4705,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4770,6 +4824,7 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, + "panels": [ ], "pluginVersion": "11.0.0-67814", "targets": [ { @@ -4882,6 +4937,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4912,6 +4968,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Misc", "type": "row" } diff --git a/production/loki-mixin-compiled/dashboards/loki-bloom-compactor.json b/production/loki-mixin-compiled/dashboards/loki-bloom-compactor.json index c667d0c01ecc..c365fab0a7e5 100644 --- a/production/loki-mixin-compiled/dashboards/loki-bloom-compactor.json +++ b/production/loki-mixin-compiled/dashboards/loki-bloom-compactor.json @@ -32,6 +32,7 @@ }, "id": 111, "panels": [ ], + "targets": [ ], "title": "Overview", "type": "row" }, @@ -52,7 +53,9 @@ "content": "## About the Bloom Compactor\nThe compactor iterates through chunks and creates blooms out of them.\nThe size of the resulting blooms depends on the bloom filter settings, the tokenizer settings, the number of ring tokens per compactor and the total number opf compactors.\n\nCompactors are horizontally scalable and uses a ring to:\n- Shard tenants\n- Shard series fingerprints within a tenant subring.\n\nThe blooms for the series are grouped together in blocks which are flushed to object store.", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "transparent": true, "type": "text" }, @@ -140,6 +143,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -290,6 +294,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -371,6 +376,7 @@ "sortOrder": "Descending", "wrapLogMessage": false }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -467,6 +473,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -576,6 +583,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -645,6 +653,7 @@ "showThresholdMarkers": false, "sizing": "auto" }, + "panels": [ ], "pluginVersion": "11.0.0-68102", "targets": [ { @@ -665,6 +674,7 @@ "type": "gauge" } ], + "targets": [ ], "title": "Progress per pod", "type": "row" }, @@ -696,7 +706,9 @@ "content": "", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "title": "We use tenant sharding so each compactor will process a subset of the tenants.", "transparent": true, "type": "text" @@ -777,6 +789,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -893,6 +906,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -1000,6 +1014,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -1149,6 +1164,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -1273,6 +1289,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -1448,6 +1465,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -1576,6 +1594,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -1678,6 +1697,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -1697,6 +1717,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Resource Usage", "type": "row" }, @@ -1791,6 +1812,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -1904,6 +1926,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -2004,6 +2027,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -2130,6 +2154,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -2150,6 +2175,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Bloom building", "type": "row" }, @@ -2238,6 +2264,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2332,6 +2359,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2426,6 +2454,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2445,6 +2474,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Blocks building", "type": "row" }, @@ -2533,6 +2563,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -2628,6 +2659,7 @@ "sort": "none" } }, + "panels": [ ], "pluginVersion": "11.1.0-69868", "targets": [ { @@ -2648,6 +2680,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Metas building", "type": "row" }, @@ -2679,7 +2712,9 @@ "content": "", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "title": "We use tenant sharding so each compactor will process a subset of the tenants.", "transparent": true, "type": "text" @@ -2759,6 +2794,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2878,6 +2914,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2914,7 +2951,9 @@ "content": "", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "title": "Number of tenant tables processed. ", "transparent": true, "type": "text" @@ -2994,6 +3033,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3113,6 +3153,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3149,7 +3190,9 @@ "content": "", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "title": "Series per compaction (includes series copied from other blocks)", "transparent": true, "type": "text" @@ -3230,6 +3273,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3349,6 +3393,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3385,7 +3430,9 @@ "content": "", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "title": "Number of bytes from chunks added to blocks during each compaction.", "transparent": true, "type": "text" @@ -3466,6 +3513,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3586,6 +3634,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3605,6 +3654,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Data processed", "type": "row" }, @@ -3640,7 +3690,9 @@ "content": "\nCompactors write blocks to the attached PVs before flushing them into the object store.\nIt also download chunks and index files.\n\nAfter compacting a given tenant, all the downloaded index files and chunks, as well as the already flushed blocks are deleted.", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-69747", + "targets": [ ], "title": "", "transparent": true, "type": "text" @@ -3726,6 +3778,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3850,6 +3903,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3946,6 +4000,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4068,6 +4123,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4164,6 +4220,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4286,6 +4343,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4304,6 +4362,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Disk Usage", "type": "row" }, @@ -4339,7 +4398,9 @@ "content": "Once all blocks and metas are built locally, the compactor flushes them to the object store.\n\nAfter each iteration, the compactor deletes the metas and blocks marked for deletion in the tombstones.", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-69747", + "targets": [ ], "title": "", "transparent": true, "type": "text" @@ -4366,7 +4427,9 @@ "content": "---\n#### GCS\n", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-69747", + "targets": [ ], "title": "", "transparent": true, "type": "text" @@ -4447,6 +4510,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4541,6 +4605,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4607,7 +4672,9 @@ "content": "---\n#### S3\n", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-69747", + "targets": [ ], "title": "", "transparent": true, "type": "text" @@ -4688,6 +4755,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4782,6 +4850,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4848,7 +4917,9 @@ "content": "---\n#### Azure\nBlob Storage", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-69747", + "targets": [ ], "title": "", "transparent": true, "type": "text" @@ -4929,6 +5000,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -5023,6 +5095,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -5068,6 +5141,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Object Store", "type": "row" } diff --git a/production/loki-mixin-compiled/dashboards/loki-bloom-gateway.json b/production/loki-mixin-compiled/dashboards/loki-bloom-gateway.json index 27a058ae800e..2d5e16a9d7e0 100644 --- a/production/loki-mixin-compiled/dashboards/loki-bloom-gateway.json +++ b/production/loki-mixin-compiled/dashboards/loki-bloom-gateway.json @@ -33,6 +33,7 @@ }, "id": 73, "panels": [ ], + "targets": [ ], "title": "Overview", "type": "row" }, @@ -131,6 +132,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -223,6 +225,7 @@ "showThresholdMarkers": true, "sizing": "auto" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", "targets": [ { @@ -395,6 +398,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -536,6 +540,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -577,6 +582,7 @@ "sortOrder": "Descending", "wrapLogMessage": true }, + "panels": [ ], "targets": [ { "datasource": { @@ -730,6 +736,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -847,6 +854,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -970,6 +978,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -1094,6 +1103,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -1234,6 +1244,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -1336,6 +1347,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -1437,6 +1449,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -1547,6 +1560,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -1647,6 +1661,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -1666,6 +1681,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Resource usage", "type": "row" }, @@ -1799,6 +1815,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -1937,6 +1954,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2032,6 +2050,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2152,6 +2171,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2171,6 +2191,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "QPS and Latency", "type": "row" }, @@ -2260,6 +2281,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2380,6 +2402,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2513,6 +2536,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2532,6 +2556,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Task Queue", "type": "row" }, @@ -2625,6 +2650,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2748,6 +2774,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2870,6 +2897,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -2901,6 +2929,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Processing", "type": "row" }, @@ -2936,7 +2965,9 @@ "content": "", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "title": "We cache bloom blocks in memory to prevent the gateway from hitting the object store too often", "transparent": true, "type": "text" @@ -3021,6 +3052,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3178,6 +3210,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3347,6 +3380,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3458,6 +3492,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3581,6 +3616,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3678,6 +3714,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Blocks Cache", "type": "row" }, @@ -3713,7 +3750,9 @@ "content": "", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "title": "The gateway download bloom meta files and blocks from the object store.", "transparent": true, "type": "text" @@ -3740,7 +3779,9 @@ "content": "---\n#### GCS\n", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "transparent": true, "type": "text" }, @@ -3820,6 +3861,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3913,6 +3955,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -3978,7 +4021,9 @@ "content": "---\n#### S3\n", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "transparent": true, "type": "text" }, @@ -4058,6 +4103,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4151,6 +4197,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4216,7 +4263,9 @@ "content": "---\n#### Azure\nBlob Storage\n\n", "mode": "markdown" }, + "panels": [ ], "pluginVersion": "11.1.0-70005", + "targets": [ ], "transparent": true, "type": "text" }, @@ -4296,6 +4345,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4389,6 +4439,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4433,6 +4484,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Object Store", "type": "row" }, @@ -4528,6 +4580,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4652,6 +4705,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4770,6 +4824,7 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, + "panels": [ ], "pluginVersion": "11.0.0-67814", "targets": [ { @@ -4882,6 +4937,7 @@ "sort": "none" } }, + "panels": [ ], "targets": [ { "datasource": { @@ -4912,6 +4968,7 @@ "type": "timeseries" } ], + "targets": [ ], "title": "Misc", "type": "row" } diff --git a/production/loki-mixin/dashboards/loki-bloom-compactor.libsonnet b/production/loki-mixin/dashboards/loki-bloom-compactor.libsonnet index cf93c7c992c5..236ebff8666d 100644 --- a/production/loki-mixin/dashboards/loki-bloom-compactor.libsonnet +++ b/production/loki-mixin/dashboards/loki-bloom-compactor.libsonnet @@ -9,11 +9,64 @@ local raw = (import './dashboard-bloom-compactor.json'); grafanaDashboards+: { 'loki-bloom-compactor.json': - raw + - $.dashboard('Loki / Bloom Compactor', uid='bloom-compactor') - .addCluster() - .addNamespace() - .addLog() - .addTag(), + raw + { + local replaceClusterMatchers(expr) = + // Replace the recording rules cluster label with the per-cluster label + std.strReplace( + // Replace the cluster label for equality matchers with the per-cluster label + std.strReplace( + // Replace the cluster label for regex matchers with the per-cluster label + std.strReplace( + expr, + 'cluster=~"$cluster"', + $._config.per_cluster_label + '=~"$cluster"' + ), + 'cluster="$cluster"', + $._config.per_cluster_label + '="$cluster"' + ), + 'cluster_job', + $._config.per_cluster_label + '_job' + ), + + panels: [ + p { + targets: if std.objectHas(p, 'targets') then [ + e { + expr: replaceClusterMatchers(e.expr), + } + for e in p.targets + ] else [], + panels: if std.objectHas(p, 'panels') then [ + sp { + targets: if std.objectHas(sp, 'targets') then [ + spe { + expr: replaceClusterMatchers(spe.expr), + } + for spe in sp.targets + ] else [], + panels: if std.objectHas(sp, 'panels') then [ + ssp { + targets: if std.objectHas(ssp, 'targets') then [ + sspe { + expr: replaceClusterMatchers(sspe.expr), + } + for sspe in ssp.targets + ] else [], + } + for ssp in sp.panels + ] else [], + } + for sp in p.panels + ] else [], + } + for p in super.panels + ], + } + + $.dashboard('Loki / Bloom Compactor', uid='bloom-compactor') + .addCluster() + .addNamespace() + .addLog() + .addTag(), }, } diff --git a/production/loki-mixin/dashboards/loki-bloom-gateway.libsonnet b/production/loki-mixin/dashboards/loki-bloom-gateway.libsonnet index e5ca9f2ff3fd..db9c90783911 100644 --- a/production/loki-mixin/dashboards/loki-bloom-gateway.libsonnet +++ b/production/loki-mixin/dashboards/loki-bloom-gateway.libsonnet @@ -9,11 +9,64 @@ local raw = (import './dashboard-bloom-gateway.json'); grafanaDashboards+: { 'loki-bloom-gateway.json': - raw + - $.dashboard('Loki / Bloom Gateway', uid='bloom-gateway') - .addCluster() - .addNamespace() - .addLog() - .addTag(), + raw + { + local replaceClusterMatchers(expr) = + // Replace the recording rules cluster label with the per-cluster label + std.strReplace( + // Replace the cluster label for equality matchers with the per-cluster label + std.strReplace( + // Replace the cluster label for regex matchers with the per-cluster label + std.strReplace( + expr, + 'cluster=~"$cluster"', + $._config.per_cluster_label + '=~"$cluster"' + ), + 'cluster="$cluster"', + $._config.per_cluster_label + '="$cluster"' + ), + 'cluster_job', + $._config.per_cluster_label + '_job' + ), + + panels: [ + p { + targets: if std.objectHas(p, 'targets') then [ + e { + expr: replaceClusterMatchers(e.expr), + } + for e in p.targets + ] else [], + panels: if std.objectHas(p, 'panels') then [ + sp { + targets: if std.objectHas(sp, 'targets') then [ + spe { + expr: replaceClusterMatchers(spe.expr), + } + for spe in sp.targets + ] else [], + panels: if std.objectHas(sp, 'panels') then [ + ssp { + targets: if std.objectHas(ssp, 'targets') then [ + sspe { + expr: replaceClusterMatchers(sspe.expr), + } + for sspe in ssp.targets + ] else [], + } + for ssp in sp.panels + ] else [], + } + for sp in p.panels + ] else [], + } + for p in super.panels + ], + } + + $.dashboard('Loki / Bloom Gateway', uid='bloom-gateway') + .addCluster() + .addNamespace() + .addLog() + .addTag(), }, } From 1111595179c77f9303ebdfd362f14b1ac50044cb Mon Sep 17 00:00:00 2001 From: Vladyslav Diachenko <82767850+vlad-diachenko@users.noreply.github.com> Date: Thu, 23 May 2024 13:18:16 +0300 Subject: [PATCH 17/41] feat: new stream count limiter (#13006) Signed-off-by: Vladyslav Diachenko Co-authored-by: JordanRushing --- docs/sources/shared/configuration.md | 5 ++ pkg/ingester/instance.go | 75 +++++++++++++++----------- pkg/ingester/limiter.go | 81 ++++++++++++++++++++-------- pkg/ingester/limiter_test.go | 47 +++++++++++++++- pkg/ingester/owned_streams.go | 44 +++++++++++++++ pkg/ingester/owned_streams_test.go | 36 +++++++++++++ pkg/validation/limits.go | 6 +++ 7 files changed, 239 insertions(+), 55 deletions(-) create mode 100644 pkg/ingester/owned_streams.go create mode 100644 pkg/ingester/owned_streams_test.go diff --git a/docs/sources/shared/configuration.md b/docs/sources/shared/configuration.md index acf11102be51..c0e4bdeeca4d 100644 --- a/docs/sources/shared/configuration.md +++ b/docs/sources/shared/configuration.md @@ -2978,6 +2978,11 @@ The `limits_config` block configures global and per-tenant limits in Loki. The v # CLI flag: -validation.discover-log-levels [discover_log_levels: | default = true] +# When true an ingester takes into account only the streams that it owns +# according to the ring while applying the stream limit. +# CLI flag: -ingester.use-owned-stream-count +[use_owned_stream_count: | default = false] + # Maximum number of active streams per user, per ingester. 0 to disable. # CLI flag: -ingester.max-streams-per-user [max_streams_per_user: | default = 0] diff --git a/pkg/ingester/instance.go b/pkg/ingester/instance.go index a4436b9d4191..7f1ec78601ff 100644 --- a/pkg/ingester/instance.go +++ b/pkg/ingester/instance.go @@ -104,7 +104,10 @@ type instance struct { tailers map[uint32]*tailer tailerMtx sync.RWMutex - limiter *Limiter + limiter *Limiter + streamCountLimiter *streamCountLimiter + ownedStreamsSvc *ownedStreamService + configs *runtime.TenantConfigs wal WAL @@ -147,11 +150,12 @@ func newInstance( if err != nil { return nil, err } - + streams := newStreamsMap() + ownedStreamsSvc := newOwnedStreamService(instanceID, limiter) c := config.SchemaConfig{Configs: periodConfigs} i := &instance{ cfg: cfg, - streams: newStreamsMap(), + streams: streams, buf: make([]byte, 0, 1024), index: invertedIndex, instanceID: instanceID, @@ -159,9 +163,11 @@ func newInstance( streamsCreatedTotal: streamsCreatedTotal.WithLabelValues(instanceID), streamsRemovedTotal: streamsRemovedTotal.WithLabelValues(instanceID), - tailers: map[uint32]*tailer{}, - limiter: limiter, - configs: configs, + tailers: map[uint32]*tailer{}, + limiter: limiter, + streamCountLimiter: newStreamCountLimiter(instanceID, streams.Len, limiter, ownedStreamsSvc), + ownedStreamsSvc: ownedStreamsSvc, + configs: configs, wal: wal, metrics: metrics, @@ -286,29 +292,11 @@ func (i *instance) createStream(ctx context.Context, pushReqStream logproto.Stre } if record != nil { - err = i.limiter.AssertMaxStreamsPerUser(i.instanceID, i.streams.Len()) + err = i.streamCountLimiter.AssertNewStreamAllowed(i.instanceID) } if err != nil { - if i.configs.LogStreamCreation(i.instanceID) { - level.Debug(util_log.Logger).Log( - "msg", "failed to create stream, exceeded limit", - "org_id", i.instanceID, - "err", err, - "stream", pushReqStream.Labels, - ) - } - - validation.DiscardedSamples.WithLabelValues(validation.StreamLimit, i.instanceID).Add(float64(len(pushReqStream.Entries))) - bytes := 0 - for _, e := range pushReqStream.Entries { - bytes += len(e.Line) - } - validation.DiscardedBytes.WithLabelValues(validation.StreamLimit, i.instanceID).Add(float64(bytes)) - if i.customStreamsTracker != nil { - i.customStreamsTracker.DiscardedBytesAdd(ctx, i.instanceID, validation.StreamLimit, labels, float64(bytes)) - } - return nil, httpgrpc.Errorf(http.StatusTooManyRequests, validation.StreamLimitErrorMsg, labels, i.instanceID) + return i.onStreamCreationError(ctx, pushReqStream, err, labels) } fp := i.getHashForLabels(labels) @@ -333,21 +321,47 @@ func (i *instance) createStream(ctx context.Context, pushReqStream logproto.Stre i.metrics.recoveredStreamsTotal.Inc() } + i.onStreamCreated(s) + + return s, nil +} + +func (i *instance) onStreamCreationError(ctx context.Context, pushReqStream logproto.Stream, err error, labels labels.Labels) (*stream, error) { + if i.configs.LogStreamCreation(i.instanceID) { + level.Debug(util_log.Logger).Log( + "msg", "failed to create stream, exceeded limit", + "org_id", i.instanceID, + "err", err, + "stream", pushReqStream.Labels, + ) + } + + validation.DiscardedSamples.WithLabelValues(validation.StreamLimit, i.instanceID).Add(float64(len(pushReqStream.Entries))) + bytes := 0 + for _, e := range pushReqStream.Entries { + bytes += len(e.Line) + } + validation.DiscardedBytes.WithLabelValues(validation.StreamLimit, i.instanceID).Add(float64(bytes)) + if i.customStreamsTracker != nil { + i.customStreamsTracker.DiscardedBytesAdd(ctx, i.instanceID, validation.StreamLimit, labels, float64(bytes)) + } + return nil, httpgrpc.Errorf(http.StatusTooManyRequests, validation.StreamLimitErrorMsg, labels, i.instanceID) +} + +func (i *instance) onStreamCreated(s *stream) { memoryStreams.WithLabelValues(i.instanceID).Inc() memoryStreamsLabelsBytes.Add(float64(len(s.labels.String()))) i.streamsCreatedTotal.Inc() i.addTailersToNewStream(s) streamsCountStats.Add(1) - + i.ownedStreamsSvc.incOwnedStreamCount() if i.configs.LogStreamCreation(i.instanceID) { level.Debug(util_log.Logger).Log( "msg", "successfully created stream", "org_id", i.instanceID, - "stream", pushReqStream.Labels, + "stream", s.labels.String(), ) } - - return s, nil } func (i *instance) createStreamByFP(ls labels.Labels, fp model.Fingerprint) (*stream, error) { @@ -407,6 +421,7 @@ func (i *instance) removeStream(s *stream) { memoryStreams.WithLabelValues(i.instanceID).Dec() memoryStreamsLabelsBytes.Sub(float64(len(s.labels.String()))) streamsCountStats.Add(-1) + i.ownedStreamsSvc.decOwnedStreamCount() } } diff --git a/pkg/ingester/limiter.go b/pkg/ingester/limiter.go index 94c77a30be7e..daa1fe7aec8d 100644 --- a/pkg/ingester/limiter.go +++ b/pkg/ingester/limiter.go @@ -24,6 +24,7 @@ type RingCount interface { type Limits interface { UnorderedWrites(userID string) bool + UseOwnedStreamCount(userID string) bool MaxLocalStreamsPerUser(userID string) int MaxGlobalStreamsPerUser(userID string) int PerStreamRateLimit(userID string) validation.RateLimit @@ -76,46 +77,39 @@ func (l *Limiter) UnorderedWrites(userID string) bool { return l.limits.UnorderedWrites(userID) } -// AssertMaxStreamsPerUser ensures limit has not been reached compared to the current -// number of streams in input and returns an error if so. -func (l *Limiter) AssertMaxStreamsPerUser(userID string, streams int) error { - // Until the limiter actually starts, all accesses are successful. - // This is used to disable limits while recovering from the WAL. - l.mtx.RLock() - defer l.mtx.RUnlock() - if l.disabled { - return nil - } - +func (l *Limiter) GetStreamCountLimit(tenantID string) (calculatedLimit, localLimit, globalLimit, adjustedGlobalLimit int) { // Start by setting the local limit either from override or default - localLimit := l.limits.MaxLocalStreamsPerUser(userID) + localLimit = l.limits.MaxLocalStreamsPerUser(tenantID) // We can assume that streams are evenly distributed across ingesters // so we do convert the global limit into a local limit - globalLimit := l.limits.MaxGlobalStreamsPerUser(userID) - adjustedGlobalLimit := l.convertGlobalToLocalLimit(globalLimit) + globalLimit = l.limits.MaxGlobalStreamsPerUser(tenantID) + adjustedGlobalLimit = l.convertGlobalToLocalLimit(globalLimit) // Set the calculated limit to the lesser of the local limit or the new calculated global limit - calculatedLimit := l.minNonZero(localLimit, adjustedGlobalLimit) + calculatedLimit = l.minNonZero(localLimit, adjustedGlobalLimit) // If both the local and global limits are disabled, we just // use the largest int value if calculatedLimit == 0 { calculatedLimit = math.MaxInt32 } + return +} - if streams < calculatedLimit { - return nil +func (l *Limiter) minNonZero(first, second int) int { + if first == 0 || (second != 0 && first > second) { + return second } - return fmt.Errorf(errMaxStreamsPerUserLimitExceeded, userID, streams, calculatedLimit, localLimit, globalLimit, adjustedGlobalLimit) + return first } func (l *Limiter) convertGlobalToLocalLimit(globalLimit int) int { if globalLimit == 0 { return 0 } - + // todo: change to healthyInstancesInZoneCount() once // Given we don't need a super accurate count (ie. when the ingesters // topology changes) and we prefer to always be in favor of the tenant, // we can use a per-ingester limit equal to: @@ -131,12 +125,53 @@ func (l *Limiter) convertGlobalToLocalLimit(globalLimit int) int { return 0 } -func (l *Limiter) minNonZero(first, second int) int { - if first == 0 || (second != 0 && first > second) { - return second +type supplier[T any] func() T + +type streamCountLimiter struct { + tenantID string + limiter *Limiter + defaultStreamCountSupplier supplier[int] + ownedStreamSvc *ownedStreamService +} + +var noopFixedLimitSupplier = func() int { + return 0 +} + +func newStreamCountLimiter(tenantID string, defaultStreamCountSupplier supplier[int], limiter *Limiter, service *ownedStreamService) *streamCountLimiter { + return &streamCountLimiter{ + tenantID: tenantID, + limiter: limiter, + defaultStreamCountSupplier: defaultStreamCountSupplier, + ownedStreamSvc: service, } +} - return first +func (l *streamCountLimiter) AssertNewStreamAllowed(tenantID string) error { + streamCountSupplier, fixedLimitSupplier := l.getSuppliers(tenantID) + calculatedLimit, localLimit, globalLimit, adjustedGlobalLimit := l.getCurrentLimit(tenantID, fixedLimitSupplier) + actualStreamsCount := streamCountSupplier() + if actualStreamsCount < calculatedLimit { + return nil + } + + return fmt.Errorf(errMaxStreamsPerUserLimitExceeded, tenantID, actualStreamsCount, calculatedLimit, localLimit, globalLimit, adjustedGlobalLimit) +} + +func (l *streamCountLimiter) getCurrentLimit(tenantID string, fixedLimitSupplier supplier[int]) (calculatedLimit, localLimit, globalLimit, adjustedGlobalLimit int) { + calculatedLimit, localLimit, globalLimit, adjustedGlobalLimit = l.limiter.GetStreamCountLimit(tenantID) + fixedLimit := fixedLimitSupplier() + if fixedLimit > calculatedLimit { + calculatedLimit = fixedLimit + } + return +} + +func (l *streamCountLimiter) getSuppliers(tenant string) (streamCountSupplier, fixedLimitSupplier supplier[int]) { + if l.limiter.limits.UseOwnedStreamCount(tenant) { + return l.ownedStreamSvc.getOwnedStreamCount, l.ownedStreamSvc.getFixedLimit + } + return l.defaultStreamCountSupplier, noopFixedLimitSupplier } type RateLimiterStrategy interface { diff --git a/pkg/ingester/limiter_test.go b/pkg/ingester/limiter_test.go index 6186e910663e..9d4d3b3037c6 100644 --- a/pkg/ingester/limiter_test.go +++ b/pkg/ingester/limiter_test.go @@ -8,12 +8,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/atomic" "golang.org/x/time/rate" "github.com/grafana/loki/v3/pkg/validation" ) -func TestLimiter_AssertMaxStreamsPerUser(t *testing.T) { +func TestStreamCountLimiter_AssertNewStreamAllowed(t *testing.T) { tests := map[string]struct { maxLocalStreamsPerUser int maxGlobalStreamsPerUser int @@ -21,6 +22,9 @@ func TestLimiter_AssertMaxStreamsPerUser(t *testing.T) { ringIngesterCount int streams int expected error + useOwnedStreamService bool + fixedLimit int32 + ownedStreamCount int64 }{ "both local and global limit are disabled": { maxLocalStreamsPerUser: 0, @@ -94,6 +98,36 @@ func TestLimiter_AssertMaxStreamsPerUser(t *testing.T) { streams: 3000, expected: fmt.Errorf(errMaxStreamsPerUserLimitExceeded, "test", 3000, 300, 500, 1000, 300), }, + "actual limit must be used if it's greater than fixed limit": { + maxLocalStreamsPerUser: 500, + maxGlobalStreamsPerUser: 1000, + ringReplicationFactor: 3, + ringIngesterCount: 10, + useOwnedStreamService: true, + fixedLimit: 20, + ownedStreamCount: 3000, + expected: fmt.Errorf(errMaxStreamsPerUserLimitExceeded, "test", 3000, 300, 500, 1000, 300), + }, + "fixed limit must be used if it's greater than actual limit": { + maxLocalStreamsPerUser: 500, + maxGlobalStreamsPerUser: 1000, + ringReplicationFactor: 3, + ringIngesterCount: 10, + useOwnedStreamService: true, + fixedLimit: 2000, + ownedStreamCount: 2001, + expected: fmt.Errorf(errMaxStreamsPerUserLimitExceeded, "test", 2001, 2000, 500, 1000, 300), + }, + "fixed limit must not be used if both limits are disabled": { + maxLocalStreamsPerUser: 0, + maxGlobalStreamsPerUser: 0, + ringReplicationFactor: 3, + ringIngesterCount: 10, + useOwnedStreamService: true, + fixedLimit: 2000, + ownedStreamCount: 2001, + expected: nil, + }, } for testName, testData := range tests { @@ -107,11 +141,20 @@ func TestLimiter_AssertMaxStreamsPerUser(t *testing.T) { limits, err := validation.NewOverrides(validation.Limits{ MaxLocalStreamsPerUser: testData.maxLocalStreamsPerUser, MaxGlobalStreamsPerUser: testData.maxGlobalStreamsPerUser, + UseOwnedStreamCount: testData.useOwnedStreamService, }, nil) require.NoError(t, err) + ownedStreamSvc := &ownedStreamService{ + fixedLimit: atomic.NewInt32(testData.fixedLimit), + ownedStreamCount: atomic.NewInt64(testData.ownedStreamCount), + } limiter := NewLimiter(limits, NilMetrics, ring, testData.ringReplicationFactor) - actual := limiter.AssertMaxStreamsPerUser("test", testData.streams) + defaultCountSupplier := func() int { + return testData.streams + } + streamCountLimiter := newStreamCountLimiter("test", defaultCountSupplier, limiter, ownedStreamSvc) + actual := streamCountLimiter.AssertNewStreamAllowed("test") assert.Equal(t, testData.expected, actual) }) diff --git a/pkg/ingester/owned_streams.go b/pkg/ingester/owned_streams.go new file mode 100644 index 000000000000..01cb8235f9b1 --- /dev/null +++ b/pkg/ingester/owned_streams.go @@ -0,0 +1,44 @@ +package ingester + +import "go.uber.org/atomic" + +type ownedStreamService struct { + tenantID string + limiter *Limiter + fixedLimit *atomic.Int32 + + //todo: implement job to recalculate it + ownedStreamCount *atomic.Int64 +} + +func newOwnedStreamService(tenantID string, limiter *Limiter) *ownedStreamService { + svc := &ownedStreamService{ + tenantID: tenantID, + limiter: limiter, + ownedStreamCount: atomic.NewInt64(0), + fixedLimit: atomic.NewInt32(0), + } + svc.updateFixedLimit() + return svc +} + +func (s *ownedStreamService) getOwnedStreamCount() int { + return int(s.ownedStreamCount.Load()) +} + +func (s *ownedStreamService) updateFixedLimit() { + limit, _, _, _ := s.limiter.GetStreamCountLimit(s.tenantID) + s.fixedLimit.Store(int32(limit)) +} + +func (s *ownedStreamService) getFixedLimit() int { + return int(s.fixedLimit.Load()) +} + +func (s *ownedStreamService) incOwnedStreamCount() { + s.ownedStreamCount.Inc() +} + +func (s *ownedStreamService) decOwnedStreamCount() { + s.ownedStreamCount.Dec() +} diff --git a/pkg/ingester/owned_streams_test.go b/pkg/ingester/owned_streams_test.go new file mode 100644 index 000000000000..c7ddd9d87f29 --- /dev/null +++ b/pkg/ingester/owned_streams_test.go @@ -0,0 +1,36 @@ +package ingester + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/grafana/loki/v3/pkg/validation" +) + +func Test_OwnedStreamService(t *testing.T) { + limits, err := validation.NewOverrides(validation.Limits{ + MaxGlobalStreamsPerUser: 100, + }, nil) + require.NoError(t, err) + // Mock the ring + ring := &ringCountMock{count: 30} + limiter := NewLimiter(limits, NilMetrics, ring, 3) + + service := newOwnedStreamService("test", limiter) + require.Equal(t, 0, service.getOwnedStreamCount()) + require.Equal(t, 10, service.getFixedLimit(), "fixed limit must be initialised during the instantiation") + + limits.DefaultLimits().MaxGlobalStreamsPerUser = 1000 + require.Equal(t, 10, service.getFixedLimit(), "fixed list must not be changed until update is triggered") + + service.updateFixedLimit() + require.Equal(t, 100, service.getFixedLimit()) + + service.incOwnedStreamCount() + service.incOwnedStreamCount() + require.Equal(t, 2, service.getOwnedStreamCount()) + + service.decOwnedStreamCount() + require.Equal(t, 1, service.getOwnedStreamCount()) +} diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index 148205b306c1..7fde990e4365 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -85,6 +85,7 @@ type Limits struct { DiscoverLogLevels bool `yaml:"discover_log_levels" json:"discover_log_levels"` // Ingester enforced limits. + UseOwnedStreamCount bool `yaml:"use_owned_stream_count" json:"use_owned_stream_count"` MaxLocalStreamsPerUser int `yaml:"max_streams_per_user" json:"max_streams_per_user"` MaxGlobalStreamsPerUser int `yaml:"max_global_streams_per_user" json:"max_global_streams_per_user"` UnorderedWrites bool `yaml:"unordered_writes" json:"unordered_writes"` @@ -270,6 +271,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.Var(&l.CreationGracePeriod, "validation.create-grace-period", "Duration which table will be created/deleted before/after it's needed; we won't accept sample from before this time.") f.IntVar(&l.MaxEntriesLimitPerQuery, "validation.max-entries-limit", 5000, "Maximum number of log entries that will be returned for a query.") + f.BoolVar(&l.UseOwnedStreamCount, "ingester.use-owned-stream-count", false, "When true an ingester takes into account only the streams that it owns according to the ring while applying the stream limit.") f.IntVar(&l.MaxLocalStreamsPerUser, "ingester.max-streams-per-user", 0, "Maximum number of active streams per user, per ingester. 0 to disable.") f.IntVar(&l.MaxGlobalStreamsPerUser, "ingester.max-global-streams-per-user", 5000, "Maximum number of active streams per user, across the cluster. 0 to disable. When the global limit is enabled, each ingester is configured with a dynamic local limit based on the replication factor and the current number of healthy ingesters, and is kept updated whenever the number of ingesters change.") @@ -588,6 +590,10 @@ func (o *Overrides) CreationGracePeriod(userID string) time.Duration { return time.Duration(o.getOverridesForUser(userID).CreationGracePeriod) } +func (o *Overrides) UseOwnedStreamCount(userID string) bool { + return o.getOverridesForUser(userID).UseOwnedStreamCount +} + // MaxLocalStreamsPerUser returns the maximum number of streams a user is allowed to store // in a single ingester. func (o *Overrides) MaxLocalStreamsPerUser(userID string) int { From 97212eadf15c2b5ee2cd59b7c1df71f6177cfe7e Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Thu, 23 May 2024 12:10:48 -0400 Subject: [PATCH 18/41] feat: Added Interactive Sandbox to Quickstart tutorial (#12701) --- docs/sources/get-started/quick-start.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/sources/get-started/quick-start.md b/docs/sources/get-started/quick-start.md index b08f07a8e797..f459e564092e 100644 --- a/docs/sources/get-started/quick-start.md +++ b/docs/sources/get-started/quick-start.md @@ -22,6 +22,18 @@ The Docker Compose configuration instantiates the following components, each in {{< figure max-width="75%" src="/media/docs/loki/get-started-flog-v3.png" caption="Getting started sample application" alt="Getting started sample application">}} +## Interactive Learning Environment + +{{< admonition type="note" >}} +The Interactive Learning Environment is currently in trial. Please provide feedback, report bugs, and raise issues in the [Grafana Killercoda Repository](https://github.com/grafana/killercoda). +{{< /admonition >}} + +Try out this demo within our interactive learning environment: [Loki Quickstart Sandbox](https://killercoda.com/grafana-labs/course/loki/loki-quickstart) + +- A free Killercoda account is required to verify you are not a bot. +- Tutorial instructions are located on the left-hand side of the screen. Click to move on to the next section. +- All commands run inside the interactive terminal. Grafana can also be accessed via the URL links provided within the sandbox. + ## Installing Loki and collecting sample logs Prerequisites From efdae3df14c47d627eb99e91466e0451db6e16f6 Mon Sep 17 00:00:00 2001 From: hayden Date: Thu, 23 May 2024 16:25:50 -0400 Subject: [PATCH 19/41] feat(helm): Support for PVC Annotations for Non-Distributed Modes (#12023) Signed-off-by: hfuss Co-authored-by: J Stickler Co-authored-by: Trevor Whitney --- docs/Makefile | 1 + docs/sources/setup/install/helm/reference.md | 39 +++++++++++++++++++ production/helm/loki/CHANGELOG.md | 4 ++ production/helm/loki/Chart.yaml | 2 +- production/helm/loki/README.md | 2 +- .../backend/statefulset-backend.yaml | 4 ++ .../loki/templates/read/statefulset-read.yaml | 4 ++ .../templates/single-binary/statefulset.yaml | 4 ++ .../templates/write/statefulset-write.yaml | 4 ++ production/helm/loki/values.yaml | 9 +++++ 10 files changed, 71 insertions(+), 2 deletions(-) diff --git a/docs/Makefile b/docs/Makefile index 63fc849789c1..4bed302d7179 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -10,6 +10,7 @@ include docs.mk PODMAN := $(shell if command -v podman >/dev/null 2>&1; then echo podman; else echo docker; fi) BUILD_IN_CONTAINER ?= true +.PHONY: sources/setup/install/helm/reference.md sources/setup/install/helm/reference.md: ../production/helm/loki/reference.md.gotmpl ../production/helm/loki/values.yaml ifeq ($(BUILD_IN_CONTAINER),true) $(PODMAN) run --rm --volume "$(realpath ..):/helm-docs" -u "$$(id -u)" "docker.io/jnorwood/helm-docs:v1.11.0" \ diff --git a/docs/sources/setup/install/helm/reference.md b/docs/sources/setup/install/helm/reference.md index 53101a483214..76b4936f20bf 100644 --- a/docs/sources/setup/install/helm/reference.md +++ b/docs/sources/setup/install/helm/reference.md @@ -315,6 +315,7 @@ This is the generated reference for the Loki Helm Chart values. "initContainers": [], "nodeSelector": {}, "persistence": { + "annotations": {}, "dataVolumeParameters": { "emptyDir": {} }, @@ -512,6 +513,15 @@ null 
 {}
+ + + + backend.persistence.annotations + object + Annotations for volume claim + 
+{}
+
@@ -6226,6 +6236,7 @@ false "drivesPerNode": 2, "enabled": false, "persistence": { + "annotations": {}, "size": "5Gi" }, "replicas": 1, @@ -8442,6 +8453,7 @@ false "lifecycle": {}, "nodeSelector": {}, "persistence": { + "annotations": {}, "enableStatefulSetAutoDeletePVC": true, "selector": null, "size": "10Gi", @@ -8653,6 +8665,15 @@ false 
 {}
+ + + + read.persistence.annotations + object + Annotations for volume claim + 
+{}
+
@@ -9893,6 +9914,15 @@ null 
 {}
+ + + + singleBinary.persistence.annotations + object + Annotations for volume claim + 
+{}
+
@@ -10677,6 +10707,15 @@ null 
 {}
+ + + + write.persistence.annotations + object + Annotations for volume claim + 
+{}
+
diff --git a/production/helm/loki/CHANGELOG.md b/production/helm/loki/CHANGELOG.md index 1606c89914f8..77b801e60363 100644 --- a/production/helm/loki/CHANGELOG.md +++ b/production/helm/loki/CHANGELOG.md @@ -13,6 +13,10 @@ Entries should include a reference to the pull request that introduced the chang [//]: # ( : do not remove this line. This locator is used by the CI pipeline to automatically create a changelog entry for each new Loki release. Add other chart versions and respective changelog entries bellow this line.) +## 6.6.0 + +- [ENHANCEMENT] Allow setting PVC annotations for all volume claim templates in simple scalable and single binary mode + ## 6.5.2 - [BUGFIX] Fixed Ingress routing for all deployment modes. diff --git a/production/helm/loki/Chart.yaml b/production/helm/loki/Chart.yaml index 989a54d146a1..637e66d70887 100644 --- a/production/helm/loki/Chart.yaml +++ b/production/helm/loki/Chart.yaml @@ -3,7 +3,7 @@ name: loki description: Helm chart for Grafana Loki and Grafana Enterprise Logs supporting both simple, scalable and distributed modes. type: application appVersion: 3.0.0 -version: 6.5.2 +version: 6.6.0 home: https://grafana.github.io/helm-charts sources: - https://github.com/grafana/loki diff --git a/production/helm/loki/README.md b/production/helm/loki/README.md index 55a7256c72f7..5fa6bd548bad 100644 --- a/production/helm/loki/README.md +++ b/production/helm/loki/README.md @@ -1,6 +1,6 @@ # loki -![Version: 6.5.2](https://img.shields.io/badge/Version-6.5.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.0.0](https://img.shields.io/badge/AppVersion-3.0.0-informational?style=flat-square) +![Version: 6.6.0](https://img.shields.io/badge/Version-6.6.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.0.0](https://img.shields.io/badge/AppVersion-3.0.0-informational?style=flat-square) Helm chart for Grafana Loki and Grafana Enterprise Logs supporting both simple, scalable and distributed modes. diff --git a/production/helm/loki/templates/backend/statefulset-backend.yaml b/production/helm/loki/templates/backend/statefulset-backend.yaml index f96f0a4d2121..534190d4a453 100644 --- a/production/helm/loki/templates/backend/statefulset-backend.yaml +++ b/production/helm/loki/templates/backend/statefulset-backend.yaml @@ -266,6 +266,10 @@ spec: kind: PersistentVolumeClaim metadata: name: data + {{- with .Values.backend.persistence.annotations }} + annotations: + {{- toYaml . | nindent 10 }} + {{- end }} spec: accessModes: - ReadWriteOnce diff --git a/production/helm/loki/templates/read/statefulset-read.yaml b/production/helm/loki/templates/read/statefulset-read.yaml index 0a31de4996df..7696d90e65bd 100644 --- a/production/helm/loki/templates/read/statefulset-read.yaml +++ b/production/helm/loki/templates/read/statefulset-read.yaml @@ -180,6 +180,10 @@ spec: kind: PersistentVolumeClaim metadata: name: data + {{- with .Values.read.persistence.annotations }} + annotations: + {{- toYaml . | nindent 10 }} + {{- end }} spec: accessModes: - ReadWriteOnce diff --git a/production/helm/loki/templates/single-binary/statefulset.yaml b/production/helm/loki/templates/single-binary/statefulset.yaml index 51c0062fc94f..7bd2b9813f60 100644 --- a/production/helm/loki/templates/single-binary/statefulset.yaml +++ b/production/helm/loki/templates/single-binary/statefulset.yaml @@ -175,6 +175,10 @@ spec: kind: PersistentVolumeClaim metadata: name: storage + {{- with .Values.singleBinary.persistence.annotations }} + annotations: + {{- toYaml . | nindent 10 }} + {{- end }} spec: accessModes: - ReadWriteOnce diff --git a/production/helm/loki/templates/write/statefulset-write.yaml b/production/helm/loki/templates/write/statefulset-write.yaml index 54c936958b55..75605c27c26c 100644 --- a/production/helm/loki/templates/write/statefulset-write.yaml +++ b/production/helm/loki/templates/write/statefulset-write.yaml @@ -193,6 +193,10 @@ spec: kind: PersistentVolumeClaim metadata: name: data + {{- with .Values.write.persistence.annotations }} + annotations: + {{- toYaml . | nindent 10 }} + {{- end }} spec: accessModes: - ReadWriteOnce diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml index 3edfc24ba34f..4c70bf16fe47 100644 --- a/production/helm/loki/values.yaml +++ b/production/helm/loki/values.yaml @@ -1294,6 +1294,8 @@ singleBinary: storageClass: null # -- Selector for persistent disk selector: null + # -- Annotations for volume claim + annotations: {} ###################################################################################################################### # # Simple Scalable Deployment (SSD) Mode @@ -1421,6 +1423,8 @@ write: storageClass: null # -- Selector for persistent disk selector: null + # -- Annotations for volume claim + annotations: {} # -- Configuration for the read pod(s) read: # -- Number of replicas for the read @@ -1528,6 +1532,8 @@ read: storageClass: null # -- Selector for persistent disk selector: null + # -- Annotations for volume claim + annotations: {} # -- Configuration for the backend pod(s) backend: # -- Number of replicas for the backend @@ -1636,6 +1642,8 @@ backend: storageClass: null # -- Selector for persistent disk selector: null + # -- Annotations for volume claim + annotations: {} ###################################################################################################################### # # Microservices Mode @@ -3091,6 +3099,7 @@ minio: purge: false persistence: size: 5Gi + annotations: {} resources: requests: cpu: 100m From ca030a5c4335b0258e83aebd8779ea4d348003f3 Mon Sep 17 00:00:00 2001 From: Trevor Whitney Date: Fri, 24 May 2024 01:55:59 -0600 Subject: [PATCH 20/41] fix: change log level since this is a known case (#13029) --- pkg/querier/queryrange/limits.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/querier/queryrange/limits.go b/pkg/querier/queryrange/limits.go index f90a9aa3c4e5..68f71680dd67 100644 --- a/pkg/querier/queryrange/limits.go +++ b/pkg/querier/queryrange/limits.go @@ -345,7 +345,7 @@ func (q *querySizeLimiter) Do(ctx context.Context, r queryrangebase.Request) (qu // Only support TSDB schemaCfg, err := q.getSchemaCfg(r) if err != nil { - level.Error(log).Log("msg", "failed to get schema config, not applying querySizeLimit", "err", err) + level.Warn(log).Log("msg", "failed to get schema config, not applying querySizeLimit", "err", err) return q.next.Do(ctx, r) } if schemaCfg.IndexType != types.TSDBType { From 8101e21f9973b8261de0ee3eb34fa4d7b88ddaac Mon Sep 17 00:00:00 2001 From: Lex Rivera Date: Fri, 24 May 2024 16:08:56 +0300 Subject: [PATCH 21/41] fix(helm): fix queryScheduler servicemonitor (#12753) Co-authored-by: Michel Hollands <42814411+MichelHollands@users.noreply.github.com> --- production/helm/loki/CHANGELOG.md | 4 ++++ production/helm/loki/Chart.yaml | 2 +- production/helm/loki/README.md | 2 +- .../templates/query-scheduler/service-query-scheduler.yaml | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/production/helm/loki/CHANGELOG.md b/production/helm/loki/CHANGELOG.md index 77b801e60363..a86a7d028182 100644 --- a/production/helm/loki/CHANGELOG.md +++ b/production/helm/loki/CHANGELOG.md @@ -13,6 +13,10 @@ Entries should include a reference to the pull request that introduced the chang [//]: # ( : do not remove this line. This locator is used by the CI pipeline to automatically create a changelog entry for each new Loki release. Add other chart versions and respective changelog entries bellow this line.) +## 6.6.1 + +- [BUGFIX] Fix query scheduler http-metrics targetPort + ## 6.6.0 - [ENHANCEMENT] Allow setting PVC annotations for all volume claim templates in simple scalable and single binary mode diff --git a/production/helm/loki/Chart.yaml b/production/helm/loki/Chart.yaml index 637e66d70887..d2ec2d3d9e59 100644 --- a/production/helm/loki/Chart.yaml +++ b/production/helm/loki/Chart.yaml @@ -3,7 +3,7 @@ name: loki description: Helm chart for Grafana Loki and Grafana Enterprise Logs supporting both simple, scalable and distributed modes. type: application appVersion: 3.0.0 -version: 6.6.0 +version: 6.6.1 home: https://grafana.github.io/helm-charts sources: - https://github.com/grafana/loki diff --git a/production/helm/loki/README.md b/production/helm/loki/README.md index 5fa6bd548bad..03ddb05bf660 100644 --- a/production/helm/loki/README.md +++ b/production/helm/loki/README.md @@ -1,6 +1,6 @@ # loki -![Version: 6.6.0](https://img.shields.io/badge/Version-6.6.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.0.0](https://img.shields.io/badge/AppVersion-3.0.0-informational?style=flat-square) +![Version: 6.6.1](https://img.shields.io/badge/Version-6.6.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.0.0](https://img.shields.io/badge/AppVersion-3.0.0-informational?style=flat-square) Helm chart for Grafana Loki and Grafana Enterprise Logs supporting both simple, scalable and distributed modes. diff --git a/production/helm/loki/templates/query-scheduler/service-query-scheduler.yaml b/production/helm/loki/templates/query-scheduler/service-query-scheduler.yaml index 89883155a27e..2b3f1b230060 100644 --- a/production/helm/loki/templates/query-scheduler/service-query-scheduler.yaml +++ b/production/helm/loki/templates/query-scheduler/service-query-scheduler.yaml @@ -21,7 +21,7 @@ spec: ports: - name: http-metrics port: 3100 - targetPort: http + targetPort: http-metrics protocol: TCP - name: grpclb port: 9095 From 4901a5c452fa6822a645f56e20e704db9366182a Mon Sep 17 00:00:00 2001 From: Vladyslav Diachenko <82767850+vlad-diachenko@users.noreply.github.com> Date: Fri, 24 May 2024 23:53:29 +0300 Subject: [PATCH 22/41] fix: not owned stream count (#13030) Signed-off-by: Vladyslav Diachenko --- pkg/ingester/limiter_test.go | 4 ++-- pkg/ingester/owned_streams.go | 33 ++++++++++++++++++++++-------- pkg/ingester/owned_streams_test.go | 32 ++++++++++++++++++++++++++++- 3 files changed, 57 insertions(+), 12 deletions(-) diff --git a/pkg/ingester/limiter_test.go b/pkg/ingester/limiter_test.go index 9d4d3b3037c6..b00bede10417 100644 --- a/pkg/ingester/limiter_test.go +++ b/pkg/ingester/limiter_test.go @@ -24,7 +24,7 @@ func TestStreamCountLimiter_AssertNewStreamAllowed(t *testing.T) { expected error useOwnedStreamService bool fixedLimit int32 - ownedStreamCount int64 + ownedStreamCount int }{ "both local and global limit are disabled": { maxLocalStreamsPerUser: 0, @@ -147,7 +147,7 @@ func TestStreamCountLimiter_AssertNewStreamAllowed(t *testing.T) { ownedStreamSvc := &ownedStreamService{ fixedLimit: atomic.NewInt32(testData.fixedLimit), - ownedStreamCount: atomic.NewInt64(testData.ownedStreamCount), + ownedStreamCount: testData.ownedStreamCount, } limiter := NewLimiter(limits, NilMetrics, ring, testData.ringReplicationFactor) defaultCountSupplier := func() int { diff --git a/pkg/ingester/owned_streams.go b/pkg/ingester/owned_streams.go index 01cb8235f9b1..3be6fb40fdd8 100644 --- a/pkg/ingester/owned_streams.go +++ b/pkg/ingester/owned_streams.go @@ -1,6 +1,10 @@ package ingester -import "go.uber.org/atomic" +import ( + "sync" + + "go.uber.org/atomic" +) type ownedStreamService struct { tenantID string @@ -8,22 +12,25 @@ type ownedStreamService struct { fixedLimit *atomic.Int32 //todo: implement job to recalculate it - ownedStreamCount *atomic.Int64 + ownedStreamCount int + notOwnedStreamCount int + lock sync.RWMutex } func newOwnedStreamService(tenantID string, limiter *Limiter) *ownedStreamService { svc := &ownedStreamService{ - tenantID: tenantID, - limiter: limiter, - ownedStreamCount: atomic.NewInt64(0), - fixedLimit: atomic.NewInt32(0), + tenantID: tenantID, + limiter: limiter, + fixedLimit: atomic.NewInt32(0), } svc.updateFixedLimit() return svc } func (s *ownedStreamService) getOwnedStreamCount() int { - return int(s.ownedStreamCount.Load()) + s.lock.RLock() + defer s.lock.RUnlock() + return s.ownedStreamCount } func (s *ownedStreamService) updateFixedLimit() { @@ -36,9 +43,17 @@ func (s *ownedStreamService) getFixedLimit() int { } func (s *ownedStreamService) incOwnedStreamCount() { - s.ownedStreamCount.Inc() + s.lock.Lock() + defer s.lock.Unlock() + s.ownedStreamCount++ } func (s *ownedStreamService) decOwnedStreamCount() { - s.ownedStreamCount.Dec() + s.lock.Lock() + defer s.lock.Unlock() + if s.notOwnedStreamCount > 0 { + s.notOwnedStreamCount-- + return + } + s.ownedStreamCount-- } diff --git a/pkg/ingester/owned_streams_test.go b/pkg/ingester/owned_streams_test.go index c7ddd9d87f29..759927a1d0cf 100644 --- a/pkg/ingester/owned_streams_test.go +++ b/pkg/ingester/owned_streams_test.go @@ -1,6 +1,7 @@ package ingester import ( + "sync" "testing" "github.com/stretchr/testify/require" @@ -29,8 +30,37 @@ func Test_OwnedStreamService(t *testing.T) { service.incOwnedStreamCount() service.incOwnedStreamCount() - require.Equal(t, 2, service.getOwnedStreamCount()) + service.incOwnedStreamCount() + require.Equal(t, 3, service.getOwnedStreamCount()) + + // simulate the effect from the recalculation job + service.notOwnedStreamCount = 1 + service.ownedStreamCount = 2 + + service.decOwnedStreamCount() + require.Equal(t, 2, service.getOwnedStreamCount(), "owned stream count must be decremented only when notOwnedStreamCount is set to 0") + require.Equal(t, 0, service.notOwnedStreamCount) service.decOwnedStreamCount() require.Equal(t, 1, service.getOwnedStreamCount()) + require.Equal(t, 0, service.notOwnedStreamCount, "notOwnedStreamCount must not be decremented lower than 0") + + group := sync.WaitGroup{} + group.Add(200) + for i := 0; i < 100; i++ { + go func() { + defer group.Done() + service.incOwnedStreamCount() + }() + } + + for i := 0; i < 100; i++ { + go func() { + defer group.Done() + service.decOwnedStreamCount() + }() + } + group.Wait() + + require.Equal(t, 1, service.getOwnedStreamCount(), "owned stream count must not be changed") } From f6529c293e3281ba1c19ff36d6018c0ef338f3e6 Mon Sep 17 00:00:00 2001 From: Salva Corts Date: Mon, 27 May 2024 12:19:13 +0200 Subject: [PATCH 23/41] refactor(blooms): Add RPC service for bloom-planner (#13015) --- pkg/bloombuild/planner/metrics.go | 7 + pkg/bloombuild/planner/planner.go | 194 +++- pkg/bloombuild/planner/planner_test.go | 214 +++- pkg/bloombuild/planner/task.go | 25 +- pkg/bloombuild/protos/compat.go | 113 +++ pkg/bloombuild/protos/service.pb.go | 1175 ++++++++++++++++++++++ pkg/bloombuild/protos/service.proto | 32 + pkg/bloombuild/protos/types.pb.go | 1255 ++++++++++++++++++++++++ pkg/bloombuild/protos/types.proto | 45 + 9 files changed, 2992 insertions(+), 68 deletions(-) create mode 100644 pkg/bloombuild/protos/compat.go create mode 100644 pkg/bloombuild/protos/service.pb.go create mode 100644 pkg/bloombuild/protos/service.proto create mode 100644 pkg/bloombuild/protos/types.pb.go create mode 100644 pkg/bloombuild/protos/types.proto diff --git a/pkg/bloombuild/planner/metrics.go b/pkg/bloombuild/planner/metrics.go index 347af1926617..9eaf453b7853 100644 --- a/pkg/bloombuild/planner/metrics.go +++ b/pkg/bloombuild/planner/metrics.go @@ -24,6 +24,7 @@ type Metrics struct { connectedBuilders prometheus.GaugeFunc queueDuration prometheus.Histogram inflightRequests prometheus.Summary + taskLost prometheus.Counter buildStarted prometheus.Counter buildCompleted *prometheus.CounterVec @@ -65,6 +66,12 @@ func NewMetrics( MaxAge: time.Minute, AgeBuckets: 6, }), + taskLost: promauto.With(r).NewCounter(prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "tasks_lost_total", + Help: "Total number of tasks lost due to not being picked up by a builder and failed to be requeued.", + }), buildStarted: promauto.With(r).NewCounter(prometheus.CounterOpts{ Namespace: metricsNamespace, diff --git a/pkg/bloombuild/planner/planner.go b/pkg/bloombuild/planner/planner.go index 9a5b9f6dc238..dfb6fea80cc3 100644 --- a/pkg/bloombuild/planner/planner.go +++ b/pkg/bloombuild/planner/planner.go @@ -4,14 +4,17 @@ import ( "context" "fmt" "sort" + "sync" "time" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/grafana/dskit/services" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/grafana/loki/v3/pkg/bloombuild/protos" "github.com/grafana/loki/v3/pkg/queue" "github.com/grafana/loki/v3/pkg/storage" v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1" @@ -22,6 +25,8 @@ import ( utillog "github.com/grafana/loki/v3/pkg/util/log" ) +var errPlannerIsNotRunning = errors.New("planner is not running") + type Planner struct { services.Service // Subservices manager. @@ -38,6 +43,8 @@ type Planner struct { tasksQueue *queue.RequestQueue activeUsers *util.ActiveUsersCleanupService + pendingTasks sync.Map + metrics *Metrics logger log.Logger } @@ -92,13 +99,23 @@ func New( return p, nil } -func (p *Planner) starting(_ context.Context) (err error) { +func (p *Planner) starting(ctx context.Context) (err error) { + if err := services.StartManagerAndAwaitHealthy(ctx, p.subservices); err != nil { + return fmt.Errorf("error starting planner subservices: %w", err) + } + p.metrics.running.Set(1) - return err + return nil } func (p *Planner) stopping(_ error) error { - p.metrics.running.Set(0) + defer p.metrics.running.Set(0) + + // This will also stop the requests queue, which stop accepting new requests and errors out any pending requests. + if err := services.StopManagerAndAwaitStopped(context.Background(), p.subservices); err != nil { + return fmt.Errorf("error stopping planner subservices: %w", err) + } + return nil } @@ -108,20 +125,32 @@ func (p *Planner) running(ctx context.Context) error { level.Error(p.logger).Log("msg", "bloom build iteration failed for the first time", "err", err) } - ticker := time.NewTicker(p.cfg.PlanningInterval) - defer ticker.Stop() + planningTicker := time.NewTicker(p.cfg.PlanningInterval) + defer planningTicker.Stop() + + inflightTasksTicker := time.NewTicker(250 * time.Millisecond) + defer inflightTasksTicker.Stop() + for { select { case <-ctx.Done(): - err := ctx.Err() - level.Debug(p.logger).Log("msg", "planner context done", "err", err) - return err + if err := ctx.Err(); !errors.Is(err, context.Canceled) { + level.Error(p.logger).Log("msg", "planner context done with error", "err", err) + return err + } - case <-ticker.C: + level.Debug(p.logger).Log("msg", "planner context done") + return nil + + case <-planningTicker.C: level.Info(p.logger).Log("msg", "starting bloom build iteration") if err := p.runOne(ctx); err != nil { level.Error(p.logger).Log("msg", "bloom build iteration failed", "err", err) } + + case <-inflightTasksTicker.C: + inflight := p.totalPendingTasks() + p.metrics.inflightRequests.Observe(float64(inflight)) } } } @@ -159,19 +188,13 @@ func (p *Planner) runOne(ctx context.Context) error { now := time.Now() for _, gap := range gaps { totalTasks++ - task := Task{ - table: w.table.Addr(), - tenant: w.tenant, - OwnershipBounds: w.ownershipRange, - tsdb: gap.tsdb, - gaps: gap.gaps, - - queueTime: now, - ctx: ctx, - } - p.activeUsers.UpdateUserTimestamp(task.tenant, now) - if err := p.tasksQueue.Enqueue(task.tenant, nil, task, nil); err != nil { + task := NewTask( + ctx, now, + protos.NewTask(w.table.Addr(), w.tenant, w.ownershipRange, gap.tsdb, gap.gaps), + ) + + if err := p.enqueueTask(task); err != nil { level.Error(logger).Log("msg", "error enqueuing task", "err", err) continue } @@ -326,7 +349,7 @@ func (p *Planner) findGapsForBounds( // This is a performance optimization to avoid expensive re-reindexing type blockPlan struct { tsdb tsdb.SingleTenantTSDBIdentifier - gaps []GapWithBlocks + gaps []protos.GapWithBlocks } func (p *Planner) findOutdatedGaps( @@ -420,12 +443,12 @@ func blockPlansForGaps(tsdbs []tsdbGaps, metas []bloomshipper.Meta) ([]blockPlan for _, idx := range tsdbs { plan := blockPlan{ tsdb: idx.tsdb, - gaps: make([]GapWithBlocks, 0, len(idx.gaps)), + gaps: make([]protos.GapWithBlocks, 0, len(idx.gaps)), } for _, gap := range idx.gaps { - planGap := GapWithBlocks{ - bounds: gap, + planGap := protos.GapWithBlocks{ + Bounds: gap, } for _, meta := range metas { @@ -442,18 +465,18 @@ func blockPlansForGaps(tsdbs []tsdbGaps, metas []bloomshipper.Meta) ([]blockPlan } // this block overlaps the gap, add it to the plan // for this gap - planGap.blocks = append(planGap.blocks, block) + planGap.Blocks = append(planGap.Blocks, block) } } // ensure we sort blocks so deduping iterator works as expected - sort.Slice(planGap.blocks, func(i, j int) bool { - return planGap.blocks[i].Bounds.Less(planGap.blocks[j].Bounds) + sort.Slice(planGap.Blocks, func(i, j int) bool { + return planGap.Blocks[i].Bounds.Less(planGap.Blocks[j].Bounds) }) peekingBlocks := v1.NewPeekingIter[bloomshipper.BlockRef]( v1.NewSliceIter[bloomshipper.BlockRef]( - planGap.blocks, + planGap.Blocks, ), ) // dedupe blocks which could be in multiple metas @@ -472,7 +495,7 @@ func blockPlansForGaps(tsdbs []tsdbGaps, metas []bloomshipper.Meta) ([]blockPlan if err != nil { return nil, fmt.Errorf("failed to dedupe blocks: %w", err) } - planGap.blocks = deduped + planGap.Blocks = deduped plan.gaps = append(plan.gaps, planGap) } @@ -482,3 +505,114 @@ func blockPlansForGaps(tsdbs []tsdbGaps, metas []bloomshipper.Meta) ([]blockPlan return plans, nil } + +func (p *Planner) addPendingTask(task *Task) { + p.pendingTasks.Store(task.ID, task) +} + +func (p *Planner) removePendingTask(task *Task) { + p.pendingTasks.Delete(task.ID) +} + +func (p *Planner) totalPendingTasks() (total int) { + p.pendingTasks.Range(func(_, _ interface{}) bool { + total++ + return true + }) + return total +} + +func (p *Planner) enqueueTask(task *Task) error { + p.activeUsers.UpdateUserTimestamp(task.Tenant, time.Now()) + return p.tasksQueue.Enqueue(task.Tenant, nil, task, func() { + p.addPendingTask(task) + }) +} + +func (p *Planner) NotifyBuilderShutdown( + _ context.Context, + req *protos.NotifyBuilderShutdownRequest, +) (*protos.NotifyBuilderShutdownResponse, error) { + level.Debug(p.logger).Log("msg", "builder shutdown", "builder", req.BuilderID) + p.tasksQueue.UnregisterConsumerConnection(req.GetBuilderID()) + + return &protos.NotifyBuilderShutdownResponse{}, nil +} + +func (p *Planner) BuilderLoop(builder protos.PlannerForBuilder_BuilderLoopServer) error { + resp, err := builder.Recv() + if err != nil { + return fmt.Errorf("error receiving message from builder: %w", err) + } + + builderID := resp.GetBuilderID() + logger := log.With(p.logger, "builder", builderID) + level.Debug(logger).Log("msg", "builder connected") + + p.tasksQueue.RegisterConsumerConnection(builderID) + defer p.tasksQueue.UnregisterConsumerConnection(builderID) + + lastIndex := queue.StartIndex + for p.isRunningOrStopping() { + item, idx, err := p.tasksQueue.Dequeue(builder.Context(), lastIndex, builderID) + if err != nil { + return fmt.Errorf("error dequeuing task: %w", err) + } + lastIndex = idx + + if item == nil { + + return fmt.Errorf("dequeue() call resulted in nil response. builder: %s", builderID) + } + task := item.(*Task) + + queueTime := time.Since(task.queueTime) + p.metrics.queueDuration.Observe(queueTime.Seconds()) + + if task.ctx.Err() != nil { + level.Warn(logger).Log("msg", "task context done after dequeue", "err", task.ctx.Err()) + lastIndex = lastIndex.ReuseLastIndex() + p.removePendingTask(task) + continue + } + + if err := p.forwardTaskToBuilder(builder, builderID, task); err != nil { + // Re-queue the task if the builder is failing to process the tasks + if err := p.enqueueTask(task); err != nil { + p.metrics.taskLost.Inc() + level.Error(logger).Log("msg", "error re-enqueuing task. this task will be lost", "err", err) + } + + return fmt.Errorf("error forwarding task to builder (%s). Task requeued: %w", builderID, err) + } + + } + + return errPlannerIsNotRunning +} + +func (p *Planner) forwardTaskToBuilder( + builder protos.PlannerForBuilder_BuilderLoopServer, + builderID string, + task *Task, +) error { + defer p.removePendingTask(task) + + msg := &protos.PlannerToBuilder{ + Task: task.ToProtoTask(), + } + + if err := builder.Send(msg); err != nil { + return fmt.Errorf("error sending task to builder (%s): %w", builderID, err) + } + + // TODO(salvacorts): Implement timeout and retry for builder response. + _, err := builder.Recv() + + return err +} + +func (p *Planner) isRunningOrStopping() bool { + st := p.State() + return st == services.Running || st == services.Stopping +} diff --git a/pkg/bloombuild/planner/planner_test.go b/pkg/bloombuild/planner/planner_test.go index 346bd145ab8d..8eccc77e19bf 100644 --- a/pkg/bloombuild/planner/planner_test.go +++ b/pkg/bloombuild/planner/planner_test.go @@ -1,15 +1,28 @@ package planner import ( + "context" + "fmt" "testing" "time" + "github.com/go-kit/log" + "github.com/grafana/dskit/flagext" + "github.com/grafana/dskit/services" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "github.com/grafana/loki/v3/pkg/bloombuild/protos" + "github.com/grafana/loki/v3/pkg/storage" v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1" + "github.com/grafana/loki/v3/pkg/storage/chunk/client/local" + "github.com/grafana/loki/v3/pkg/storage/config" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper" + bloomshipperconfig "github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper/config" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb" + "github.com/grafana/loki/v3/pkg/storage/types" ) func tsdbID(n int) tsdb.SingleTenantTSDBIdentifier { @@ -155,9 +168,9 @@ func Test_blockPlansForGaps(t *testing.T) { exp: []blockPlan{ { tsdb: tsdbID(0), - gaps: []GapWithBlocks{ + gaps: []protos.GapWithBlocks{ { - bounds: v1.NewBounds(0, 10), + Bounds: v1.NewBounds(0, 10), }, }, }, @@ -173,10 +186,10 @@ func Test_blockPlansForGaps(t *testing.T) { exp: []blockPlan{ { tsdb: tsdbID(0), - gaps: []GapWithBlocks{ + gaps: []protos.GapWithBlocks{ { - bounds: v1.NewBounds(0, 10), - blocks: []bloomshipper.BlockRef{genBlockRef(9, 20)}, + Bounds: v1.NewBounds(0, 10), + Blocks: []bloomshipper.BlockRef{genBlockRef(9, 20)}, }, }, }, @@ -196,9 +209,9 @@ func Test_blockPlansForGaps(t *testing.T) { exp: []blockPlan{ { tsdb: tsdbID(0), - gaps: []GapWithBlocks{ + gaps: []protos.GapWithBlocks{ { - bounds: v1.NewBounds(0, 8), + Bounds: v1.NewBounds(0, 8), }, }, }, @@ -215,10 +228,10 @@ func Test_blockPlansForGaps(t *testing.T) { exp: []blockPlan{ { tsdb: tsdbID(0), - gaps: []GapWithBlocks{ + gaps: []protos.GapWithBlocks{ { - bounds: v1.NewBounds(0, 8), - blocks: []bloomshipper.BlockRef{genBlockRef(5, 20)}, + Bounds: v1.NewBounds(0, 8), + Blocks: []bloomshipper.BlockRef{genBlockRef(5, 20)}, }, }, }, @@ -241,32 +254,32 @@ func Test_blockPlansForGaps(t *testing.T) { exp: []blockPlan{ { tsdb: tsdbID(0), - gaps: []GapWithBlocks{ + gaps: []protos.GapWithBlocks{ // tsdb (id=0) can source chunks from the blocks built from tsdb (id=1) { - bounds: v1.NewBounds(3, 5), - blocks: []bloomshipper.BlockRef{genBlockRef(3, 5)}, + Bounds: v1.NewBounds(3, 5), + Blocks: []bloomshipper.BlockRef{genBlockRef(3, 5)}, }, { - bounds: v1.NewBounds(9, 10), - blocks: []bloomshipper.BlockRef{genBlockRef(8, 10)}, + Bounds: v1.NewBounds(9, 10), + Blocks: []bloomshipper.BlockRef{genBlockRef(8, 10)}, }, }, }, // tsdb (id=1) can source chunks from the blocks built from tsdb (id=0) { tsdb: tsdbID(1), - gaps: []GapWithBlocks{ + gaps: []protos.GapWithBlocks{ { - bounds: v1.NewBounds(0, 2), - blocks: []bloomshipper.BlockRef{ + Bounds: v1.NewBounds(0, 2), + Blocks: []bloomshipper.BlockRef{ genBlockRef(0, 1), genBlockRef(1, 2), }, }, { - bounds: v1.NewBounds(6, 7), - blocks: []bloomshipper.BlockRef{genBlockRef(6, 8)}, + Bounds: v1.NewBounds(6, 7), + Blocks: []bloomshipper.BlockRef{genBlockRef(6, 8)}, }, }, }, @@ -289,10 +302,10 @@ func Test_blockPlansForGaps(t *testing.T) { exp: []blockPlan{ { tsdb: tsdbID(0), - gaps: []GapWithBlocks{ + gaps: []protos.GapWithBlocks{ { - bounds: v1.NewBounds(0, 10), - blocks: []bloomshipper.BlockRef{ + Bounds: v1.NewBounds(0, 10), + Blocks: []bloomshipper.BlockRef{ genBlockRef(1, 4), genBlockRef(5, 10), genBlockRef(9, 20), @@ -319,3 +332,158 @@ func Test_blockPlansForGaps(t *testing.T) { }) } } + +func Test_BuilderLoop(t *testing.T) { + const ( + nTasks = 100 + nBuilders = 10 + ) + logger := log.NewNopLogger() + + limits := &fakeLimits{} + cfg := Config{ + PlanningInterval: 1 * time.Hour, + MaxQueuedTasksPerTenant: 10000, + } + schemaCfg := config.SchemaConfig{ + Configs: []config.PeriodConfig{ + { + From: parseDayTime("2023-09-01"), + IndexTables: config.IndexPeriodicTableConfig{ + PeriodicTableConfig: config.PeriodicTableConfig{ + Prefix: "index_", + Period: 24 * time.Hour, + }, + }, + IndexType: types.TSDBType, + ObjectType: types.StorageTypeFileSystem, + Schema: "v13", + RowShards: 16, + }, + }, + } + storageCfg := storage.Config{ + BloomShipperConfig: bloomshipperconfig.Config{ + WorkingDirectory: []string{t.TempDir()}, + DownloadParallelism: 1, + BlocksCache: bloomshipperconfig.BlocksCacheConfig{ + SoftLimit: flagext.Bytes(10 << 20), + HardLimit: flagext.Bytes(20 << 20), + TTL: time.Hour, + }, + }, + FSConfig: local.FSConfig{ + Directory: t.TempDir(), + }, + } + + // Create planner + planner, err := New(cfg, limits, schemaCfg, storageCfg, storage.NewClientMetrics(), nil, logger, prometheus.DefaultRegisterer) + require.NoError(t, err) + + // Start planner + err = services.StartAndAwaitRunning(context.Background(), planner) + require.NoError(t, err) + t.Cleanup(func() { + err := services.StopAndAwaitTerminated(context.Background(), planner) + require.NoError(t, err) + }) + + // Enqueue tasks + for i := 0; i < nTasks; i++ { + task := NewTask( + context.Background(), time.Now(), + protos.NewTask("fakeTable", "fakeTenant", v1.NewBounds(0, 10), tsdbID(1), nil), + ) + + err = planner.enqueueTask(task) + require.NoError(t, err) + } + + // All tasks should be pending + require.Equal(t, nTasks, planner.totalPendingTasks()) + + // Create builders and call planner.BuilderLoop + builders := make([]*fakeBuilder, 0, nBuilders) + for i := 0; i < nBuilders; i++ { + builder := newMockBuilder(fmt.Sprintf("builder-%d", i)) + builders = append(builders, builder) + + go func() { + // We ignore the error since when the planner is stopped, + // the loop will return an error (queue closed) + _ = planner.BuilderLoop(builder) + }() + } + + // Eventually, all tasks should be sent to builders + require.Eventually(t, func() bool { + var receivedTasks int + for _, builder := range builders { + receivedTasks += len(builder.ReceivedTasks()) + } + return receivedTasks == nTasks + }, 15*time.Second, 10*time.Millisecond) + + // Finally, the queue should be empty + require.Equal(t, 0, planner.totalPendingTasks()) +} + +type fakeBuilder struct { + id string + tasks []*protos.Task + grpc.ServerStream +} + +func newMockBuilder(id string) *fakeBuilder { + return &fakeBuilder{id: id} +} + +func (f *fakeBuilder) ReceivedTasks() []*protos.Task { + return f.tasks +} + +func (f *fakeBuilder) Context() context.Context { + return context.Background() +} + +func (f *fakeBuilder) Send(req *protos.PlannerToBuilder) error { + task, err := protos.FromProtoTask(req.Task) + if err != nil { + return err + } + + f.tasks = append(f.tasks, task) + return nil +} + +func (f *fakeBuilder) Recv() (*protos.BuilderToPlanner, error) { + return &protos.BuilderToPlanner{ + BuilderID: f.id, + }, nil +} + +type fakeLimits struct { +} + +func (f *fakeLimits) BloomCreationEnabled(_ string) bool { + return true +} + +func (f *fakeLimits) BloomSplitSeriesKeyspaceBy(_ string) int { + return 1 +} + +func (f *fakeLimits) BloomBuildMaxBuilders(_ string) int { + return 0 +} + +func parseDayTime(s string) config.DayTime { + t, err := time.Parse("2006-01-02", s) + if err != nil { + panic(err) + } + return config.DayTime{ + Time: model.TimeFromUnix(t.Unix()), + } +} diff --git a/pkg/bloombuild/planner/task.go b/pkg/bloombuild/planner/task.go index bff459fe1764..84c6d7617eaf 100644 --- a/pkg/bloombuild/planner/task.go +++ b/pkg/bloombuild/planner/task.go @@ -4,26 +4,21 @@ import ( "context" "time" - v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1" - "github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper" - "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb" + "github.com/grafana/loki/v3/pkg/bloombuild/protos" ) -// TODO: Extract this definiton to a proto file at pkg/bloombuild/protos/protos.proto - -type GapWithBlocks struct { - bounds v1.FingerprintBounds - blocks []bloomshipper.BlockRef -} - type Task struct { - table string - tenant string - OwnershipBounds v1.FingerprintBounds - tsdb tsdb.SingleTenantTSDBIdentifier - gaps []GapWithBlocks + *protos.Task // Tracking queueTime time.Time ctx context.Context } + +func NewTask(ctx context.Context, queueTime time.Time, task *protos.Task) *Task { + return &Task{ + Task: task, + ctx: ctx, + queueTime: queueTime, + } +} diff --git a/pkg/bloombuild/protos/compat.go b/pkg/bloombuild/protos/compat.go new file mode 100644 index 000000000000..b1ae7cccdbab --- /dev/null +++ b/pkg/bloombuild/protos/compat.go @@ -0,0 +1,113 @@ +package protos + +import ( + "fmt" + + "github.com/google/uuid" + + v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1" + "github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper" + "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb" +) + +type GapWithBlocks struct { + Bounds v1.FingerprintBounds + Blocks []bloomshipper.BlockRef +} + +type Task struct { + ID string + + Table string + Tenant string + OwnershipBounds v1.FingerprintBounds + TSDB tsdb.SingleTenantTSDBIdentifier + Gaps []GapWithBlocks +} + +func NewTask(table, tenant string, bounds v1.FingerprintBounds, tsdb tsdb.SingleTenantTSDBIdentifier, gaps []GapWithBlocks) *Task { + return &Task{ + ID: uuid.NewString(), + + Table: table, + Tenant: tenant, + OwnershipBounds: bounds, + TSDB: tsdb, + Gaps: gaps, + } +} + +// TODO: Use it in the builder to parse the task +func FromProtoTask(task *ProtoTask) (*Task, error) { + if task == nil { + return nil, nil + } + + tsdbRef, ok := tsdb.ParseSingleTenantTSDBPath(task.Tsdb) + if !ok { + return nil, fmt.Errorf("failed to parse tsdb path %s", task.Tsdb) + } + + gaps := make([]GapWithBlocks, 0, len(task.Gaps)) + for _, gap := range task.Gaps { + bounds := v1.FingerprintBounds{ + Min: gap.Bounds.Min, + Max: gap.Bounds.Max, + } + blocks := make([]bloomshipper.BlockRef, 0, len(gap.BlockRef)) + for _, block := range gap.BlockRef { + b, err := bloomshipper.BlockRefFromKey(block) + if err != nil { + return nil, fmt.Errorf("failed to parse block ref %s: %w", block, err) + } + + blocks = append(blocks, b) + } + gaps = append(gaps, GapWithBlocks{ + Bounds: bounds, + Blocks: blocks, + }) + } + + return &Task{ + ID: task.Id, + Table: task.Table, + Tenant: task.Tenant, + OwnershipBounds: v1.FingerprintBounds{ + Min: task.Bounds.Min, + Max: task.Bounds.Max, + }, + TSDB: tsdbRef, + Gaps: gaps, + }, nil +} + +func (t *Task) ToProtoTask() *ProtoTask { + protoGaps := make([]*ProtoGapWithBlocks, 0, len(t.Gaps)) + for _, gap := range t.Gaps { + blockRefs := make([]string, 0, len(gap.Blocks)) + for _, block := range gap.Blocks { + blockRefs = append(blockRefs, block.String()) + } + + protoGaps = append(protoGaps, &ProtoGapWithBlocks{ + Bounds: ProtoFingerprintBounds{ + Min: gap.Bounds.Min, + Max: gap.Bounds.Max, + }, + BlockRef: blockRefs, + }) + } + + return &ProtoTask{ + Id: t.ID, + Table: t.Table, + Tenant: t.Tenant, + Bounds: ProtoFingerprintBounds{ + Min: t.OwnershipBounds.Min, + Max: t.OwnershipBounds.Max, + }, + Tsdb: t.TSDB.Path(), + Gaps: protoGaps, + } +} diff --git a/pkg/bloombuild/protos/service.pb.go b/pkg/bloombuild/protos/service.pb.go new file mode 100644 index 000000000000..91684dd90ef8 --- /dev/null +++ b/pkg/bloombuild/protos/service.pb.go @@ -0,0 +1,1175 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: pkg/bloombuild/protos/service.proto + +package protos + +import ( + context "context" + fmt "fmt" + _ "github.com/gogo/protobuf/gogoproto" + proto "github.com/gogo/protobuf/proto" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + io "io" + math "math" + math_bits "math/bits" + reflect "reflect" + strings "strings" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +type BuilderToPlanner struct { + BuilderID string `protobuf:"bytes,1,opt,name=builderID,proto3" json:"builderID,omitempty"` +} + +func (m *BuilderToPlanner) Reset() { *m = BuilderToPlanner{} } +func (*BuilderToPlanner) ProtoMessage() {} +func (*BuilderToPlanner) Descriptor() ([]byte, []int) { + return fileDescriptor_89de33e08b859356, []int{0} +} +func (m *BuilderToPlanner) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *BuilderToPlanner) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_BuilderToPlanner.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *BuilderToPlanner) XXX_Merge(src proto.Message) { + xxx_messageInfo_BuilderToPlanner.Merge(m, src) +} +func (m *BuilderToPlanner) XXX_Size() int { + return m.Size() +} +func (m *BuilderToPlanner) XXX_DiscardUnknown() { + xxx_messageInfo_BuilderToPlanner.DiscardUnknown(m) +} + +var xxx_messageInfo_BuilderToPlanner proto.InternalMessageInfo + +func (m *BuilderToPlanner) GetBuilderID() string { + if m != nil { + return m.BuilderID + } + return "" +} + +type PlannerToBuilder struct { + Task *ProtoTask `protobuf:"bytes,1,opt,name=task,proto3" json:"task,omitempty"` +} + +func (m *PlannerToBuilder) Reset() { *m = PlannerToBuilder{} } +func (*PlannerToBuilder) ProtoMessage() {} +func (*PlannerToBuilder) Descriptor() ([]byte, []int) { + return fileDescriptor_89de33e08b859356, []int{1} +} +func (m *PlannerToBuilder) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *PlannerToBuilder) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_PlannerToBuilder.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *PlannerToBuilder) XXX_Merge(src proto.Message) { + xxx_messageInfo_PlannerToBuilder.Merge(m, src) +} +func (m *PlannerToBuilder) XXX_Size() int { + return m.Size() +} +func (m *PlannerToBuilder) XXX_DiscardUnknown() { + xxx_messageInfo_PlannerToBuilder.DiscardUnknown(m) +} + +var xxx_messageInfo_PlannerToBuilder proto.InternalMessageInfo + +func (m *PlannerToBuilder) GetTask() *ProtoTask { + if m != nil { + return m.Task + } + return nil +} + +type NotifyBuilderShutdownRequest struct { + BuilderID string `protobuf:"bytes,1,opt,name=builderID,proto3" json:"builderID,omitempty"` +} + +func (m *NotifyBuilderShutdownRequest) Reset() { *m = NotifyBuilderShutdownRequest{} } +func (*NotifyBuilderShutdownRequest) ProtoMessage() {} +func (*NotifyBuilderShutdownRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_89de33e08b859356, []int{2} +} +func (m *NotifyBuilderShutdownRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *NotifyBuilderShutdownRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_NotifyBuilderShutdownRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *NotifyBuilderShutdownRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_NotifyBuilderShutdownRequest.Merge(m, src) +} +func (m *NotifyBuilderShutdownRequest) XXX_Size() int { + return m.Size() +} +func (m *NotifyBuilderShutdownRequest) XXX_DiscardUnknown() { + xxx_messageInfo_NotifyBuilderShutdownRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_NotifyBuilderShutdownRequest proto.InternalMessageInfo + +func (m *NotifyBuilderShutdownRequest) GetBuilderID() string { + if m != nil { + return m.BuilderID + } + return "" +} + +type NotifyBuilderShutdownResponse struct { +} + +func (m *NotifyBuilderShutdownResponse) Reset() { *m = NotifyBuilderShutdownResponse{} } +func (*NotifyBuilderShutdownResponse) ProtoMessage() {} +func (*NotifyBuilderShutdownResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_89de33e08b859356, []int{3} +} +func (m *NotifyBuilderShutdownResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *NotifyBuilderShutdownResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_NotifyBuilderShutdownResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *NotifyBuilderShutdownResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NotifyBuilderShutdownResponse.Merge(m, src) +} +func (m *NotifyBuilderShutdownResponse) XXX_Size() int { + return m.Size() +} +func (m *NotifyBuilderShutdownResponse) XXX_DiscardUnknown() { + xxx_messageInfo_NotifyBuilderShutdownResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_NotifyBuilderShutdownResponse proto.InternalMessageInfo + +func init() { + proto.RegisterType((*BuilderToPlanner)(nil), "protos.BuilderToPlanner") + proto.RegisterType((*PlannerToBuilder)(nil), "protos.PlannerToBuilder") + proto.RegisterType((*NotifyBuilderShutdownRequest)(nil), "protos.NotifyBuilderShutdownRequest") + proto.RegisterType((*NotifyBuilderShutdownResponse)(nil), "protos.NotifyBuilderShutdownResponse") +} + +func init() { + proto.RegisterFile("pkg/bloombuild/protos/service.proto", fileDescriptor_89de33e08b859356) +} + +var fileDescriptor_89de33e08b859356 = []byte{ + // 323 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x2e, 0xc8, 0x4e, 0xd7, + 0x4f, 0xca, 0xc9, 0xcf, 0xcf, 0x4d, 0x2a, 0xcd, 0xcc, 0x49, 0xd1, 0x2f, 0x28, 0xca, 0x2f, 0xc9, + 0x2f, 0xd6, 0x2f, 0x4e, 0x2d, 0x2a, 0xcb, 0x4c, 0x4e, 0xd5, 0x03, 0x73, 0x85, 0xd8, 0x20, 0xa2, + 0x52, 0x22, 0xe9, 0xf9, 0xe9, 0xf9, 0x60, 0xb6, 0x3e, 0x88, 0x05, 0x91, 0x95, 0x52, 0xc4, 0x6e, + 0x44, 0x49, 0x65, 0x41, 0x6a, 0x31, 0x44, 0x89, 0x92, 0x01, 0x97, 0x80, 0x13, 0x48, 0x2e, 0xb5, + 0x28, 0x24, 0x3f, 0x20, 0x27, 0x31, 0x2f, 0x2f, 0xb5, 0x48, 0x48, 0x86, 0x8b, 0x33, 0x09, 0x22, + 0xe6, 0xe9, 0x22, 0xc1, 0xa8, 0xc0, 0xa8, 0xc1, 0x19, 0x84, 0x10, 0x50, 0xb2, 0xe4, 0x12, 0x80, + 0x2a, 0x0c, 0xc9, 0x87, 0x6a, 0x15, 0x52, 0xe5, 0x62, 0x29, 0x49, 0x2c, 0xce, 0x06, 0x2b, 0xe6, + 0x36, 0x12, 0x84, 0x98, 0x5d, 0xac, 0x17, 0x00, 0xa2, 0x42, 0x12, 0x8b, 0xb3, 0x83, 0xc0, 0xd2, + 0x4a, 0x36, 0x5c, 0x32, 0x7e, 0xf9, 0x25, 0x99, 0x69, 0x95, 0x50, 0x7d, 0xc1, 0x19, 0xa5, 0x25, + 0x29, 0xf9, 0xe5, 0x79, 0x41, 0xa9, 0x85, 0xa5, 0xa9, 0xc5, 0x25, 0x04, 0x2c, 0x96, 0xe7, 0x92, + 0xc5, 0xa1, 0xbb, 0xb8, 0x20, 0x3f, 0xaf, 0x38, 0xd5, 0xe8, 0x08, 0x23, 0x97, 0x20, 0xd4, 0x69, + 0x6e, 0xf9, 0x45, 0x30, 0xb7, 0xb9, 0x73, 0x71, 0x43, 0x99, 0x3e, 0xf9, 0xf9, 0x05, 0x42, 0x12, + 0x30, 0xc7, 0xa1, 0x7b, 0x5b, 0x0a, 0x2e, 0x83, 0xee, 0x3d, 0x25, 0x06, 0x0d, 0x46, 0x03, 0x46, + 0xa1, 0x34, 0x2e, 0x51, 0xac, 0xf6, 0x0b, 0xa9, 0xc0, 0x34, 0xe2, 0xf3, 0x9c, 0x94, 0x2a, 0x01, + 0x55, 0x10, 0x4f, 0x28, 0x31, 0x38, 0xd9, 0x5c, 0x78, 0x28, 0xc7, 0x70, 0xe3, 0xa1, 0x1c, 0xc3, + 0x87, 0x87, 0x72, 0x8c, 0x0d, 0x8f, 0xe4, 0x18, 0x57, 0x3c, 0x92, 0x63, 0x3c, 0xf1, 0x48, 0x8e, + 0xf1, 0xc2, 0x23, 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x5f, 0x3c, 0x92, 0x63, 0xf8, 0xf0, 0x48, + 0x8e, 0x71, 0xc2, 0x63, 0x39, 0x86, 0x0b, 0x8f, 0xe5, 0x18, 0x6e, 0x3c, 0x96, 0x63, 0x88, 0x82, + 0xa6, 0x84, 0x24, 0x08, 0x6d, 0x0c, 0x08, 0x00, 0x00, 0xff, 0xff, 0x28, 0x86, 0x3f, 0xfe, 0x3f, + 0x02, 0x00, 0x00, +} + +func (this *BuilderToPlanner) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*BuilderToPlanner) + if !ok { + that2, ok := that.(BuilderToPlanner) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.BuilderID != that1.BuilderID { + return false + } + return true +} +func (this *PlannerToBuilder) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*PlannerToBuilder) + if !ok { + that2, ok := that.(PlannerToBuilder) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if !this.Task.Equal(that1.Task) { + return false + } + return true +} +func (this *NotifyBuilderShutdownRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*NotifyBuilderShutdownRequest) + if !ok { + that2, ok := that.(NotifyBuilderShutdownRequest) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.BuilderID != that1.BuilderID { + return false + } + return true +} +func (this *NotifyBuilderShutdownResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*NotifyBuilderShutdownResponse) + if !ok { + that2, ok := that.(NotifyBuilderShutdownResponse) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + return true +} +func (this *BuilderToPlanner) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 5) + s = append(s, "&protos.BuilderToPlanner{") + s = append(s, "BuilderID: "+fmt.Sprintf("%#v", this.BuilderID)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} +func (this *PlannerToBuilder) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 5) + s = append(s, "&protos.PlannerToBuilder{") + if this.Task != nil { + s = append(s, "Task: "+fmt.Sprintf("%#v", this.Task)+",\n") + } + s = append(s, "}") + return strings.Join(s, "") +} +func (this *NotifyBuilderShutdownRequest) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 5) + s = append(s, "&protos.NotifyBuilderShutdownRequest{") + s = append(s, "BuilderID: "+fmt.Sprintf("%#v", this.BuilderID)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} +func (this *NotifyBuilderShutdownResponse) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 4) + s = append(s, "&protos.NotifyBuilderShutdownResponse{") + s = append(s, "}") + return strings.Join(s, "") +} +func valueToGoStringService(v interface{}, typ string) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// PlannerForBuilderClient is the client API for PlannerForBuilder service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type PlannerForBuilderClient interface { + BuilderLoop(ctx context.Context, opts ...grpc.CallOption) (PlannerForBuilder_BuilderLoopClient, error) + NotifyBuilderShutdown(ctx context.Context, in *NotifyBuilderShutdownRequest, opts ...grpc.CallOption) (*NotifyBuilderShutdownResponse, error) +} + +type plannerForBuilderClient struct { + cc *grpc.ClientConn +} + +func NewPlannerForBuilderClient(cc *grpc.ClientConn) PlannerForBuilderClient { + return &plannerForBuilderClient{cc} +} + +func (c *plannerForBuilderClient) BuilderLoop(ctx context.Context, opts ...grpc.CallOption) (PlannerForBuilder_BuilderLoopClient, error) { + stream, err := c.cc.NewStream(ctx, &_PlannerForBuilder_serviceDesc.Streams[0], "/protos.PlannerForBuilder/BuilderLoop", opts...) + if err != nil { + return nil, err + } + x := &plannerForBuilderBuilderLoopClient{stream} + return x, nil +} + +type PlannerForBuilder_BuilderLoopClient interface { + Send(*BuilderToPlanner) error + Recv() (*PlannerToBuilder, error) + grpc.ClientStream +} + +type plannerForBuilderBuilderLoopClient struct { + grpc.ClientStream +} + +func (x *plannerForBuilderBuilderLoopClient) Send(m *BuilderToPlanner) error { + return x.ClientStream.SendMsg(m) +} + +func (x *plannerForBuilderBuilderLoopClient) Recv() (*PlannerToBuilder, error) { + m := new(PlannerToBuilder) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *plannerForBuilderClient) NotifyBuilderShutdown(ctx context.Context, in *NotifyBuilderShutdownRequest, opts ...grpc.CallOption) (*NotifyBuilderShutdownResponse, error) { + out := new(NotifyBuilderShutdownResponse) + err := c.cc.Invoke(ctx, "/protos.PlannerForBuilder/NotifyBuilderShutdown", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// PlannerForBuilderServer is the server API for PlannerForBuilder service. +type PlannerForBuilderServer interface { + BuilderLoop(PlannerForBuilder_BuilderLoopServer) error + NotifyBuilderShutdown(context.Context, *NotifyBuilderShutdownRequest) (*NotifyBuilderShutdownResponse, error) +} + +// UnimplementedPlannerForBuilderServer can be embedded to have forward compatible implementations. +type UnimplementedPlannerForBuilderServer struct { +} + +func (*UnimplementedPlannerForBuilderServer) BuilderLoop(srv PlannerForBuilder_BuilderLoopServer) error { + return status.Errorf(codes.Unimplemented, "method BuilderLoop not implemented") +} +func (*UnimplementedPlannerForBuilderServer) NotifyBuilderShutdown(ctx context.Context, req *NotifyBuilderShutdownRequest) (*NotifyBuilderShutdownResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method NotifyBuilderShutdown not implemented") +} + +func RegisterPlannerForBuilderServer(s *grpc.Server, srv PlannerForBuilderServer) { + s.RegisterService(&_PlannerForBuilder_serviceDesc, srv) +} + +func _PlannerForBuilder_BuilderLoop_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(PlannerForBuilderServer).BuilderLoop(&plannerForBuilderBuilderLoopServer{stream}) +} + +type PlannerForBuilder_BuilderLoopServer interface { + Send(*PlannerToBuilder) error + Recv() (*BuilderToPlanner, error) + grpc.ServerStream +} + +type plannerForBuilderBuilderLoopServer struct { + grpc.ServerStream +} + +func (x *plannerForBuilderBuilderLoopServer) Send(m *PlannerToBuilder) error { + return x.ServerStream.SendMsg(m) +} + +func (x *plannerForBuilderBuilderLoopServer) Recv() (*BuilderToPlanner, error) { + m := new(BuilderToPlanner) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _PlannerForBuilder_NotifyBuilderShutdown_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(NotifyBuilderShutdownRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(PlannerForBuilderServer).NotifyBuilderShutdown(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/protos.PlannerForBuilder/NotifyBuilderShutdown", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(PlannerForBuilderServer).NotifyBuilderShutdown(ctx, req.(*NotifyBuilderShutdownRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _PlannerForBuilder_serviceDesc = grpc.ServiceDesc{ + ServiceName: "protos.PlannerForBuilder", + HandlerType: (*PlannerForBuilderServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "NotifyBuilderShutdown", + Handler: _PlannerForBuilder_NotifyBuilderShutdown_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "BuilderLoop", + Handler: _PlannerForBuilder_BuilderLoop_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "pkg/bloombuild/protos/service.proto", +} + +func (m *BuilderToPlanner) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *BuilderToPlanner) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *BuilderToPlanner) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.BuilderID) > 0 { + i -= len(m.BuilderID) + copy(dAtA[i:], m.BuilderID) + i = encodeVarintService(dAtA, i, uint64(len(m.BuilderID))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *PlannerToBuilder) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PlannerToBuilder) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *PlannerToBuilder) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Task != nil { + { + size, err := m.Task.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintService(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *NotifyBuilderShutdownRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *NotifyBuilderShutdownRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *NotifyBuilderShutdownRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.BuilderID) > 0 { + i -= len(m.BuilderID) + copy(dAtA[i:], m.BuilderID) + i = encodeVarintService(dAtA, i, uint64(len(m.BuilderID))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *NotifyBuilderShutdownResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *NotifyBuilderShutdownResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *NotifyBuilderShutdownResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + return len(dAtA) - i, nil +} + +func encodeVarintService(dAtA []byte, offset int, v uint64) int { + offset -= sovService(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *BuilderToPlanner) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.BuilderID) + if l > 0 { + n += 1 + l + sovService(uint64(l)) + } + return n +} + +func (m *PlannerToBuilder) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Task != nil { + l = m.Task.Size() + n += 1 + l + sovService(uint64(l)) + } + return n +} + +func (m *NotifyBuilderShutdownRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.BuilderID) + if l > 0 { + n += 1 + l + sovService(uint64(l)) + } + return n +} + +func (m *NotifyBuilderShutdownResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + return n +} + +func sovService(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozService(x uint64) (n int) { + return sovService(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (this *BuilderToPlanner) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&BuilderToPlanner{`, + `BuilderID:` + fmt.Sprintf("%v", this.BuilderID) + `,`, + `}`, + }, "") + return s +} +func (this *PlannerToBuilder) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&PlannerToBuilder{`, + `Task:` + strings.Replace(fmt.Sprintf("%v", this.Task), "ProtoTask", "ProtoTask", 1) + `,`, + `}`, + }, "") + return s +} +func (this *NotifyBuilderShutdownRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&NotifyBuilderShutdownRequest{`, + `BuilderID:` + fmt.Sprintf("%v", this.BuilderID) + `,`, + `}`, + }, "") + return s +} +func (this *NotifyBuilderShutdownResponse) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&NotifyBuilderShutdownResponse{`, + `}`, + }, "") + return s +} +func valueToStringService(v interface{}) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("*%v", pv) +} +func (m *BuilderToPlanner) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowService + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BuilderToPlanner: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BuilderToPlanner: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BuilderID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowService + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthService + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthService + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.BuilderID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipService(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthService + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthService + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PlannerToBuilder) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowService + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PlannerToBuilder: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PlannerToBuilder: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Task", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowService + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthService + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthService + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Task == nil { + m.Task = &ProtoTask{} + } + if err := m.Task.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipService(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthService + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthService + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *NotifyBuilderShutdownRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowService + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: NotifyBuilderShutdownRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: NotifyBuilderShutdownRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BuilderID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowService + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthService + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthService + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.BuilderID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipService(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthService + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthService + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *NotifyBuilderShutdownResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowService + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: NotifyBuilderShutdownResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: NotifyBuilderShutdownResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipService(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthService + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthService + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipService(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowService + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowService + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowService + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthService + } + iNdEx += length + if iNdEx < 0 { + return 0, ErrInvalidLengthService + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowService + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipService(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + if iNdEx < 0 { + return 0, ErrInvalidLengthService + } + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthService = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowService = fmt.Errorf("proto: integer overflow") +) diff --git a/pkg/bloombuild/protos/service.proto b/pkg/bloombuild/protos/service.proto new file mode 100644 index 000000000000..e061684c41be --- /dev/null +++ b/pkg/bloombuild/protos/service.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +package protos; + +import "gogoproto/gogo.proto"; +import "pkg/bloombuild/protos/types.proto"; + +option go_package = "protos"; +option (gogoproto.marshaler_all) = true; +option (gogoproto.unmarshaler_all) = true; + +service PlannerForBuilder { + rpc BuilderLoop(stream BuilderToPlanner) returns (stream PlannerToBuilder) {} + + rpc NotifyBuilderShutdown(NotifyBuilderShutdownRequest) returns (NotifyBuilderShutdownResponse) {} +} + +message BuilderToPlanner { + string builderID = 1; +} + +message PlannerToBuilder { + ProtoTask task = 1; +} + +message NotifyBuilderShutdownRequest { + string builderID = 1; +} + +message NotifyBuilderShutdownResponse { + // empty: just to acknowledge the request +} diff --git a/pkg/bloombuild/protos/types.pb.go b/pkg/bloombuild/protos/types.pb.go new file mode 100644 index 000000000000..5d3b9bcb729a --- /dev/null +++ b/pkg/bloombuild/protos/types.pb.go @@ -0,0 +1,1255 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: pkg/bloombuild/protos/types.proto + +package protos + +import ( + fmt "fmt" + _ "github.com/gogo/protobuf/gogoproto" + proto "github.com/gogo/protobuf/proto" + github_com_prometheus_common_model "github.com/prometheus/common/model" + io "io" + math "math" + math_bits "math/bits" + reflect "reflect" + strings "strings" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +// FPBounds is identical to the definition in `pkg/storage/bloom/v1/bounds.FingerprintBounds` +// which ensures we can cast between them without allocations. +// TODO(salvacorts): Reuse from `pkg/logproto/indexgateway.proto` +type ProtoFingerprintBounds struct { + Min github_com_prometheus_common_model.Fingerprint `protobuf:"varint,1,opt,name=min,proto3,casttype=github.com/prometheus/common/model.Fingerprint" json:"min"` + Max github_com_prometheus_common_model.Fingerprint `protobuf:"varint,2,opt,name=max,proto3,casttype=github.com/prometheus/common/model.Fingerprint" json:"max"` +} + +func (m *ProtoFingerprintBounds) Reset() { *m = ProtoFingerprintBounds{} } +func (*ProtoFingerprintBounds) ProtoMessage() {} +func (*ProtoFingerprintBounds) Descriptor() ([]byte, []int) { + return fileDescriptor_5325fb0610e1e9ae, []int{0} +} +func (m *ProtoFingerprintBounds) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ProtoFingerprintBounds) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ProtoFingerprintBounds.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ProtoFingerprintBounds) XXX_Merge(src proto.Message) { + xxx_messageInfo_ProtoFingerprintBounds.Merge(m, src) +} +func (m *ProtoFingerprintBounds) XXX_Size() int { + return m.Size() +} +func (m *ProtoFingerprintBounds) XXX_DiscardUnknown() { + xxx_messageInfo_ProtoFingerprintBounds.DiscardUnknown(m) +} + +var xxx_messageInfo_ProtoFingerprintBounds proto.InternalMessageInfo + +func (m *ProtoFingerprintBounds) GetMin() github_com_prometheus_common_model.Fingerprint { + if m != nil { + return m.Min + } + return 0 +} + +func (m *ProtoFingerprintBounds) GetMax() github_com_prometheus_common_model.Fingerprint { + if m != nil { + return m.Max + } + return 0 +} + +type ProtoGapWithBlocks struct { + Bounds ProtoFingerprintBounds `protobuf:"bytes,1,opt,name=bounds,proto3" json:"bounds"` + BlockRef []string `protobuf:"bytes,2,rep,name=blockRef,proto3" json:"blockRef,omitempty"` +} + +func (m *ProtoGapWithBlocks) Reset() { *m = ProtoGapWithBlocks{} } +func (*ProtoGapWithBlocks) ProtoMessage() {} +func (*ProtoGapWithBlocks) Descriptor() ([]byte, []int) { + return fileDescriptor_5325fb0610e1e9ae, []int{1} +} +func (m *ProtoGapWithBlocks) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ProtoGapWithBlocks) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ProtoGapWithBlocks.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ProtoGapWithBlocks) XXX_Merge(src proto.Message) { + xxx_messageInfo_ProtoGapWithBlocks.Merge(m, src) +} +func (m *ProtoGapWithBlocks) XXX_Size() int { + return m.Size() +} +func (m *ProtoGapWithBlocks) XXX_DiscardUnknown() { + xxx_messageInfo_ProtoGapWithBlocks.DiscardUnknown(m) +} + +var xxx_messageInfo_ProtoGapWithBlocks proto.InternalMessageInfo + +func (m *ProtoGapWithBlocks) GetBounds() ProtoFingerprintBounds { + if m != nil { + return m.Bounds + } + return ProtoFingerprintBounds{} +} + +func (m *ProtoGapWithBlocks) GetBlockRef() []string { + if m != nil { + return m.BlockRef + } + return nil +} + +// TODO: Define BlockRef and SingleTenantTSDBIdentifier as messages so we can use them right away +// +// instead of unmarshaling them from strings or doing unsafe casts. +type ProtoTask struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Table string `protobuf:"bytes,2,opt,name=table,proto3" json:"table,omitempty"` + Tenant string `protobuf:"bytes,3,opt,name=tenant,proto3" json:"tenant,omitempty"` + Bounds ProtoFingerprintBounds `protobuf:"bytes,4,opt,name=bounds,proto3" json:"bounds"` + Tsdb string `protobuf:"bytes,5,opt,name=tsdb,proto3" json:"tsdb,omitempty"` + Gaps []*ProtoGapWithBlocks `protobuf:"bytes,6,rep,name=gaps,proto3" json:"gaps,omitempty"` +} + +func (m *ProtoTask) Reset() { *m = ProtoTask{} } +func (*ProtoTask) ProtoMessage() {} +func (*ProtoTask) Descriptor() ([]byte, []int) { + return fileDescriptor_5325fb0610e1e9ae, []int{2} +} +func (m *ProtoTask) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ProtoTask) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ProtoTask.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ProtoTask) XXX_Merge(src proto.Message) { + xxx_messageInfo_ProtoTask.Merge(m, src) +} +func (m *ProtoTask) XXX_Size() int { + return m.Size() +} +func (m *ProtoTask) XXX_DiscardUnknown() { + xxx_messageInfo_ProtoTask.DiscardUnknown(m) +} + +var xxx_messageInfo_ProtoTask proto.InternalMessageInfo + +func (m *ProtoTask) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *ProtoTask) GetTable() string { + if m != nil { + return m.Table + } + return "" +} + +func (m *ProtoTask) GetTenant() string { + if m != nil { + return m.Tenant + } + return "" +} + +func (m *ProtoTask) GetBounds() ProtoFingerprintBounds { + if m != nil { + return m.Bounds + } + return ProtoFingerprintBounds{} +} + +func (m *ProtoTask) GetTsdb() string { + if m != nil { + return m.Tsdb + } + return "" +} + +func (m *ProtoTask) GetGaps() []*ProtoGapWithBlocks { + if m != nil { + return m.Gaps + } + return nil +} + +func init() { + proto.RegisterType((*ProtoFingerprintBounds)(nil), "protos.ProtoFingerprintBounds") + proto.RegisterType((*ProtoGapWithBlocks)(nil), "protos.ProtoGapWithBlocks") + proto.RegisterType((*ProtoTask)(nil), "protos.ProtoTask") +} + +func init() { proto.RegisterFile("pkg/bloombuild/protos/types.proto", fileDescriptor_5325fb0610e1e9ae) } + +var fileDescriptor_5325fb0610e1e9ae = []byte{ + // 393 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x92, 0x31, 0xeb, 0x9b, 0x40, + 0x18, 0xc6, 0x3d, 0xf5, 0x2f, 0xf5, 0x02, 0x19, 0x8e, 0x10, 0x24, 0xc3, 0x99, 0x66, 0xca, 0xa4, + 0x90, 0x4e, 0x85, 0x4e, 0x0e, 0xe9, 0xd0, 0xa5, 0x48, 0xa1, 0xd0, 0xed, 0x2e, 0x5e, 0xcd, 0x11, + 0xf5, 0x24, 0x77, 0x82, 0xdd, 0xfa, 0x11, 0xfa, 0x31, 0x3a, 0xf7, 0x53, 0x64, 0xcc, 0x52, 0xc8, + 0x24, 0x8d, 0x59, 0x4a, 0xa6, 0xcc, 0x9d, 0x8a, 0xa7, 0x94, 0x04, 0x3a, 0xb5, 0xd3, 0xfb, 0x3c, + 0xaf, 0xbe, 0xbf, 0xf7, 0x79, 0x45, 0xf8, 0xbc, 0xdc, 0xa5, 0x21, 0xcd, 0x84, 0xc8, 0x69, 0xc5, + 0xb3, 0x24, 0x2c, 0xf7, 0x42, 0x09, 0x19, 0xaa, 0x4f, 0x25, 0x93, 0x81, 0x36, 0xc8, 0xe9, 0x7b, + 0xb3, 0x49, 0x2a, 0x52, 0xa1, 0x75, 0xd8, 0xa9, 0xfe, 0xe9, 0xe2, 0x1b, 0x80, 0xd3, 0xb7, 0x9d, + 0x5a, 0xf3, 0x22, 0x65, 0xfb, 0x72, 0xcf, 0x0b, 0x15, 0x89, 0xaa, 0x48, 0x24, 0x7a, 0x03, 0xad, + 0x9c, 0x17, 0x1e, 0x98, 0x83, 0xa5, 0x1d, 0xbd, 0xbc, 0x36, 0x7e, 0x67, 0x7f, 0x35, 0x7e, 0x90, + 0x72, 0xb5, 0xad, 0x68, 0xb0, 0x11, 0x79, 0xb7, 0x2f, 0x67, 0x6a, 0xcb, 0x2a, 0x19, 0x6e, 0x44, + 0x9e, 0x8b, 0x22, 0xcc, 0x45, 0xc2, 0xb2, 0xe0, 0x8e, 0x16, 0x77, 0x63, 0x1a, 0x46, 0x6a, 0xcf, + 0xbc, 0x83, 0x91, 0xfa, 0x9f, 0x60, 0xa4, 0x5e, 0xd4, 0x10, 0xe9, 0xcc, 0xaf, 0x49, 0xf9, 0x9e, + 0xab, 0x6d, 0x94, 0x89, 0xcd, 0x4e, 0xa2, 0x35, 0x74, 0xa8, 0x4e, 0xae, 0x23, 0x8f, 0x56, 0xb8, + 0x3f, 0x51, 0x06, 0x7f, 0xbf, 0x2f, 0x1a, 0x1f, 0x1a, 0xdf, 0xb8, 0x36, 0xfe, 0x30, 0x15, 0x0f, + 0x15, 0xcd, 0xe0, 0x33, 0xda, 0x11, 0x63, 0xf6, 0xd1, 0x33, 0xe7, 0xd6, 0xd2, 0x8d, 0xff, 0xf8, + 0xc5, 0x77, 0x00, 0x5d, 0x8d, 0x7b, 0x47, 0xe4, 0x0e, 0x8d, 0xa1, 0xc9, 0x13, 0xbd, 0xcd, 0x8d, + 0x4d, 0x9e, 0xa0, 0x09, 0x7c, 0x52, 0x84, 0x66, 0x4c, 0x9f, 0xe9, 0xc6, 0xbd, 0x41, 0x53, 0xe8, + 0x28, 0x56, 0x90, 0x42, 0x79, 0x96, 0x6e, 0x0f, 0xee, 0x2e, 0xaf, 0xfd, 0x5f, 0x79, 0x11, 0xb4, + 0x95, 0x4c, 0xa8, 0xf7, 0xa4, 0xe9, 0x5a, 0xa3, 0x00, 0xda, 0x29, 0x29, 0xa5, 0xe7, 0xcc, 0xad, + 0xe5, 0x68, 0x35, 0x7b, 0x20, 0x3f, 0x7c, 0xb5, 0x58, 0xbf, 0x17, 0xbd, 0x3a, 0x9e, 0xb1, 0x71, + 0x3a, 0x63, 0xe3, 0x76, 0xc6, 0xe0, 0x73, 0x8b, 0xc1, 0xd7, 0x16, 0x83, 0x43, 0x8b, 0xc1, 0xb1, + 0xc5, 0xe0, 0x47, 0x8b, 0xc1, 0xcf, 0x16, 0x1b, 0xb7, 0x16, 0x83, 0x2f, 0x17, 0x6c, 0x1c, 0x2f, + 0xd8, 0x38, 0x5d, 0xb0, 0xf1, 0x61, 0xf8, 0xb5, 0x68, 0x5f, 0x5f, 0xfc, 0x0e, 0x00, 0x00, 0xff, + 0xff, 0x57, 0x05, 0xc7, 0x5d, 0x8e, 0x02, 0x00, 0x00, +} + +func (this *ProtoFingerprintBounds) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*ProtoFingerprintBounds) + if !ok { + that2, ok := that.(ProtoFingerprintBounds) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Min != that1.Min { + return false + } + if this.Max != that1.Max { + return false + } + return true +} +func (this *ProtoGapWithBlocks) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*ProtoGapWithBlocks) + if !ok { + that2, ok := that.(ProtoGapWithBlocks) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if !this.Bounds.Equal(&that1.Bounds) { + return false + } + if len(this.BlockRef) != len(that1.BlockRef) { + return false + } + for i := range this.BlockRef { + if this.BlockRef[i] != that1.BlockRef[i] { + return false + } + } + return true +} +func (this *ProtoTask) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*ProtoTask) + if !ok { + that2, ok := that.(ProtoTask) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Id != that1.Id { + return false + } + if this.Table != that1.Table { + return false + } + if this.Tenant != that1.Tenant { + return false + } + if !this.Bounds.Equal(&that1.Bounds) { + return false + } + if this.Tsdb != that1.Tsdb { + return false + } + if len(this.Gaps) != len(that1.Gaps) { + return false + } + for i := range this.Gaps { + if !this.Gaps[i].Equal(that1.Gaps[i]) { + return false + } + } + return true +} +func (this *ProtoFingerprintBounds) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 6) + s = append(s, "&protos.ProtoFingerprintBounds{") + s = append(s, "Min: "+fmt.Sprintf("%#v", this.Min)+",\n") + s = append(s, "Max: "+fmt.Sprintf("%#v", this.Max)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} +func (this *ProtoGapWithBlocks) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 6) + s = append(s, "&protos.ProtoGapWithBlocks{") + s = append(s, "Bounds: "+strings.Replace(this.Bounds.GoString(), `&`, ``, 1)+",\n") + s = append(s, "BlockRef: "+fmt.Sprintf("%#v", this.BlockRef)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} +func (this *ProtoTask) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 10) + s = append(s, "&protos.ProtoTask{") + s = append(s, "Id: "+fmt.Sprintf("%#v", this.Id)+",\n") + s = append(s, "Table: "+fmt.Sprintf("%#v", this.Table)+",\n") + s = append(s, "Tenant: "+fmt.Sprintf("%#v", this.Tenant)+",\n") + s = append(s, "Bounds: "+strings.Replace(this.Bounds.GoString(), `&`, ``, 1)+",\n") + s = append(s, "Tsdb: "+fmt.Sprintf("%#v", this.Tsdb)+",\n") + if this.Gaps != nil { + s = append(s, "Gaps: "+fmt.Sprintf("%#v", this.Gaps)+",\n") + } + s = append(s, "}") + return strings.Join(s, "") +} +func valueToGoStringTypes(v interface{}, typ string) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) +} +func (m *ProtoFingerprintBounds) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ProtoFingerprintBounds) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ProtoFingerprintBounds) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Max != 0 { + i = encodeVarintTypes(dAtA, i, uint64(m.Max)) + i-- + dAtA[i] = 0x10 + } + if m.Min != 0 { + i = encodeVarintTypes(dAtA, i, uint64(m.Min)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *ProtoGapWithBlocks) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ProtoGapWithBlocks) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ProtoGapWithBlocks) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.BlockRef) > 0 { + for iNdEx := len(m.BlockRef) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.BlockRef[iNdEx]) + copy(dAtA[i:], m.BlockRef[iNdEx]) + i = encodeVarintTypes(dAtA, i, uint64(len(m.BlockRef[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + { + size, err := m.Bounds.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintTypes(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + +func (m *ProtoTask) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ProtoTask) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ProtoTask) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Gaps) > 0 { + for iNdEx := len(m.Gaps) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Gaps[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintTypes(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x32 + } + } + if len(m.Tsdb) > 0 { + i -= len(m.Tsdb) + copy(dAtA[i:], m.Tsdb) + i = encodeVarintTypes(dAtA, i, uint64(len(m.Tsdb))) + i-- + dAtA[i] = 0x2a + } + { + size, err := m.Bounds.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintTypes(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + if len(m.Tenant) > 0 { + i -= len(m.Tenant) + copy(dAtA[i:], m.Tenant) + i = encodeVarintTypes(dAtA, i, uint64(len(m.Tenant))) + i-- + dAtA[i] = 0x1a + } + if len(m.Table) > 0 { + i -= len(m.Table) + copy(dAtA[i:], m.Table) + i = encodeVarintTypes(dAtA, i, uint64(len(m.Table))) + i-- + dAtA[i] = 0x12 + } + if len(m.Id) > 0 { + i -= len(m.Id) + copy(dAtA[i:], m.Id) + i = encodeVarintTypes(dAtA, i, uint64(len(m.Id))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func encodeVarintTypes(dAtA []byte, offset int, v uint64) int { + offset -= sovTypes(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *ProtoFingerprintBounds) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Min != 0 { + n += 1 + sovTypes(uint64(m.Min)) + } + if m.Max != 0 { + n += 1 + sovTypes(uint64(m.Max)) + } + return n +} + +func (m *ProtoGapWithBlocks) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = m.Bounds.Size() + n += 1 + l + sovTypes(uint64(l)) + if len(m.BlockRef) > 0 { + for _, s := range m.BlockRef { + l = len(s) + n += 1 + l + sovTypes(uint64(l)) + } + } + return n +} + +func (m *ProtoTask) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Id) + if l > 0 { + n += 1 + l + sovTypes(uint64(l)) + } + l = len(m.Table) + if l > 0 { + n += 1 + l + sovTypes(uint64(l)) + } + l = len(m.Tenant) + if l > 0 { + n += 1 + l + sovTypes(uint64(l)) + } + l = m.Bounds.Size() + n += 1 + l + sovTypes(uint64(l)) + l = len(m.Tsdb) + if l > 0 { + n += 1 + l + sovTypes(uint64(l)) + } + if len(m.Gaps) > 0 { + for _, e := range m.Gaps { + l = e.Size() + n += 1 + l + sovTypes(uint64(l)) + } + } + return n +} + +func sovTypes(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozTypes(x uint64) (n int) { + return sovTypes(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (this *ProtoFingerprintBounds) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ProtoFingerprintBounds{`, + `Min:` + fmt.Sprintf("%v", this.Min) + `,`, + `Max:` + fmt.Sprintf("%v", this.Max) + `,`, + `}`, + }, "") + return s +} +func (this *ProtoGapWithBlocks) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ProtoGapWithBlocks{`, + `Bounds:` + strings.Replace(strings.Replace(this.Bounds.String(), "ProtoFingerprintBounds", "ProtoFingerprintBounds", 1), `&`, ``, 1) + `,`, + `BlockRef:` + fmt.Sprintf("%v", this.BlockRef) + `,`, + `}`, + }, "") + return s +} +func (this *ProtoTask) String() string { + if this == nil { + return "nil" + } + repeatedStringForGaps := "[]*ProtoGapWithBlocks{" + for _, f := range this.Gaps { + repeatedStringForGaps += strings.Replace(f.String(), "ProtoGapWithBlocks", "ProtoGapWithBlocks", 1) + "," + } + repeatedStringForGaps += "}" + s := strings.Join([]string{`&ProtoTask{`, + `Id:` + fmt.Sprintf("%v", this.Id) + `,`, + `Table:` + fmt.Sprintf("%v", this.Table) + `,`, + `Tenant:` + fmt.Sprintf("%v", this.Tenant) + `,`, + `Bounds:` + strings.Replace(strings.Replace(this.Bounds.String(), "ProtoFingerprintBounds", "ProtoFingerprintBounds", 1), `&`, ``, 1) + `,`, + `Tsdb:` + fmt.Sprintf("%v", this.Tsdb) + `,`, + `Gaps:` + repeatedStringForGaps + `,`, + `}`, + }, "") + return s +} +func valueToStringTypes(v interface{}) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("*%v", pv) +} +func (m *ProtoFingerprintBounds) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ProtoFingerprintBounds: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ProtoFingerprintBounds: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Min", wireType) + } + m.Min = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Min |= github_com_prometheus_common_model.Fingerprint(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Max", wireType) + } + m.Max = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Max |= github_com_prometheus_common_model.Fingerprint(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipTypes(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ProtoGapWithBlocks) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ProtoGapWithBlocks: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ProtoGapWithBlocks: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Bounds", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Bounds.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BlockRef", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.BlockRef = append(m.BlockRef, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipTypes(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ProtoTask) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ProtoTask: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ProtoTask: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Id", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Id = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Table", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Table = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Tenant", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Tenant = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Bounds", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Bounds.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Tsdb", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Tsdb = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Gaps", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Gaps = append(m.Gaps, &ProtoGapWithBlocks{}) + if err := m.Gaps[len(m.Gaps)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipTypes(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipTypes(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowTypes + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowTypes + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowTypes + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthTypes + } + iNdEx += length + if iNdEx < 0 { + return 0, ErrInvalidLengthTypes + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowTypes + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipTypes(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + if iNdEx < 0 { + return 0, ErrInvalidLengthTypes + } + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthTypes = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowTypes = fmt.Errorf("proto: integer overflow") +) diff --git a/pkg/bloombuild/protos/types.proto b/pkg/bloombuild/protos/types.proto new file mode 100644 index 000000000000..58ea8ee4679e --- /dev/null +++ b/pkg/bloombuild/protos/types.proto @@ -0,0 +1,45 @@ +syntax = "proto3"; + +package protos; + +import "gogoproto/gogo.proto"; + +option go_package = "protos"; +option (gogoproto.marshaler_all) = true; +option (gogoproto.unmarshaler_all) = true; + +// FPBounds is identical to the definition in `pkg/storage/bloom/v1/bounds.FingerprintBounds` +// which ensures we can cast between them without allocations. +// TODO(salvacorts): Reuse from `pkg/logproto/indexgateway.proto` +message ProtoFingerprintBounds { + uint64 min = 1 [ + (gogoproto.casttype) = "github.com/prometheus/common/model.Fingerprint", + (gogoproto.jsontag) = "min" + ]; + uint64 max = 2 [ + (gogoproto.casttype) = "github.com/prometheus/common/model.Fingerprint", + (gogoproto.jsontag) = "max" + ]; +} + +message ProtoGapWithBlocks { + ProtoFingerprintBounds bounds = 1 [ + (gogoproto.nullable) = false, + (gogoproto.jsontag) = "bounds" + ]; + repeated string blockRef = 2; +} + +// TODO: Define BlockRef and SingleTenantTSDBIdentifier as messages so we can use them right away +// instead of unmarshaling them from strings or doing unsafe casts. +message ProtoTask { + string id = 1; + string table = 2; + string tenant = 3; + ProtoFingerprintBounds bounds = 4 [ + (gogoproto.nullable) = false, + (gogoproto.jsontag) = "bounds" + ]; + string tsdb = 5; + repeated ProtoGapWithBlocks gaps = 6; +} From 2171f6409f7157888df9637a635664c67b7ca844 Mon Sep 17 00:00:00 2001 From: benclive Date: Tue, 28 May 2024 09:47:34 +0100 Subject: [PATCH 24/41] fix: Fix panic on requesting out-of-order Pattern samples (#13010) --- pkg/pattern/drain/chunk.go | 8 ++++++++ pkg/pattern/drain/chunk_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/pkg/pattern/drain/chunk.go b/pkg/pattern/drain/chunk.go index 9b1e34e2e3a1..133329946758 100644 --- a/pkg/pattern/drain/chunk.go +++ b/pkg/pattern/drain/chunk.go @@ -66,6 +66,11 @@ func (c Chunk) ForRange(start, end, step model.Time) []logproto.PatternSample { return c.Samples[i].Timestamp >= end }) } + + if c.Samples[lo].Timestamp > c.Samples[hi-1].Timestamp { + return nil + } + if step == TimeResolution { return c.Samples[lo:hi] } @@ -110,6 +115,9 @@ func (c *Chunks) Add(ts model.Time) { *c = append(*c, newChunk(t)) return } + if ts.Before(last.Samples[len(last.Samples)-1].Timestamp) { + return + } last.Samples = append(last.Samples, logproto.PatternSample{ Timestamp: t, Value: 1, diff --git a/pkg/pattern/drain/chunk_test.go b/pkg/pattern/drain/chunk_test.go index 4863a6629729..17429da594e1 100644 --- a/pkg/pattern/drain/chunk_test.go +++ b/pkg/pattern/drain/chunk_test.go @@ -21,6 +21,9 @@ func TestAdd(t *testing.T) { cks.Add(model.TimeFromUnixNano(time.Hour.Nanoseconds()) + TimeResolution + 1) require.Equal(t, 2, len(cks)) require.Equal(t, 1, len(cks[1].Samples)) + cks.Add(model.TimeFromUnixNano(time.Hour.Nanoseconds()) - TimeResolution) + require.Equal(t, 2, len(cks)) + require.Equalf(t, 1, len(cks[1].Samples), "Older samples should not be added if they arrive out of order") } func TestIterator(t *testing.T) { @@ -52,6 +55,7 @@ func TestForRange(t *testing.T) { c *Chunk start model.Time end model.Time + step model.Time expected []logproto.PatternSample }{ { @@ -180,6 +184,28 @@ func TestForRange(t *testing.T) { {Timestamp: 4, Value: 10}, }, }, + { + name: "Out-of-order samples generate nil result", + c: &Chunk{Samples: []logproto.PatternSample{ + {Timestamp: 5, Value: 2}, + {Timestamp: 3, Value: 2}, + }}, + start: 4, + end: 6, + expected: nil, + }, + { + name: "Internally out-of-order samples generate nil result", + c: &Chunk{Samples: []logproto.PatternSample{ + {Timestamp: 1, Value: 2}, + {Timestamp: 5, Value: 2}, + {Timestamp: 3, Value: 2}, + {Timestamp: 7, Value: 2}, + }}, + start: 2, + end: 6, + expected: nil, + }, } for _, tc := range testCases { From 1432a3e84a7e5df18b8dc0e217121fd78da9e75e Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 28 May 2024 11:44:48 -0400 Subject: [PATCH 25/41] feat: Added video and updated Grafana Agent -> Alloy (#13032) Co-authored-by: J Stickler --- docs/sources/send-data/_index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/sources/send-data/_index.md b/docs/sources/send-data/_index.md index 4fd5f7681ea5..2064860dbbcd 100644 --- a/docs/sources/send-data/_index.md +++ b/docs/sources/send-data/_index.md @@ -12,6 +12,8 @@ weight: 500 There are a number of different clients available to send log data to Loki. While all clients can be used simultaneously to cover multiple use cases, which client is initially picked to send logs depends on your use case. +{{< youtube id="xtEppndO7F8" >}} + ## Grafana Clients The following clients are developed and supported (for those customers who have purchased a support contract) by Grafana Labs for sending logs to Loki: From d6374bc2ce3041005842edd353a3bb010f467abe Mon Sep 17 00:00:00 2001 From: Christian Haudum Date: Tue, 28 May 2024 20:09:08 +0200 Subject: [PATCH 26/41] feat(blooms): Add counter metric for blocks that are not available at query time (#12968) When filtering chunks on the bloom gateway, bloom block may not be available and they will be downloaded asynchronously in the background. This new metric `loki_bloom_gateway_blocks_not_available_total` counts the blocks that are not available at query time. Signed-off-by: Christian Haudum --- pkg/bloomgateway/bloomgateway.go | 19 ++++++++++--------- pkg/bloomgateway/metrics.go | 19 +++++++++++++------ pkg/bloomgateway/processor.go | 2 +- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/pkg/bloomgateway/bloomgateway.go b/pkg/bloomgateway/bloomgateway.go index ee0e6f9940fd..165e2d652473 100644 --- a/pkg/bloomgateway/bloomgateway.go +++ b/pkg/bloomgateway/bloomgateway.go @@ -290,6 +290,13 @@ func (g *Gateway) FilterChunkRefs(ctx context.Context, req *logproto.FilterChunk g.activeUsers.UpdateUserTimestamp(tenantID, time.Now()) + var preFilterSeries, preFilterChunks int + + preFilterSeries = len(req.Refs) + for _, series := range req.Refs { + preFilterChunks += len(series.Refs) + } + // Ideally we could use an unbuffered channel here, but since we return the // request on the first error, there can be cases where the request context // is not done yet and the consumeTask() function wants to send to the @@ -316,13 +323,6 @@ func (g *Gateway) FilterChunkRefs(ctx context.Context, req *logproto.FilterChunk remaining := len(tasks) - preFilterSeries := len(req.Refs) - var preFilterChunks, postFilterChunks int - - for _, series := range req.Refs { - preFilterChunks += len(series.Refs) - } - combinedRecorder := v1.NewBloomRecorder(ctx, "combined") for remaining > 0 { select { @@ -353,11 +353,12 @@ func (g *Gateway) FilterChunkRefs(ctx context.Context, req *logproto.FilterChunk responsesPool.Put(resp) } - postFilterSeries := len(filtered) - + var postFilterSeries, postFilterChunks int + postFilterSeries = len(filtered) for _, group := range filtered { postFilterChunks += len(group.Refs) } + g.metrics.requestedSeries.Observe(float64(preFilterSeries)) g.metrics.filteredSeries.Observe(float64(preFilterSeries - postFilterSeries)) g.metrics.requestedChunks.Observe(float64(preFilterChunks)) diff --git a/pkg/bloomgateway/metrics.go b/pkg/bloomgateway/metrics.go index 0885bc2ae7cb..5c046d3147c3 100644 --- a/pkg/bloomgateway/metrics.go +++ b/pkg/bloomgateway/metrics.go @@ -116,12 +116,13 @@ func newServerMetrics(registerer prometheus.Registerer, namespace, subsystem str } type workerMetrics struct { - dequeueDuration *prometheus.HistogramVec - queueDuration *prometheus.HistogramVec - processDuration *prometheus.HistogramVec - tasksDequeued *prometheus.CounterVec - tasksProcessed *prometheus.CounterVec - blockQueryLatency *prometheus.HistogramVec + dequeueDuration *prometheus.HistogramVec + queueDuration *prometheus.HistogramVec + processDuration *prometheus.HistogramVec + tasksDequeued *prometheus.CounterVec + tasksProcessed *prometheus.CounterVec + blocksNotAvailable *prometheus.CounterVec + blockQueryLatency *prometheus.HistogramVec } func newWorkerMetrics(registerer prometheus.Registerer, namespace, subsystem string) *workerMetrics { @@ -158,6 +159,12 @@ func newWorkerMetrics(registerer prometheus.Registerer, namespace, subsystem str Name: "tasks_processed_total", Help: "Total amount of tasks that the worker processed", }, append(labels, "status")), + blocksNotAvailable: r.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "blocks_not_available_total", + Help: "Total amount of blocks that have been skipped because they were not found or not downloaded yet", + }, labels), blockQueryLatency: r.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: subsystem, diff --git a/pkg/bloomgateway/processor.go b/pkg/bloomgateway/processor.go index 1e8452ded5d6..6973ad1f565b 100644 --- a/pkg/bloomgateway/processor.go +++ b/pkg/bloomgateway/processor.go @@ -126,7 +126,7 @@ func (p *processor) processBlocks(ctx context.Context, bqs []*bloomshipper.Close return concurrency.ForEachJob(ctx, len(bqs), p.concurrency, func(ctx context.Context, i int) error { bq := bqs[i] if bq == nil { - // TODO(chaudum): Add metric for skipped blocks + p.metrics.blocksNotAvailable.WithLabelValues(p.id).Inc() return nil } From 1ab9d271c354caf0ba589691e6477fb9a19039f0 Mon Sep 17 00:00:00 2001 From: archimeid <68751170+archimeid@users.noreply.github.com> Date: Tue, 28 May 2024 19:35:28 +0100 Subject: [PATCH 27/41] fix(helm): fix query-frontend and ruler targetPort 'http-metrics' in Service template (#13024) --- production/helm/loki/CHANGELOG.md | 4 ++++ production/helm/loki/Chart.yaml | 2 +- .../query-frontend/service-query-frontend-headless.yaml | 2 +- production/helm/loki/templates/ruler/service-ruler.yaml | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/production/helm/loki/CHANGELOG.md b/production/helm/loki/CHANGELOG.md index a86a7d028182..1bb316cf3202 100644 --- a/production/helm/loki/CHANGELOG.md +++ b/production/helm/loki/CHANGELOG.md @@ -13,6 +13,10 @@ Entries should include a reference to the pull request that introduced the chang [//]: # ( : do not remove this line. This locator is used by the CI pipeline to automatically create a changelog entry for each new Loki release. Add other chart versions and respective changelog entries bellow this line.) +## 6.6.2 + +- [BUGFIX] Fix query-frontend (headless) and ruler http-metrics targetPort + ## 6.6.1 - [BUGFIX] Fix query scheduler http-metrics targetPort diff --git a/production/helm/loki/Chart.yaml b/production/helm/loki/Chart.yaml index d2ec2d3d9e59..47606001b954 100644 --- a/production/helm/loki/Chart.yaml +++ b/production/helm/loki/Chart.yaml @@ -3,7 +3,7 @@ name: loki description: Helm chart for Grafana Loki and Grafana Enterprise Logs supporting both simple, scalable and distributed modes. type: application appVersion: 3.0.0 -version: 6.6.1 +version: 6.6.2 home: https://grafana.github.io/helm-charts sources: - https://github.com/grafana/loki diff --git a/production/helm/loki/templates/query-frontend/service-query-frontend-headless.yaml b/production/helm/loki/templates/query-frontend/service-query-frontend-headless.yaml index 258413aa1d57..b168ce6ce952 100644 --- a/production/helm/loki/templates/query-frontend/service-query-frontend-headless.yaml +++ b/production/helm/loki/templates/query-frontend/service-query-frontend-headless.yaml @@ -22,7 +22,7 @@ spec: ports: - name: http-metrics port: 3100 - targetPort: http + targetPort: http-metrics protocol: TCP - name: grpc port: 9095 diff --git a/production/helm/loki/templates/ruler/service-ruler.yaml b/production/helm/loki/templates/ruler/service-ruler.yaml index 8200af2b69a9..1a1f0f4d2e91 100644 --- a/production/helm/loki/templates/ruler/service-ruler.yaml +++ b/production/helm/loki/templates/ruler/service-ruler.yaml @@ -19,7 +19,7 @@ spec: ports: - name: http-metrics port: 3100 - targetPort: http + targetPort: http-metrics protocol: TCP - name: grpc port: 9095 From 6ec4712aaa12d68e320681694678af881a0f1e35 Mon Sep 17 00:00:00 2001 From: Sandeep Sukhani Date: Wed, 29 May 2024 00:44:15 +0530 Subject: [PATCH 28/41] docs: update otlp ingestion docs to correct some info and add more details (#12969) --- docs/sources/send-data/otel/_index.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/sources/send-data/otel/_index.md b/docs/sources/send-data/otel/_index.md index b7a67fcb14d0..6fa17c317054 100644 --- a/docs/sources/send-data/otel/_index.md +++ b/docs/sources/send-data/otel/_index.md @@ -72,7 +72,7 @@ service: Since the OpenTelemetry protocol differs from the Loki storage model, here is how data in the OpenTelemetry format will be mapped by default to the Loki data model during ingestion, which can be changed as explained later: -- Index labels: Resource attributes map well to index labels in Loki, since both usually identify the source of the logs. Because Loki has a limit of 30 index labels, we have selected the following resource attributes to be stored as index labels, while the remaining attributes are stored as [Structured Metadata]({{< relref "../../get-started/labels/structured-metadata" >}}) with each log entry: +- Index labels: Resource attributes map well to index labels in Loki, since both usually identify the source of the logs. The default list of Resource Attributes to store as Index labels can be configured using `default_resource_attributes_as_index_labels` under [distributor's otlp_config](https://grafana.com/docs/loki//configure/#distributor). By default, the following resource attributes will be stored as index labels, while the remaining attributes are stored as [Structured Metadata]({{< relref "../../get-started/labels/structured-metadata" >}}) with each log entry: - cloud.availability_zone - cloud.region - container.name @@ -91,6 +91,10 @@ Since the OpenTelemetry protocol differs from the Loki storage model, here is ho - service.name - service.namespace + {{% admonition type="note" %}} + Because Loki has a default limit of 15 index labels, we recommend storing only select resource attributes as index labels. Although the default config selects more than 15 Resource Attributes, it should be fine since a few are mutually exclusive. + {{% /admonition %}} + - Timestamp: One of `LogRecord.TimeUnixNano` or `LogRecord.ObservedTimestamp`, based on which one is set. If both are not set, the ingestion timestamp will be used. - LogLine: `LogRecord.Body` holds the body of the log. However, since Loki only supports Log body in string format, we will stringify non-string values using the [AsString method from the OTEL collector lib](https://github.com/open-telemetry/opentelemetry-collector/blob/ab3d6c5b64701e690aaa340b0a63f443ff22c1f0/pdata/pcommon/value.go#L353). From 74a5bbc5caa3cea306aa7047b73fb81738d80872 Mon Sep 17 00:00:00 2001 From: J Stickler Date: Tue, 28 May 2024 15:54:56 -0400 Subject: [PATCH 29/41] docs: Update Grafana Agent to Grafana Alloy (#12602) --- .../get-started/labels/structured-metadata.md | 4 +-- docs/sources/get-started/overview.md | 2 +- docs/sources/operations/loki-canary/_index.md | 2 +- docs/sources/release-notes/v3.0.md | 4 ++- docs/sources/send-data/_index.md | 26 +++++++++++-------- docs/sources/send-data/k6/log-generation.md | 4 +-- docs/sources/send-data/otel/_index.md | 8 +++--- docs/sources/send-data/promtail/_index.md | 4 +++ .../send-data/promtail/installation.md | 4 +++ docs/sources/setup/install/helm/concepts.md | 2 +- .../migrate/migrate-from-distributed/index.md | 2 +- .../setup/migrate/migrate-to-alloy/_index.md | 25 ++++++++++++++++++ .../setup/migrate/migrate-to-tsdb/_index.md | 2 +- 13 files changed, 64 insertions(+), 25 deletions(-) create mode 100644 docs/sources/setup/migrate/migrate-to-alloy/_index.md diff --git a/docs/sources/get-started/labels/structured-metadata.md b/docs/sources/get-started/labels/structured-metadata.md index 99f46f708792..91fe5d80ab67 100644 --- a/docs/sources/get-started/labels/structured-metadata.md +++ b/docs/sources/get-started/labels/structured-metadata.md @@ -21,7 +21,7 @@ Structured metadata can also be used to query commonly needed metadata from log You should only use structured metadata in the following situations: -- If you are ingesting data in OpenTelemetry format, using the Grafana Agent or an OpenTelemetry Collector. Structured metadata was designed to support native ingestion of OpenTelemetry data. +- If you are ingesting data in OpenTelemetry format, using Grafana Alloy or an OpenTelemetry Collector. Structured metadata was designed to support native ingestion of OpenTelemetry data. - If you have high cardinality metadata that should not be used as a label and does not exist in the log line. Some examples might include `process_id` or `thread_id` or Kubernetes pod names. It is an antipattern to extract information that already exists in your log lines and put it into structured metadata. @@ -31,7 +31,7 @@ It is an antipattern to extract information that already exists in your log line You have the option to attach structured metadata to log lines in the push payload along with each log line and the timestamp. For more information on how to push logs to Loki via the HTTP endpoint, refer to the [HTTP API documentation](https://grafana.com/docs/loki//reference/api/#ingest-logs). -Alternatively, you can use the Grafana Agent or Promtail to extract and attach structured metadata to your log lines. +Alternatively, you can use Grafana Alloy or Promtail to extract and attach structured metadata to your log lines. See the [Promtail: Structured metadata stage](https://grafana.com/docs/loki//send-data/promtail/stages/structured_metadata/) for more information. With Loki version 1.2.0, support for structured metadata has been added to the Logstash output plugin. For more information, see [logstash](https://grafana.com/docs/loki//send-data/logstash/). diff --git a/docs/sources/get-started/overview.md b/docs/sources/get-started/overview.md index 4051ba63cc11..1194398c38f0 100644 --- a/docs/sources/get-started/overview.md +++ b/docs/sources/get-started/overview.md @@ -22,7 +22,7 @@ Log data is then compressed and stored in chunks in an object store such as Amaz A typical Loki-based logging stack consists of 3 components: -- **Agent** - An agent or client, for example Promtail, which is distributed with Loki, or the Grafana Agent. The agent scrapes logs, turns the logs into streams by adding labels, and pushes the streams to Loki through an HTTP API. +- **Agent** - An agent or client, for example Grafana Alloy, or Promtail, which is distributed with Loki. The agent scrapes logs, turns the logs into streams by adding labels, and pushes the streams to Loki through an HTTP API. - **Loki** - The main server, responsible for ingesting and storing logs and processing queries. It can be deployed in three different configurations, for more information see [deployment modes]({{< relref "../get-started/deployment-modes" >}}). diff --git a/docs/sources/operations/loki-canary/_index.md b/docs/sources/operations/loki-canary/_index.md index cf2a1075d3c0..f6c1bf23a938 100644 --- a/docs/sources/operations/loki-canary/_index.md +++ b/docs/sources/operations/loki-canary/_index.md @@ -29,7 +29,7 @@ array. The contents look something like this: The relevant part of the log entry is the timestamp; the `p`s are just filler bytes to make the size of the log configurable. -An agent (like Promtail) should be configured to read the log file and ship it +An agent (like Grafana Alloy) should be configured to read the log file and ship it to Loki. Meanwhile, Loki Canary will open a WebSocket connection to Loki and will tail diff --git a/docs/sources/release-notes/v3.0.md b/docs/sources/release-notes/v3.0.md index a44483d57d2f..ea3c7603ff82 100644 --- a/docs/sources/release-notes/v3.0.md +++ b/docs/sources/release-notes/v3.0.md @@ -20,7 +20,7 @@ Key features in Loki 3.0.0 include the following: - **Query acceleration with Bloom filters** (experimental): This is designed to speed up filter queries, with best results for queries that are looking for a specific text string like an error message or UUID. For more information, refer to [Query acceleration with Blooms](https://grafana.com/docs/loki//operations/query-acceleration-blooms/). -- **Native OpenTelemetry Support**: A simplified ingestion pipeline (Loki Exporter no longer needed) and a more intuitive query experience for OTel logs. For more information, refer to the [OTEL documentation](https://grafana.com/docs/loki//send-data/otel/). +- **Native OpenTelemetry Support**: A simplified ingestion pipeline (Loki Exporter no longer needed) and a more intuitive query experience for OTel logs. For more information, refer to the [OTel documentation](https://grafana.com/docs/loki//send-data/otel/). - **Helm charts**: A major upgrade to the Loki helm chart introduces support for `Distributed` mode (also known as [microservices](https://grafana.com/docs/loki//get-started/deployment-modes/#microservices-mode) mode), includes memcached by default, and includes several updates to configurations to improve Loki operations. @@ -46,6 +46,8 @@ One of the focuses of Loki 3.0 was cleaning up unused code and old features that To learn more about breaking changes in this release, refer to the [Upgrade guide](https://grafana.com/docs/loki//setup/upgrade/). +{{< docs/shared source="alloy" lookup="agent-deprecation.md" version="next" >}} + ## Upgrade Considerations The path from 2.9 to 3.0 includes several breaking changes. For important upgrade guidance, refer to the [Upgrade Guide](https://grafana.com/docs/loki//setup/upgrade/) and the separate [Helm Upgrade Guide](https://grafana.com/docs/loki//setup/upgrade/upgrade-to-6x/). diff --git a/docs/sources/send-data/_index.md b/docs/sources/send-data/_index.md index 2064860dbbcd..0ef9432d3caf 100644 --- a/docs/sources/send-data/_index.md +++ b/docs/sources/send-data/_index.md @@ -18,16 +18,20 @@ While all clients can be used simultaneously to cover multiple use cases, which The following clients are developed and supported (for those customers who have purchased a support contract) by Grafana Labs for sending logs to Loki: -- [Grafana Agent](/docs/agent/latest/) - The Grafana Agent is the recommended client for the Grafana stack. It can collect telemetry data for metrics, logs, traces, and continuous profiles and is fully compatible with the Prometheus, OpenTelemetry, and Grafana open source ecosystems. -- [Promtail]({{< relref "./promtail" >}}) - Promtail is the client of choice when you're running Kubernetes, as you can configure it to automatically scrape logs from pods running on the same node that Promtail runs on. Promtail and Prometheus running together in Kubernetes enables powerful debugging: if Prometheus and Promtail use the same labels, users can use tools like Grafana to switch between metrics and logs based on the label set. -Promtail is also the client of choice on bare-metal since it can be configured to tail logs from all files given a host path. It is the easiest way to send logs to Loki from plain-text files (for example, things that log to `/var/log/*.log`). -Lastly, Promtail works well if you want to extract metrics from logs such as counting the occurrences of a particular message. -- [xk6-loki extension](https://github.com/grafana/xk6-loki) - The k6-loki extension lets you perform [load testing on Loki]({{< relref "./k6" >}}). +- [Grafana Alloy](https://grafana.com/docs/alloy/latest/) - Grafana Alloy is a vendor-neutral distribution of the OpenTelemetry (OTel) Collector. Alloy offers native pipelines for OTel, Prometheus, Pyroscope, Loki, and many other metrics, logs, traces, and profile tools. In addition, you can use Alloy pipelines to do different tasks, such as configure alert rules in Loki and Mimir. Alloy is fully compatible with the OTel Collector, Prometheus Agent, and Promtail. You can use Alloy as an alternative to either of these solutions or combine it into a hybrid system of multiple collectors and agents. You can deploy Alloy anywhere within your IT infrastructure and pair it with your Grafana LGTM stack, a telemetry backend from Grafana Cloud, or any other compatible backend from any other vendor. + {{< docs/shared source="alloy" lookup="agent-deprecation.md" version="next" >}} +- [Grafana Agent](/docs/agent/latest/) - The Grafana Agent is a client for the Grafana stack. It can collect telemetry data for metrics, logs, traces, and continuous profiles and is fully compatible with the Prometheus, OpenTelemetry, and Grafana open source ecosystems. +- [Promtail](https://grafana.com/docs/loki//send-data/promtail/) - Promtail can be configured to automatically scrape logs from Kubernetes pods running on the same node that Promtail runs on. Promtail and Prometheus running together in Kubernetes enables powerful debugging: if Prometheus and Promtail use the same labels, users can use tools like Grafana to switch between metrics and logs based on the label set. Promtail can be configured to tail logs from all files given a host path. It is the easiest way to send logs to Loki from plain-text files (for example, things that log to `/var/log/*.log`). +Promtail works well if you want to extract metrics from logs such as counting the occurrences of a particular message. +{{< admonition type="note" >}} +Promtail is feature complete. All future feature development will occur in Grafana Alloy. +{{< /admonition >}} +- [xk6-loki extension](https://github.com/grafana/xk6-loki) - The k6-loki extension lets you perform [load testing on Loki](https://grafana.com/docs/loki//send-data/k6/). ## OpenTelemetry Collector Loki natively supports ingesting OpenTelemetry logs over HTTP. -See [Ingesting logs to Loki using OpenTelemetry Collector]({{< relref "./otel" >}}) for more details. +For more information, see [Ingesting logs to Loki using OpenTelemetry Collector](https://grafana.com/docs/loki//send-data/otel/). ## Third-party clients @@ -39,14 +43,14 @@ Grafana Labs cannot provide support for third-party clients. Once an issue has b The following are popular third-party Loki clients: -- [Docker Driver]({{< relref "./docker-driver" >}}) - When using Docker and not Kubernetes, the Docker logging driver for Loki should +- [Docker Driver](https://grafana.com/docs/loki//send-data/docker-driver/) - When using Docker and not Kubernetes, the Docker logging driver for Loki should be used as it automatically adds labels appropriate to the running container. -- [Fluent Bit]({{< relref "./fluentbit" >}}) - The Fluent Bit plugin is ideal when you already have Fluentd deployed +- [Fluent Bit](https://grafana.com/docs/loki//send-data/fluentbit/) - The Fluent Bit plugin is ideal when you already have Fluentd deployed and you already have configured `Parser` and `Filter` plugins. -- [Fluentd]({{< relref "./fluentd" >}}) - The Fluentd plugin is ideal when you already have Fluentd deployed +- [Fluentd](https://grafana.com/docs/loki//send-data/fluentd/) - The Fluentd plugin is ideal when you already have Fluentd deployed and you already have configured `Parser` and `Filter` plugins. Fluentd also works well for extracting metrics from logs when using itsPrometheus plugin. -- [Lambda Promtail]({{< relref "./lambda-promtail" >}}) - This is a workflow combining the Promtail push-api [scrape config]({{< relref "./promtail/configuration#loki_push_api" >}}) and the [lambda-promtail]({{< relref "./lambda-promtail" >}}) AWS Lambda function which pipes logs from Cloudwatch to Loki. This is a good choice if you're looking to try out Loki in a low-footprint way or if you wish to monitor AWS lambda logs in Loki -- [Logstash]({{< relref "./logstash" >}}) - If you are already using logstash and/or beats, this will be the easiest way to start. +- [Lambda Promtail](https://grafana.com/docs/loki//send-data/lambda-promtail/) - This is a workflow combining the Promtail push-api [scrape config](https://grafana.com/docs/loki//send-data/promtail/configuration/#loki_push_api) and the lambda-promtail AWS Lambda function which pipes logs from Cloudwatch to Loki. This is a good choice if you're looking to try out Loki in a low-footprint way or if you wish to monitor AWS lambda logs in Loki +- [Logstash](https://grafana.com/docs/loki//send-data/logstash/) - If you are already using logstash and/or beats, this will be the easiest way to start. By adding our output plugin you can quickly try Loki without doing big configuration changes. These third-party clients also enable sending logs to Loki: diff --git a/docs/sources/send-data/k6/log-generation.md b/docs/sources/send-data/k6/log-generation.md index 635f042f90b8..8ad79309191b 100644 --- a/docs/sources/send-data/k6/log-generation.md +++ b/docs/sources/send-data/k6/log-generation.md @@ -61,8 +61,8 @@ export default () => { The second and third argument of the method take the lower and upper bound of the batch size. The resulting batch size is a random value between the two -arguments. This mimics the behaviour of a log client, such as Promtail or -the Grafana Agent, where logs are buffered and pushed once a certain batch size +arguments. This mimics the behavior of a log client, such as Grafana Alloy or Promtail, +where logs are buffered and pushed once a certain batch size is reached or after a certain size when no logs have been received. The batch size is not equal to the payload size, as the batch size only counts diff --git a/docs/sources/send-data/otel/_index.md b/docs/sources/send-data/otel/_index.md index 6fa17c317054..4b28cbf16c7c 100644 --- a/docs/sources/send-data/otel/_index.md +++ b/docs/sources/send-data/otel/_index.md @@ -1,6 +1,6 @@ --- title: Ingesting logs to Loki using OpenTelemetry Collector -menuTitle: OTEL Collector +menuTitle: OTel Collector description: Configuring the OpenTelemetry Collector to send logs to Loki. aliases: - ../clients/k6/ @@ -97,7 +97,7 @@ Since the OpenTelemetry protocol differs from the Loki storage model, here is ho - Timestamp: One of `LogRecord.TimeUnixNano` or `LogRecord.ObservedTimestamp`, based on which one is set. If both are not set, the ingestion timestamp will be used. -- LogLine: `LogRecord.Body` holds the body of the log. However, since Loki only supports Log body in string format, we will stringify non-string values using the [AsString method from the OTEL collector lib](https://github.com/open-telemetry/opentelemetry-collector/blob/ab3d6c5b64701e690aaa340b0a63f443ff22c1f0/pdata/pcommon/value.go#L353). +- LogLine: `LogRecord.Body` holds the body of the log. However, since Loki only supports Log body in string format, we will stringify non-string values using the [AsString method from the OTel collector lib](https://github.com/open-telemetry/opentelemetry-collector/blob/ab3d6c5b64701e690aaa340b0a63f443ff22c1f0/pdata/pcommon/value.go#L353). - [Structured Metadata]({{< relref "../../get-started/labels/structured-metadata" >}}): Anything which can’t be stored in Index labels and LogLine would be stored as Structured Metadata. Here is a non-exhaustive list of what will be stored in Structured Metadata to give a sense of what it will hold: - Resource Attributes not stored as Index labels is replicated and stored with each log entry. @@ -109,7 +109,7 @@ Things to note before ingesting OpenTelemetry logs to Loki: - Dots (.) are converted to underscores (_). Loki does not support `.` or any other special characters other than `_` in label names. The unsupported characters are replaced with an `_` while converting Attributes to Index Labels or Structured Metadata. - Also, please note that while writing the queries, you must use the normalized format, i.e. use `_` instead of special characters while querying data using OTEL Attributes. + Also, please note that while writing the queries, you must use the normalized format, i.e. use `_` instead of special characters while querying data using OTel Attributes. For example, `service.name` in OTLP would become `service_name` in Loki. @@ -120,7 +120,7 @@ Things to note before ingesting OpenTelemetry logs to Loki: - Stringification of non-string Attribute values - While converting Attribute values in OTLP to Index label values or Structured Metadata, any non-string values are converted to string using [AsString method from the OTEL collector lib](https://github.com/open-telemetry/opentelemetry-collector/blob/ab3d6c5b64701e690aaa340b0a63f443ff22c1f0/pdata/pcommon/value.go#L353). + While converting Attribute values in OTLP to Index label values or Structured Metadata, any non-string values are converted to string using [AsString method from the OTel collector lib](https://github.com/open-telemetry/opentelemetry-collector/blob/ab3d6c5b64701e690aaa340b0a63f443ff22c1f0/pdata/pcommon/value.go#L353). ### Changing the default mapping of OTLP to Loki Format diff --git a/docs/sources/send-data/promtail/_index.md b/docs/sources/send-data/promtail/_index.md index 7e560e661438..03bbf6487c39 100644 --- a/docs/sources/send-data/promtail/_index.md +++ b/docs/sources/send-data/promtail/_index.md @@ -12,6 +12,10 @@ Promtail is an agent which ships the contents of local logs to a private Grafana instance or [Grafana Cloud](/oss/loki). It is usually deployed to every machine that runs applications which need to be monitored. +{{< admonition type="note" >}} +Promtail is feature complete. All future feature development will occur in Grafana Alloy. +{{< /admonition >}} + It primarily: - Discovers targets diff --git a/docs/sources/send-data/promtail/installation.md b/docs/sources/send-data/promtail/installation.md index 4d2359e94c17..25a818458a80 100644 --- a/docs/sources/send-data/promtail/installation.md +++ b/docs/sources/send-data/promtail/installation.md @@ -9,6 +9,10 @@ weight: 100 # Install Promtail +{{< admonition type="note" >}} +Promtail is feature complete. All future feature development will occur in Grafana Alloy. +{{< /admonition >}} + Promtail is distributed as a binary, in a Docker container, or there is a Helm chart to install it in a Kubernetes cluster. diff --git a/docs/sources/setup/install/helm/concepts.md b/docs/sources/setup/install/helm/concepts.md index fd8f81ebe474..581498af89b2 100644 --- a/docs/sources/setup/install/helm/concepts.md +++ b/docs/sources/setup/install/helm/concepts.md @@ -21,7 +21,7 @@ By default Loki will be installed in the scalable mode. This consists of a read ## Dashboards -This chart includes dashboards for monitoring Loki. These require the scrape configs defined in the `monitoring.serviceMonitor` and `monitoring.selfMonitoring` sections described below. The dashboards are deployed via a config map which can be mounted on a Grafana instance. The Dashboard require an installation of the Grafana Agent and the Prometheus operator. The agent is installed with this chart. +This chart includes dashboards for monitoring Loki. These require the scrape configs defined in the `monitoring.serviceMonitor` and `monitoring.selfMonitoring` sections described below. The dashboards are deployed via a config map which can be mounted on a Grafana instance. The Dashboard requires an installation of the Grafana Agent and the Prometheus operator. The agent is installed with this chart. ## Canary diff --git a/docs/sources/setup/migrate/migrate-from-distributed/index.md b/docs/sources/setup/migrate/migrate-from-distributed/index.md index 1618716fd26e..01b016b8a937 100644 --- a/docs/sources/setup/migrate/migrate-from-distributed/index.md +++ b/docs/sources/setup/migrate/migrate-from-distributed/index.md @@ -48,7 +48,7 @@ This leverages the fact that the new deployment adds a `app.kubernetes.io/compon Once the new cluster is up, add the appropriate data source in Grafana for the new cluster. Check that the following queries return results: - Confirm new and old logs are in the new deployment. Using the new deployment's Loki data source in Grafana, look for: - - Logs with a job that is unqiue to your existing Promtail or Grafana Agent, the one we adjusted above to exclude logs from the new deployment which is not yet pushing logs to the new deployment. If you can query those via the new deployment in shows we have not lost historical logs. + - Logs with a job that is unique to your existing Promtail or Grafana Agent, the one we adjusted above to exclude logs from the new deployment which is not yet pushing logs to the new deployment. If you can query those via the new deployment in shows we have not lost historical logs. - Logs with the label `job="loki/loki-read"`. The read component does not exist in `loki-distributed`, so this show the new Loki cluster's self monitoring is working correctly. - Confirm new logs are in the old deployment. Using the old deployment's Loki data source in Grafana, look for: - Logs with the label `job="loki/loki-read"`. Since you have excluded logs from the new deployment from going to the `loki-distributed` deployment, if you can query them through the `loki-distributed` Loki data source that show the ingesters have joined the same ring, and are queryable from the `loki-distributed` queriers. diff --git a/docs/sources/setup/migrate/migrate-to-alloy/_index.md b/docs/sources/setup/migrate/migrate-to-alloy/_index.md new file mode 100644 index 000000000000..adb4dac9b3a1 --- /dev/null +++ b/docs/sources/setup/migrate/migrate-to-alloy/_index.md @@ -0,0 +1,25 @@ +--- +title: Migrate to Alloy +description: Provides links to documentation to migrate to Grafana Alloy. +weight: 100 +--- + +# Migrate to Alloy + +Grafana Alloy is the new name for the Grafana Labs distribution of the OpenTelemetry collector. Grafana Agent Static, Grafana Agent Flow, and Grafana Agent Operator have been deprecated and are in Long-Term Support (LTS) through October 31, 2025. They will reach an End-of-Life (EOL) on November 1, 2025. Grafana Labs has provided tools and migration documentation to assist you in migrating to Grafana Alloy. + +Read more about why we recommend migrating to [Grafana Alloy](https://grafana.com/blog/2024/04/09/grafana-alloy-opentelemetry-collector-with-prometheus-pipelines/). + +This section provides links to documentation for how to migrate to Alloy. + +- [Migrate from Grafana Agent Static](https://grafana.com/docs/alloy/latest/tasks/migrate/from-static/) + +- [Migrate from Grafana Agent Flow](https://grafana.com/docs/alloy/latest/tasks/migrate/from-flow/) + +- [Migrate from Grafana Agent Operator](https://grafana.com/docs/alloy/latest/tasks/migrate/from-operator/) + +- [Migrate from OpenTelemetry Collector](https://grafana.com/docs/alloy/latest/tasks/migrate/from-otelcol/) + +- [Migrate from Prometheus](https://grafana.com/docs/alloy/latest/tasks/migrate/from-prometheus/) + +- [Migrate from Promtail](https://grafana.com/docs/alloy/latest/tasks/migrate/from-promtail/) diff --git a/docs/sources/setup/migrate/migrate-to-tsdb/_index.md b/docs/sources/setup/migrate/migrate-to-tsdb/_index.md index 963913e21ef9..49ba506dc553 100644 --- a/docs/sources/setup/migrate/migrate-to-tsdb/_index.md +++ b/docs/sources/setup/migrate/migrate-to-tsdb/_index.md @@ -2,7 +2,7 @@ title: Migrate to TSDB menuTitle: Migrate to TSDB description: Migration guide for moving from any of the older indexes to TSDB -weight: 100 +weight: 300 keywords: - migrate - tsdb From 7ffe0fb6490e171e0100cb35ce6fde9377eff237 Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Wed, 29 May 2024 05:13:42 -0400 Subject: [PATCH 30/41] fix: temporarily moving from alloy -> alloy dev (#13062) There is currently a bug within version 1.1.0 of Alloy which causes are getting started demo to fail. 1.20 fixes this but is currently in public preview on dev. --- examples/getting-started/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/getting-started/docker-compose.yaml b/examples/getting-started/docker-compose.yaml index 449fe55f2b6e..5fdf6d066b72 100644 --- a/examples/getting-started/docker-compose.yaml +++ b/examples/getting-started/docker-compose.yaml @@ -46,7 +46,7 @@ services: <<: *loki-dns alloy: - image: grafana/alloy:latest + image: grafana/alloy-dev:latest volumes: - ./alloy-local-config.yaml:/etc/alloy/config.alloy:ro - /var/run/docker.sock:/var/run/docker.sock From d0a285926b7257d54cf948ba644c619a4b49a871 Mon Sep 17 00:00:00 2001 From: benclive Date: Wed, 29 May 2024 10:54:59 +0100 Subject: [PATCH 31/41] feat: Increase drain max depth from 8 -> 30 (#13063) --- pkg/pattern/drain/drain.go | 2 +- pkg/pattern/drain/drain_benchmark_test.go | 78 +++ pkg/pattern/drain/drain_test.go | 578 ++++++++++++---------- 3 files changed, 403 insertions(+), 255 deletions(-) create mode 100644 pkg/pattern/drain/drain_benchmark_test.go diff --git a/pkg/pattern/drain/drain.go b/pkg/pattern/drain/drain.go index 31932832f701..c3076386c4d6 100644 --- a/pkg/pattern/drain/drain.go +++ b/pkg/pattern/drain/drain.go @@ -130,7 +130,7 @@ func DefaultConfig() *Config { // > message are more likely to be constants. Specifically, Drain // > selects the next internal node by the tokens in the beginning // > positions of the log message - LogClusterDepth: 8, + LogClusterDepth: 30, // SimTh is basically a ratio of matching/total in the cluster. // Cluster tokens: "foo <*> bar fred" // Log line: "foo bar baz qux" diff --git a/pkg/pattern/drain/drain_benchmark_test.go b/pkg/pattern/drain/drain_benchmark_test.go new file mode 100644 index 000000000000..cf9378025102 --- /dev/null +++ b/pkg/pattern/drain/drain_benchmark_test.go @@ -0,0 +1,78 @@ +package drain + +import ( + "bufio" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func BenchmarkDrain_TrainExtractsPatterns(b *testing.B) { + tests := []struct { + name string + drain *Drain + inputFile string + }{ + { + name: `Patterns for agent logfmt logs`, + inputFile: `testdata/agent-logfmt.txt`, + }, + { + name: `Patterns for ingester logfmt logs`, + inputFile: `testdata/ingester-logfmt.txt`, + }, + { + name: `Patterns for Drone json logs`, + inputFile: `testdata/drone-json.txt`, + }, + { + name: "Patterns for distributor logfmt logs", + inputFile: "testdata/distributor-logfmt.txt", + }, + { + name: "Patterns for journald logs", + inputFile: "testdata/journald.txt", + }, + { + name: "Patterns for kafka logs", + inputFile: "testdata/kafka.txt", + }, + { + name: "Patterns for kubernetes logs", + inputFile: "testdata/kubernetes.txt", + }, + { + name: "Patterns for vault logs", + inputFile: "testdata/vault.txt", + }, + { + name: "Patterns for calico logs", + inputFile: "testdata/calico.txt", + }, + } + + for _, tt := range tests { + b.Run(tt.name, func(b *testing.B) { + file, err := os.Open(tt.inputFile) + require.NoError(b, err) + defer file.Close() + + scanner := bufio.NewScanner(file) + var lines []string + for scanner.Scan() { + line := scanner.Text() + lines = append(lines, line) + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, line := range lines { + drain := New(DefaultConfig(), nil) + drain.Train(line, 0) + } + } + }) + } +} diff --git a/pkg/pattern/drain/drain_test.go b/pkg/pattern/drain/drain_test.go index e9709aed3fec..754ac54b3fb7 100644 --- a/pkg/pattern/drain/drain_test.go +++ b/pkg/pattern/drain/drain_test.go @@ -2,16 +2,22 @@ package drain import ( "bufio" + "fmt" "os" "testing" "github.com/stretchr/testify/require" + "golang.org/x/exp/slices" "github.com/grafana/loki/v3/pkg/logql/log/pattern" ) func TestDrain_TrainExtractsPatterns(t *testing.T) { t.Parallel() + + // Set this so the test will print the patterns found, in string slice format for easy copy-paste + outputPatternsForTestUpdate := false + tests := []struct { name string drain *Drain @@ -24,20 +30,38 @@ func TestDrain_TrainExtractsPatterns(t *testing.T) { drain: New(DefaultConfig(), nil), inputFile: `testdata/agent-logfmt.txt`, patterns: []string{ - "ts=2024-04-16T15:10:43.192290389Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg=\"Adding target\" key=\"/var/log/pods/*19a1cce8-5f04-46e0-a124-292b0dd9b343/testcoordinator/*.log:{batch_kubernetes_io_controller_uid=\\\"25ec5edf-f78e-468b-b6f3-3b9685f0cc8f\\\", batch_kubernetes_io_job_name=\\\"testcoordinator-job-2665838\\\", container=\\\"testcoordinator\\\", controller_uid=\\\"25ec5edf-f78e-468b-b6f3-3b9685f0cc8f\\\", job=\\\"k6-cloud/testcoordinator\\\", job_name=\\\"testcoordinator-job-2665838\\\", name=\\\"testcoordinator\\\", namespace=\\\"k6-cloud\\\", pod=\\\"testcoordinator-job-2665838-9g8ds\\\"}\"", - "<_> <_> level=info component=logs logs_config=default <_> target\" <_> <_> <_> <_> <_> <_>", - "<_> caller=filetarget.go:192 level=info component=logs logs_config=default msg=\"filetarget: watcher closed, tailer stopped, positions saved\" <_>", - "<_> caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg=\"tail routine: tail channel closed, stopping tailer\" <_> reason=null", - "<_> caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg=\"skipping update of position for a file which does not currently exist\" <_>", - "<_> caller=log.go:168 component=logs logs_config=default level=info msg=\"Successfully reopened <_>", - "<_> caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg=\"failed to decode logfmt\" err=\"bufio.Scanner: token too long\"", - "<_> caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg=\"received file watcher event\" <_> op=CREATE", - "<_> caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg=\"failed to decode logfmt\" err=\"logfmt syntax error at pos <_> on line 1: unexpected '\\\"'\"", - "<_> <_> level=info component=logs logs_config=default <_> <_> <_> <_> <_>", - "<_> caller=log.go:168 component=logs logs_config=default level=info <_> <_> <_> <_> <_>", - "<_> caller=filetarget.go:313 level=info component=logs logs_config=default msg=\"watching new directory\" <_>", - "<_> <_> level=info component=logs logs_config=default <_> target\" <_> conprof=\\\"true\\\", <_> <_> job=\\\"hosted-grafana/grafana\\\", name=\\\"grafana\\\", namespace=\\\"hosted-grafana\\\", <_> plan=\\\"free\\\", <_> <_> <_> <_> <_>", - "<_> level=info msg=\"finished node evaluation\" controller_id=module.http.cloudwatch_pipelines <_> <_>", + `<_> caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" <_>`, + `<_> caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" <_>`, + `<_> caller=filetarget.go:326 level=info component=logs logs_config=default msg="removing directory from watcher" <_>`, + `<_> caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" <_> op=CREATE`, + `<_> caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" <_> container=\"kube-proxy\", <_> namespace=\"kube-system\", pod=\"kube-proxy-gke-ops-us-east-0-main-n2s32-1-1dd39c-32ae1dde-hmhw\", tier=\"node\"}"`, + `<_> caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" <_> conprof=\"true\", container=\"grafana\", <_> job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", <_> plan=\"free\", <_> <_> <_> <_> <_>`, + `<_> caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" <_> conprof=\"true\", container=\"hg-plugins\", <_> job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", <_> plan=\"free\", <_> <_> <_> <_> <_>`, + `<_> caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" <_> conprof=\"true\", container=\"hgrun\", <_> job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", <_> plan=\"free\", <_> <_> <_> <_> <_>`, + `<_> caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" <_> conprof=\"true\", container=\"hosted-grafana-security\", <_> job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", <_> plan=\"free\", <_> <_> <_> <_> <_>`, + `<_> caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file <_> ..."`, + `<_> caller=log.go:168 component=logs logs_config=default level=info msg="Seeked <_> - &{Offset:0 Whence:0}"`, + `<_> caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened <_>`, + `<_> caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for <_> to appear..."`, + `<_> caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"`, + `<_> caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos <_> on line 1: unexpected '\"'"`, + `<_> caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" <_>`, + `<_> caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" <_>`, + `<_> caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" <_>`, + `<_> caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" <_> reason=null`, + `<_> caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" <_>`, + `<_> caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" <_>`, + `<_> level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines <_> <_>`, + `ts=2024-04-16T15:10:42.556278698Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*b92ee988-5c26-4c64-bba3-ff6a01723759/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i1111\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"orgnamehere\", plan=\"free\", pod=\"orgnamehere-grafana-7c65678f86-9zhlb\", pod_template_hash=\"7c65678f86\", resource_version=\"143638246\", slug=\"orgnamehere\", stackId=\"866772\"}"`, + `ts=2024-04-16T15:10:42.556706613Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*b92ee988-5c26-4c64-bba3-ff6a01723759/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i1111\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"orgnamehere\", plan=\"free\", pod=\"orgnamehere-grafana-7c65678f86-9zhlb\", pod_template_hash=\"7c65678f86\", resource_version=\"143638246\", slug=\"orgnamehere\", stackId=\"866772\"}"`, + `ts=2024-04-16T15:10:42.556930066Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*b92ee988-5c26-4c64-bba3-ff6a01723759/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i1111\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"orgnamehere\", plan=\"free\", pod=\"orgnamehere-grafana-7c65678f86-9zhlb\", pod_template_hash=\"7c65678f86\", resource_version=\"143638246\", slug=\"orgnamehere\", stackId=\"866772\"}"`, + `ts=2024-04-16T15:10:42.557102408Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*b92ee988-5c26-4c64-bba3-ff6a01723759/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i1111\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"orgnamehere\", plan=\"free\", pod=\"orgnamehere-grafana-7c65678f86-9zhlb\", pod_template_hash=\"7c65678f86\", resource_version=\"143638246\", slug=\"orgnamehere\", stackId=\"866772\"}"`, + `ts=2024-04-16T15:10:43.192290389Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*19a1cce8-5f04-46e0-a124-292b0dd9b343/testcoordinator/*.log:{batch_kubernetes_io_controller_uid=\"25ec5edf-f78e-468b-b6f3-3b9685f0cc8f\", batch_kubernetes_io_job_name=\"testcoordinator-job-2665838\", container=\"testcoordinator\", controller_uid=\"25ec5edf-f78e-468b-b6f3-3b9685f0cc8f\", job=\"k6-cloud/testcoordinator\", job_name=\"testcoordinator-job-2665838\", name=\"testcoordinator\", namespace=\"k6-cloud\", pod=\"testcoordinator-job-2665838-9g8ds\"}"`, + `ts=2024-04-16T15:10:43.551543875Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*35649bfd-52ff-4281-9294-5f65fd5a89fc/marketplaces-api/*.log:{container=\"marketplaces-api\", job=\"grafana-com/marketplaces-api\", name=\"marketplaces-api\", namespace=\"grafana-com\", pod=\"marketplaces-api-f67ff7567-gqrvb\", pod_template_hash=\"f67ff7567\"}"`, + `ts=2024-04-16T15:10:43.869370539Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i2222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"someorg\", plan=\"free\", pod=\"someorg-grafana-666bd48cf9-7zrtv\", pod_template_hash=\"666bd48cf9\", resource_version=\"167212086\", slug=\"someorg\", stackId=\"444444\"}"`, + `ts=2024-04-16T15:10:43.869672113Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i2222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"someorg\", plan=\"free\", pod=\"someorg-grafana-666bd48cf9-7zrtv\", pod_template_hash=\"666bd48cf9\", resource_version=\"167212086\", slug=\"someorg\", stackId=\"444444\"}"`, + `ts=2024-04-16T15:10:43.869833185Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i2222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"someorg\", plan=\"free\", pod=\"someorg-grafana-666bd48cf9-7zrtv\", pod_template_hash=\"666bd48cf9\", resource_version=\"167212086\", slug=\"someorg\", stackId=\"444444\"}"`, + `ts=2024-04-16T15:10:43.870016638Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i2222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"someorg\", plan=\"free\", pod=\"someorg-grafana-666bd48cf9-7zrtv\", pod_template_hash=\"666bd48cf9\", resource_version=\"167212086\", slug=\"someorg\", stackId=\"444444\"}"`, }, }, { @@ -46,9 +70,9 @@ func TestDrain_TrainExtractsPatterns(t *testing.T) { drain: New(DefaultConfig(), nil), inputFile: `testdata/ingester-logfmt.txt`, patterns: []string{ - "<_> caller=head.go:216 level=debug tenant=987678 msg=\"profile is empty after delta computation\" metricName=memory", - "ts=2024-04-17T09:52:46.363974185Z caller=http.go:194 level=debug traceID=1b48f5156a61ca69 msg=\"GET /debug/pprof/delta_mutex (200) 1.161082ms\"", - "<_> caller=http.go:194 level=debug <_> <_> msg=\"POST /ingester.v1.IngesterService/Push (200) <_>", // A perfect log line: Abstracted the variable part but kept the constants. + `<_> caller=head.go:216 level=debug tenant=987678 msg="profile is empty after delta computation" metricName=memory`, + `<_> caller=http.go:194 level=debug <_> <_> msg="POST /ingester.v1.IngesterService/Push (200) <_>`, + `ts=2024-04-17T09:52:46.363974185Z caller=http.go:194 level=debug traceID=1b48f5156a61ca69 msg="GET /debug/pprof/delta_mutex (200) 1.161082ms"`, }, }, { @@ -57,41 +81,41 @@ func TestDrain_TrainExtractsPatterns(t *testing.T) { drain: New(DefaultConfig(), nil), inputFile: `testdata/drone-json.txt`, patterns: []string{ - "<_> capacity <_>", - "<_> capacity changes <_>", - "{\"id\":\"D4Oh1ivB6cdLWa08\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:48:52Z\"}", - "{\"id\":\"q62wCcIkEOueqFKF\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T15:03:28Z\"}", - "{\"id\":\"m6SpYHzdXrDAFqDR\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T15:02:58Z\"}", - "{\"id\":\"T0I8Dsnw3uSi3Gal\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T15:02:28Z\"}", - "{\"id\":\"9eA72xOtx8kzMhXn\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T15:01:58Z\"}", - "{\"id\":\"pet7QVfO1yE8fk56\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T15:01:28Z\"}", - "{\"id\":\"15eSzaEG0enf86Kl\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T15:00:57Z\"}", - "{\"id\":\"JO1OT5ADoNA8NYqr\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T15:00:27Z\"}", - "{\"id\":\"Xz2OCJhgeBSRFyoN\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:59:57Z\"}", - "{\"id\":\"pPc2ORUhHAhFgBg3\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:59:27Z\"}", - "{\"id\":\"4G6Srn6lSwzYrx19\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:58:57Z\"}", - "{\"id\":\"1Lu90T1fWzsWOKlc\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:58:27Z\"}", - "{\"id\":\"4XjwwNoOwZFaWePQ\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:57:57Z\"}", - "{\"id\":\"IQy23J3NON0BV10V\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:57:26Z\"}", - "{\"id\":\"FQ8wCQfaR9W387cH\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:56:56Z\"}", - "{\"id\":\"Hhwn7ecXjxF67DG6\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:56:26Z\"}", - "{\"id\":\"luflyGZvZnLzhQEH\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:55:56Z\"}", - "{\"id\":\"q20GZcvyzMwrTGx5\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:55:26Z\"}", - "{\"id\":\"3K61Yf6ImKYexoFx\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:54:56Z\"}", - "{\"id\":\"SmbOO0l5aADX9BaQ\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:54:23Z\"}", - "{\"id\":\"96TvvsMzSkkaW8oW\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:53:53Z\"}", - "{\"id\":\"C7aYn8cb4NCrkkYI\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:53:23Z\"}", - "{\"id\":\"CMG7ZwwYqNPBonAn\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:52:53Z\"}", - "{\"id\":\"focV9BzODwRbWwKE\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:52:23Z\"}", - "{\"id\":\"HphRnJOM8uYohf1p\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:51:53Z\"}", - "{\"id\":\"m3n8GndhG45uGIQA\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:51:23Z\"}", - "{\"id\":\"nTO38tWtnvRWRl1G\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:50:52Z\"}", - "{\"id\":\"5qEIzErDfiALVPAN\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:50:22Z\"}", - "{\"id\":\"q61oHTtF4MMiQVGH\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:49:52Z\"}", - "{\"id\":\"4rNxIlhDKxGgzBHe\",\"level\":\"debug\",\"max-pool\":4,\"min-pool\":0,\"msg\":\"check capacity\",\"pending-builds\":0,\"running-builds\":0,\"server-buffer\":0,\"server-capacity\":0,\"server-count\":0,\"time\":\"2024-04-16T14:49:22Z\"}", - "<_> server <_>", - "<_> unfinished <_>", - "<_> <_> (flow; linux; helm)\"}", + `<_> <_> (flow; linux; helm)"}`, + `<_> capacity <_>`, + `<_> capacity changes <_>`, + `<_> server <_>`, + `<_> unfinished <_>`, + `{"id":"15eSzaEG0enf86Kl","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:00:57Z"}`, + `{"id":"1Lu90T1fWzsWOKlc","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:58:27Z"}`, + `{"id":"3K61Yf6ImKYexoFx","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:54:56Z"}`, + `{"id":"4G6Srn6lSwzYrx19","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:58:57Z"}`, + `{"id":"4XjwwNoOwZFaWePQ","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:57:57Z"}`, + `{"id":"4rNxIlhDKxGgzBHe","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:49:22Z"}`, + `{"id":"5qEIzErDfiALVPAN","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:50:22Z"}`, + `{"id":"96TvvsMzSkkaW8oW","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:53:53Z"}`, + `{"id":"9eA72xOtx8kzMhXn","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:01:58Z"}`, + `{"id":"C7aYn8cb4NCrkkYI","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:53:23Z"}`, + `{"id":"CMG7ZwwYqNPBonAn","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:52:53Z"}`, + `{"id":"D4Oh1ivB6cdLWa08","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:48:52Z"}`, + `{"id":"FQ8wCQfaR9W387cH","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:56:56Z"}`, + `{"id":"Hhwn7ecXjxF67DG6","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:56:26Z"}`, + `{"id":"HphRnJOM8uYohf1p","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:51:53Z"}`, + `{"id":"IQy23J3NON0BV10V","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:57:26Z"}`, + `{"id":"JO1OT5ADoNA8NYqr","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:00:27Z"}`, + `{"id":"SmbOO0l5aADX9BaQ","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:54:23Z"}`, + `{"id":"T0I8Dsnw3uSi3Gal","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:02:28Z"}`, + `{"id":"Xz2OCJhgeBSRFyoN","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:59:57Z"}`, + `{"id":"focV9BzODwRbWwKE","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:52:23Z"}`, + `{"id":"luflyGZvZnLzhQEH","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:55:56Z"}`, + `{"id":"m3n8GndhG45uGIQA","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:51:23Z"}`, + `{"id":"m6SpYHzdXrDAFqDR","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:02:58Z"}`, + `{"id":"nTO38tWtnvRWRl1G","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:50:52Z"}`, + `{"id":"pPc2ORUhHAhFgBg3","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:59:27Z"}`, + `{"id":"pet7QVfO1yE8fk56","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:01:28Z"}`, + `{"id":"q20GZcvyzMwrTGx5","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:55:26Z"}`, + `{"id":"q61oHTtF4MMiQVGH","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:49:52Z"}`, + `{"id":"q62wCcIkEOueqFKF","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:03:28Z"}`, }, }, { @@ -107,90 +131,123 @@ func TestDrain_TrainExtractsPatterns(t *testing.T) { drain: New(DefaultConfig(), nil), inputFile: "testdata/journald.txt", patterns: []string{ - "2024-05-07T11:59:43.484606Z INFO ExtHandler ExtHandler Downloading agent manifest", - "<_> INFO TelemetryEventsCollector ExtHandler Collected 2 events for extension: Microsoft.Azure.Extensions.CustomScript", - "E0507 11:59:41.375655 4736 kuberuntime_manager.go:1256] container &Container{Name:ruler,Image:grafana/enterprise-metrics:v2.12.0,Command:[],Args:[-target=ruler -config.expand-env=true -config.file=/etc/mimir/mimir.yaml -distributor.remote-timeout=10s],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:http-metrics,HostPort:0,ContainerPort:8080,Protocol:TCP,HostIP:,},ContainerPort{Name:grpc,HostPort:0,ContainerPort:9095,Protocol:TCP,HostIP:,},ContainerPort{Name:memberlist,HostPort:0,ContainerPort:7946,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:JAEGER_AGENT_HOST,Value:alloy-otlp.alloy-otlp.svc.cluster.local.,ValueFrom:nil,},EnvVar{Name:JAEGER_TAGS,Value:namespace=ge-metrics-federation,cluster=dev-us-central-0,ValueFrom:nil,},EnvVar{Name:JAEGER_SAMPLER_MANAGER_HOST_PORT,Value:http://alloy-otlp.alloy-otlp.svc.cluster.local.:5778/sampling,ValueFrom:nil,},EnvVar{Name:GOOGLE_APPLICATION_CREDENTIALS,Value:/var/secrets/google/credentials.json,ValueFrom:nil,},EnvVar{Name:AM_TOKEN,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:ruler-alertmanager-token,},Key:token,Optional:nil,},},},EnvVar{Name:JAEGER_REPORTER_MAX_QUEUE_SIZE,Value:1000,ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{cpu: {{100 -3} {} 100m DecimalSI},memory: {{134217728 0} {} BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:gcs-credentials,ReadOnly:false,MountPath:/var/secrets/google/,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:config,ReadOnly:false,MountPath:/etc/mimir,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:license,ReadOnly:false,MountPath:/license,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:runtime-config,ReadOnly:false,MountPath:/var/mimir,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:storage,ReadOnly:false,MountPath:/data,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:active-queries,ReadOnly:false,MountPath:/active-query-tracker,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-jtnbs,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/ready,Port:{1 0 http-metrics},Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:45,TimeoutSeconds:1,PeriodSeconds:10,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:*true,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod gem-mimir-ruler-5f56f7846b-fgxdm_ge-metrics-federation(07c06e21-137b-4fdd-b7d3-703f0a567720): CreateContainerConfigError: secret \"ruler-alertmanager-token\" not found", - "\tts=2024-05-07T11:59:32.025687537Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg=\"request", - "time=\"2024-05-07T11:59:38.484586527Z\" level=error msg=\"Failed to delete exec process \\\"d9e0a1867ce73695ad859f2b0a76fe8f5053db8a5e49142d747e53a445729bd4\\\" for container \\\"6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408\\\"\" error=\"ttrpc: closed: unknown\"", - "I0507 <_> <_> prober.go:107] \"Probe failed\" probeType=\"Readiness\" <_> <_> <_> probeResult=\"failure\" output=\"HTTP probe failed with statuscode: <_>", - "net_ratelimit: 2 callbacks suppressed", - "kauditd_printk_skb: <_> callbacks suppressed", - "Started cri-containerd-95bf586cd79d43120ff44582d4dbd2476de61744411f8515b9b2c527a41fd5d9.scope.", - "Removed slice libcontainer container kubepods-burstable-pod25cb986c_3d6c_4ed0_abf3_ee59ed6175f9.slice.", - "E0507 11:59:34.923938 3027 kuberuntime_manager.go:1261] container &Container{Name:mysqld-exporter,Image:prom/mysqld-exporter:v0.13.0,Command:[],Args:[--collect.info_schema.innodb_metrics],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:http-metrics,HostPort:0,ContainerPort:9104,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:MYSQL_USER,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:testcrossplane-user-exporter,},Key:username,Optional:nil,},},},EnvVar{Name:MYSQL_PASSWORD,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:testcrossplane-user-exporter,},Key:password,Optional:nil,},},},EnvVar{Name:MYSQL_HOST,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:testcrossplane-user-exporter,},Key:endpoint,Optional:nil,},},},EnvVar{Name:MYSQL_PORT,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:testcrossplane-user-exporter,},Key:port,Optional:nil,},},},EnvVar{Name:MYSQL_TLS_MODE,Value:preferred,ValueFrom:nil,},EnvVar{Name:DATA_SOURCE_NAME,Value:$(MYSQL_USER):$(MYSQL_PASSWORD)@tcp($(MYSQL_HOST):$(MYSQL_PORT))/?tls=$(MYSQL_TLS_MODE),ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-dzx7d,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod testcrossplane-exporter-c67cfc58f-vbzl4_crossplane-playground(3d49134d-3378-4ec3-824c-5ff4ea2590a5): CreateContainerConfigError: secret \"testcrossplane-user-exporter\" not found", - "I0507 <_> 3224 <_> <_> <_> for volume <_> (UniqueName: <_> <_> <_> <_> <_> <_>", - "E0507 <_> <_> kuberuntime_manager.go:1256] container &Container{Name:ruler,Image:grafana/enterprise-metrics:v2.11.1,Command:[],Args:[-target=ruler -config.expand-env=true <_> {{100 -3} {} 100m DecimalSI},memory: {{134217728 0} {} <_> 0 http-metrics},Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:45,TimeoutSeconds:1,PeriodSeconds:10,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:*true,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod <_> CreateContainerConfigError: secret \"ruler-alertmanager-token\" not found", - "time=\"2024-05-07T11:59:34.707025668Z\" level=info msg=\"StopPodSandbox for \\\"c605ad2cdc74c6b5288f2532ad71cce81a28ef6965f97a89ff6609deb825553a\\\" returns successfully\"", - "time=\"2024-05-07T11:59:34.706960850Z\" level=info msg=\"TearDown network for sandbox \\\"c605ad2cdc74c6b5288f2532ad71cce81a28ef6965f97a89ff6609deb825553a\\\" successfully\"", - "time=\"2024-05-07T11:59:34.592084495Z\" level=info msg=\"Container to stop \\\"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\\\" must be in running or unknown state, current state \\\"CONTAINER_EXITED\\\"\"", - "time=\"2024-05-07T11:59:34.592005066Z\" level=info msg=\"StopPodSandbox for \\\"c605ad2cdc74c6b5288f2532ad71cce81a28ef6965f97a89ff6609deb825553a\\\"\"", - "time=\"2024-05-07T11:59:34.591282703Z\" level=info msg=\"StopContainer for \\\"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\\\" returns successfully\"", - "time=\"2024-05-07T11:59:34.520032214Z\" level=info msg=\"Stop container \\\"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\\\" with signal terminated\"", - "time=\"2024-05-07T11:59:34.519591759Z\" level=info msg=\"StopContainer for \\\"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\\\" with timeout 30 (s)\"", - "E0507 <_> <_> pod_workers.go:1300] \"Error syncing pod, skipping\" err=\"failed to \\\"StartContainer\\\" for \\\"grafana\\\" with ErrImagePull: \\\"[rpc error: code = NotFound desc = failed to pull and unpack image <_> failed to resolve reference <_> <_> not found, failed to pull and unpack image <_> failed to resolve reference <_> unexpected status from HEAD request to <_> 403 Forbidden]\\\"\" <_> <_>", - "\t\t\t\t\t\twhile [ \"$(pidof plugins-pause)\" = \"\" ]; do sleep 0.5; done;", - "\t\t\t\t\t\tln --force -s /proc/$(pidof hgrun-pause)/root/bin/hgrun /bin/hgrun;", - "\t\t\t\t\t\texec /bin/hgrun -log.level=debug launch -bundledPluginsManifest /proc/$(pidof plugins-pause)/root/manifest.json -bundledPluginsDir /proc/$(pidof <_> -profile-port=6060 <_> {{536870912 0} {} BinarySI},},Requests:ResourceList{cpu: {{26 -3} {} 26m DecimalSI},memory: {{293601280 0} {} BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{},LivenessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/api/health,Port:{0 80 },Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:300,TimeoutSeconds:10,PeriodSeconds:30,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:&ExecAction{Command:[/bin/hgrun check],},HTTPGet:nil,TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:0,TimeoutSeconds:30,PeriodSeconds:30,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:&Lifecycle{PostStart:nil,PreStop:&LifecycleHandler{Exec:&ExecAction{Command:[/bin/hgrun drain -timeout 1m0s -waitTime 55s],},HTTPGet:nil,TCPSocket:nil,},},TerminationMessagePath:/dev/termination-log,ImagePullPolicy:Always,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[SYS_PTRACE],Drop:[],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod <_> ErrImagePull: [rpc error: code = NotFound desc = failed to pull and unpack image <_> failed to resolve reference <_> <_> not found, failed to pull and unpack image <_> failed to resolve reference <_> unexpected status from HEAD request to <_> 403 Forbidden]", - "<_> level=error msg=\"PullImage <_> failed\" error=\"failed to pull and unpack image <_> failed to resolve reference <_> unexpected status from HEAD request to <_> 403 Forbidden\"", - "<_> level=error msg=\"PullImage <_> failed\" error=\"rpc error: code = NotFound desc = failed to pull and unpack image <_> failed to resolve reference <_> <_> not found\"", - "<_> level=info msg=\"trying next host - response was http.StatusNotFound\" host=us.gcr.io", - "I0507 11:59:34.518822 3224 kuberuntime_container.go:745] \"Killing container with a grace period\" pod=\"hosted-grafana/hosted-grafana-api-7b6bd9b949-9csb4\" podUID=\"25cb986c-3d6c-4ed0-abf3-ee59ed6175f9\" containerName=\"hgapi\" containerID=\"containerd://c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\" gracePeriod=30", - "E0507 <_> <_> prober.go:239] \"Unable to write all bytes from execInContainer\" err=\"short write\" <_> actualBytes=10240", - "I0507 11:59:33.422254 1537502 kubelet_getters.go:187] \"Pod status updated\" pod=\"kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-x28r\" status=\"Running\"", - "<_> level=info msg=\"RemoveContainer for <_> returns successfully\"", - "<_> level=info msg=\"RemoveContainer for <_>", - "E0507 <_> <_> prober.go:104] \"Probe errored\" err=\"rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task <_> not found: not found\" probeType=\"Readiness\" <_> <_> containerName=\"grafana\"", - "<_> level=error msg=\"ExecSync for <_> failed\" error=\"rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task <_> not found: not found\"", - " >", - "E0507 <_> <_> remote_image.go:180] \"PullImage from image service failed\" err=\"rpc error: code = Unknown desc = failed to pull and unpack image <_> failed to resolve reference <_> unexpected status from HEAD request to <_> 403 Forbidden\" <_>", - "E0507 <_> <_> remote_runtime.go:496] \"ExecSync cmd from runtime service failed\" err=\"rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task <_> not found: not found\" <_> cmd=[\"/bin/hgrun\",\"check\"]", - "<_> level=error caller=http_client.go:56 app=hgrun <_> msg=\"request failed\" error=\"Get \\\"http://127.0.0.1:3000/api/health\\\": dial tcp 127.0.0.1:3000: connect: connection refused\" method=GET url=http://127.0.0.1:3000/api/health", - "<_> level=warning msg=\"cleaning up after shim disconnected\" <_> namespace=k8s.io", - "<_> level=info msg=\"cleaning up dead shim\" namespace=k8s.io", - "<_> level=info msg=\"shim disconnected\" <_> namespace=k8s.io", - "I0507 11:59:32.409568 581823 cache.go:40] re-using cached key and certificate", - "I0507 <_> <_> <_> <_> (PLEG): <_> <_> <_> <_> <_>", - "<_> level=info msg=\"StartContainer for <_> returns successfully\"", - "audit: type=1400 <_> apparmor=\"DENIED\" operation=\"ptrace\" profile=\"cri-containerd.apparmor.d\" <_> comm=\"pidof\" requested_mask=\"read\" denied_mask=\"read\" peer=\"unconfined\"", - "AVC apparmor=\"DENIED\" operation=\"ptrace\" profile=\"cri-containerd.apparmor.d\" <_> comm=\"pidof\" requested_mask=\"read\" denied_mask=\"read\" peer=\"unconfined\"", - "Started libcontainer container <_>", - "<_> level=info msg=\"StartContainer for <_>", - "<_> level=info msg=\"CreateContainer within sandbox <_> for <_> returns container id <_>", - "<_> level=info msg=\"CreateContainer within sandbox <_> for container <_>", - "<_> level=info msg=\"PullImage <_>", - "<_> level=info msg=\"PullImage <_> returns image reference <_>", - "<_> level=info msg=\"Pulled image <_> with image id <_> repo tag <_> repo digest <_> size <_> in <_>", - "<_> level=info msg=\"ImageUpdate event <_> labels:{key:\\\"io.cri-containerd.image\\\" value:\\\"managed\\\"}\"", - "<_> level=info msg=\"stop pulling image <_> active requests=0, bytes <_>", - "<_> level=info msg=\"ImageCreate event <_> labels:{key:\\\"io.cri-containerd.image\\\" value:\\\"managed\\\"}\"", - "E0507 <_> <_> kuberuntime_manager.go:1256] container <_> set -e; while [ \"$(pidof hgrun-pause)\" = \"\" ]; do sleep 0.5; done;", - "I0507 <_> 6247 prober.go:107] \"Probe failed\" probeType=\"Readiness\" pod=\"grafana-agent/grafana-agent-helm-4\" podUID=\"c36c5200-1cd6-4093-893c-c022f91af996\" containerName=\"grafana-agent\" probeResult=\"failure\" output=\"Get \\\"http://10.0.99.125:3090/-/ready\\\": dial tcp 10.0.99.125:3090: connect: connection refused\"", - "<_> Consumed <_> CPU time.", - "<_> Deactivated successfully.", - "RCV: Reply message on eth0 from fe80::e9:7eff:fedf:3d37.", - "XMT: Renew on eth0, interval 9700ms.", - "PRC: Renewing lease on eth0.", - "I0507 <_> <_> prober.go:107] \"Probe failed\" probeType=\"Readiness\" <_> <_> containerName=\"grafana\" probeResult=\"failure\" output=<", - "I0507 <_> 2791 azure_credentials.go:220] <_> is not from ACR, return empty authentication", - "I0507 <_> <_> <_> \"Cleaned up orphaned pod volumes dir\" <_> <_>", - "XMT: Solicit on eth0, interval <_>", - "I0507 <_> <_> cache.go:40] re-using cached key and certificate", - "ll header: 00000000: 42 01 0a 80 00 <_> 42 01 0a 80 00 01 08 00", - "IPv4: martian source <_> from <_> on dev eth0", - "I0507 11:59:29.320184 1537502 kubelet_pods.go:906] \"Unable to retrieve pull secret, the image pull may not succeed.\" pod=\"logs-endpoint-dev-005/kafka-controller-0\" secret=\"\" err=\"secret \\\"not-needed\\\" not found\"", - "E0507 <_> <_> kuberuntime_manager.go:1256] container &Container{Name:pdc,Image:us.gcr.io/hosted-grafana/pdc:0.1.415,Command:[],Args:[-proxy.auth.ca-keys-dir=/var/run/secrets/pdc-certs -proxy.socks-server.addr=:10443 -proxy.ssh-server.addr=:2222 -proxy.use-socks-username-for-routing -proxy.api.http-address=:9182 -proxy.check-connpool-address-in-ring -memberlist.join=dns+gossip-ring.pdc.svc.cluster.local:7946 -api.http-address=:11443 -distributor.enabled=true -distributor.addr=:10444 -distributor.use-socks-username-for-routing -gateway.enabled=true -gateway.addr=:2244 -log.level=debug -certs.ca-private-key-file=/var/run/secrets/pdc-certs/ca.key -certs.ca-cert-file=/var/run/secrets/pdc-certs/ca.crt -certs.ca-pub-file=/var/run/secrets/pdc-certs/ca.pub -certs.cluster=local-k8s -shard-size=3 -graceful-shutdown-period=30s -enable-multiple-networks],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:socks,HostPort:0,ContainerPort:10443,Protocol:TCP,HostIP:,},ContainerPort{Name:ssh,HostPort:0,ContainerPort:2222,Protocol:TCP,HostIP:,},ContainerPort{Name:distributor,HostPort:0,ContainerPort:10444,Protocol:TCP,HostIP:,},ContainerPort{Name:gateway,HostPort:0,ContainerPort:2244,Protocol:TCP,HostIP:,},ContainerPort{Name:api,HostPort:0,ContainerPort:11443,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:POD_NAME,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.name,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},},Resources:ResourceRequirements{Limits:ResourceList{cpu: {{500 -3} {} 500m DecimalSI},memory: {{134217728 0} {} BinarySI},},Requests:ResourceList{cpu: {{250 -3} {} 250m DecimalSI},memory: {{67108864 0} {} <_> 11443 },Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:40,TimeoutSeconds:1,PeriodSeconds:5,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:&Lifecycle{PostStart:nil,PreStop:&LifecycleHandler{Exec:&ExecAction{Command:[/bin/sleep 5],},HTTPGet:nil,TCPSocket:nil,},},TerminationMessagePath:/dev/termination-log,ImagePullPolicy:Never,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod <_> ErrImageNeverPull: Container image \"us.gcr.io/hosted-grafana/pdc:0.1.415\" is not present with pull policy of Never", - "I0507 <_> <_> kubelet_pods.go:906] \"Unable to retrieve pull secret, the image pull may not succeed.\" <_> secret=\"\" err=\"secret <_> not found\"", - "I0507 <_> 3224 operation_generator.go:888] UnmountVolume.TearDown succeeded for volume <_> (OuterVolumeSpecName: <_> pod \"25cb986c-3d6c-4ed0-abf3-ee59ed6175f9\" (UID: \"25cb986c-3d6c-4ed0-abf3-ee59ed6175f9\"). InnerVolumeSpecName <_> PluginName <_> VolumeGidValue \"\"", - "E0507 <_> <_> pod_workers.go:1300] \"Error syncing pod, skipping\" <_> <_> <_> <_> <_> <_> <_> <_> <_> <_> <_> <_> <_>", - "E0507 <_> <_> pod_workers.go:1300] \"Error syncing pod, skipping\" err=\"failed to \\\"StartContainer\\\" for <_> with CrashLoopBackOff: \\\"back-off <_> restarting failed <_> <_> <_> <_>", - "I0507 <_> <_> <_> <_> <_> <_> <_> <_>", - "I0507 <_> <_> <_> \"SyncLoop <_> source=\"api\" <_>", - "<_> level=error msg=\"ContainerStatus for <_> failed\" error=\"rpc error: code = NotFound desc = an error occurred when try to find container <_> not found\"", - "I0507 <_> <_> scope.go:117] \"RemoveContainer\" <_>", - "E0507 <_> <_> remote_image.go:180] \"PullImage from image service failed\" err=\"rpc error: code = NotFound desc = failed to pull and unpack image <_> failed to resolve reference <_> <_> not found\" <_>", - "I0507 <_> <_> pod_container_deletor.go:53] \"DeleteContainer returned error\" <_> err=\"failed to get container status <_> rpc error: code = NotFound desc = an error occurred when try to find container <_> not found\"", - "E0507 <_> <_> pod_workers.go:1300] \"Error syncing pod, skipping\" err=\"failed to \\\"StartContainer\\\" for \\\"pdc\\\" with ErrImageNeverPull: \\\"Container image \\\\\\\"us.gcr.io/hosted-grafana/pdc:0.1.415\\\\\\\" is not present with pull policy of Never\\\"\" <_> <_>", - "E0507 <_> <_> remote_runtime.go:432] \"ContainerStatus from runtime service failed\" err=\"rpc error: code = NotFound desc = an error occurred when try to find container <_> not found\" <_>", + ` exec /bin/hgrun -log.level=debug launch -bundledPluginsManifest /proc/$(pidof plugins-pause)/root/manifest.json -bundledPluginsDir /proc/$(pidof <_> -profile-port=6060 <_> {{536870912 0} {} BinarySI},},Requests:ResourceList{cpu: {{26 -3} {} 26m DecimalSI},memory: {{293601280 0} {} BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{},LivenessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/api/health,Port:{0 80 },Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:300,TimeoutSeconds:10,PeriodSeconds:30,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:&ExecAction{Command:[/bin/hgrun check],},HTTPGet:nil,TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:0,TimeoutSeconds:30,PeriodSeconds:30,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:&Lifecycle{PostStart:nil,PreStop:&LifecycleHandler{Exec:&ExecAction{Command:[/bin/hgrun drain -timeout 1m0s -waitTime 55s],},HTTPGet:nil,TCPSocket:nil,},},TerminationMessagePath:/dev/termination-log,ImagePullPolicy:Always,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[SYS_PTRACE],Drop:[],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod <_> ErrImagePull: [rpc error: code = NotFound desc = failed to pull and unpack image <_> failed to resolve reference <_> <_> not found, failed to pull and unpack image <_> failed to resolve reference <_> unexpected status from HEAD request to <_> 403 Forbidden]`, + ` ln --force -s /proc/$(pidof hgrun-pause)/root/bin/hgrun /bin/hgrun;`, + ` while [ "$(pidof plugins-pause)" = "" ]; do sleep 0.5; done;`, + ` ts=2024-05-07T11:59:32.025687537Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request`, + ` >`, + `2024-05-07T11:59:43.484606Z INFO ExtHandler ExtHandler Downloading agent manifest`, + `<_> Consumed <_> CPU time.`, + `<_> Deactivated successfully.`, + `<_> INFO TelemetryEventsCollector ExtHandler Collected 2 events for extension: Microsoft.Azure.Extensions.CustomScript`, + `<_> level=error caller=http_client.go:56 app=hgrun <_> msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health`, + `<_> level=error msg="ContainerStatus for <_> failed" error="rpc error: code = NotFound desc = an error occurred when try to find container <_> not found"`, + `<_> level=error msg="ExecSync for <_> failed" error="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task <_> not found: not found"`, + `<_> level=error msg="PullImage <_> failed" error="failed to pull and unpack image <_> failed to resolve reference <_> unexpected status from HEAD request to <_> 403 Forbidden"`, + `<_> level=error msg="PullImage <_> failed" error="rpc error: code = NotFound desc = failed to pull and unpack image <_> failed to resolve reference <_> <_> not found"`, + `<_> level=info msg="CreateContainer within sandbox <_> for <_> returns container id <_>`, + `<_> level=info msg="CreateContainer within sandbox <_> for container <_>`, + `<_> level=info msg="ImageCreate event <_> labels:{key:\"io.cri-containerd.image\" value:\"managed\"}"`, + `<_> level=info msg="ImageUpdate event <_> labels:{key:\"io.cri-containerd.image\" value:\"managed\"}"`, + `<_> level=info msg="PullImage <_>`, + `<_> level=info msg="PullImage <_> returns image reference <_>`, + `<_> level=info msg="Pulled image <_> with image id <_> repo tag <_> repo digest <_> size <_> in <_>`, + `<_> level=info msg="RemoveContainer for <_>`, + `<_> level=info msg="RemoveContainer for <_> returns successfully"`, + `<_> level=info msg="StartContainer for <_>`, + `<_> level=info msg="StartContainer for <_> returns successfully"`, + `<_> level=info msg="cleaning up dead shim" namespace=k8s.io`, + `<_> level=info msg="shim disconnected" <_> namespace=k8s.io`, + `<_> level=info msg="stop pulling image <_> active requests=0, bytes <_>`, + `<_> level=info msg="trying next host - response was http.StatusNotFound" host=us.gcr.io`, + `<_> level=warning msg="cleaning up after shim disconnected" <_> namespace=k8s.io`, + `AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" <_> comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined"`, + `E0507 11:59:29.725681 3089 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"azure-resourcemanager-exporter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=azure-resourcemanager-exporter pod=azure-resourcemanager-exporter-6b5b58c666-rsttd_infra-exporters(5a95f801-309c-4f33-864a-406262c6ece6)\"" pod="infra-exporters/azure-resourcemanager-exporter-6b5b58c666-rsttd" podUID="5a95f801-309c-4f33-864a-406262c6ece6"`, + `E0507 11:59:31.554203 4531 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"frontend\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=frontend pod=otel-demo-alt-dev-frontend-79ccf98858-mbj4x_otel-demo-alt(d08e620e-00d0-49f1-a195-820a62e8de8f)\"" pod="otel-demo-alt/otel-demo-alt-dev-frontend-79ccf98858-mbj4x" podUID="d08e620e-00d0-49f1-a195-820a62e8de8f"`, + `E0507 11:59:31.928148 4734 pod_workers.go:1300] "Error syncing pod, skipping" err="unmounted volumes=[terraform-drift-detector-data], unattached volumes=[terraform-drift-detector-data], failed to process volumes=[]: context deadline exceeded" pod="terraform-drift-detector/terraform-drift-detector-d68b4c545-jg2vj" podUID="6c607496-ef26-454e-b2f2-4cb75b233fa3"`, + `E0507 11:59:34.856101 4727 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana-render-security\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-security:0.1.181\\\"\"" pod="integration/grafana-render-service-cbff479fc-cj9tp" podUID="0e3114d1-2f3a-49d6-a71d-dbc75050d8e0"`, + `E0507 11:59:34.923938 3027 kuberuntime_manager.go:1261] container &Container{Name:mysqld-exporter,Image:prom/mysqld-exporter:v0.13.0,Command:[],Args:[--collect.info_schema.innodb_metrics],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:http-metrics,HostPort:0,ContainerPort:9104,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:MYSQL_USER,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:testcrossplane-user-exporter,},Key:username,Optional:nil,},},},EnvVar{Name:MYSQL_PASSWORD,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:testcrossplane-user-exporter,},Key:password,Optional:nil,},},},EnvVar{Name:MYSQL_HOST,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:testcrossplane-user-exporter,},Key:endpoint,Optional:nil,},},},EnvVar{Name:MYSQL_PORT,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:testcrossplane-user-exporter,},Key:port,Optional:nil,},},},EnvVar{Name:MYSQL_TLS_MODE,Value:preferred,ValueFrom:nil,},EnvVar{Name:DATA_SOURCE_NAME,Value:$(MYSQL_USER):$(MYSQL_PASSWORD)@tcp($(MYSQL_HOST):$(MYSQL_PORT))/?tls=$(MYSQL_TLS_MODE),ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-dzx7d,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod testcrossplane-exporter-c67cfc58f-vbzl4_crossplane-playground(3d49134d-3378-4ec3-824c-5ff4ea2590a5): CreateContainerConfigError: secret "testcrossplane-user-exporter" not found`, + `E0507 11:59:34.923984 3027 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"mysqld-exporter\" with CreateContainerConfigError: \"secret \\\"testcrossplane-user-exporter\\\" not found\"" pod="crossplane-playground/testcrossplane-exporter-c67cfc58f-vbzl4" podUID="3d49134d-3378-4ec3-824c-5ff4ea2590a5"`, + `E0507 11:59:35.928465 4734 pod_workers.go:1300] "Error syncing pod, skipping" err="unmounted volumes=[custom-grafana-agent], unattached volumes=[], failed to process volumes=[]: context deadline exceeded" pod="loki-dev-010/custom-grafana-agent-856948968f-6jfks" podUID="17b244cc-ecb9-4fbc-beaa-8fa47fafe013"`, + `E0507 11:59:37.252214 4736 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"ksm\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=ksm pod=new-relic-nri-bundle-nrk8s-ksm-6c785668f5-jcxh2_integration(f7cc3cca-2ffb-4fde-a73e-a4ba8b0f6b3c)\"" pod="integration/new-relic-nri-bundle-nrk8s-ksm-6c785668f5-jcxh2" podUID="f7cc3cca-2ffb-4fde-a73e-a4ba8b0f6b3c"`, + `E0507 11:59:39.149450 4729 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"cluster-agent\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=cluster-agent pod=appdynamics-cluster-agent-appdynamics-cluster-agent-56667dmbnkv_integration(69bc5e6c-0451-443e-af8a-c831871afbb8)\"" pod="integration/appdynamics-cluster-agent-appdynamics-cluster-agent-56667dmbnkv" podUID="69bc5e6c-0451-443e-af8a-c831871afbb8"`, + `E0507 11:59:41.375655 4736 kuberuntime_manager.go:1256] container &Container{Name:ruler,Image:grafana/enterprise-metrics:v2.12.0,Command:[],Args:[-target=ruler -config.expand-env=true -config.file=/etc/mimir/mimir.yaml -distributor.remote-timeout=10s],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:http-metrics,HostPort:0,ContainerPort:8080,Protocol:TCP,HostIP:,},ContainerPort{Name:grpc,HostPort:0,ContainerPort:9095,Protocol:TCP,HostIP:,},ContainerPort{Name:memberlist,HostPort:0,ContainerPort:7946,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:JAEGER_AGENT_HOST,Value:alloy-otlp.alloy-otlp.svc.cluster.local.,ValueFrom:nil,},EnvVar{Name:JAEGER_TAGS,Value:namespace=ge-metrics-federation,cluster=dev-us-central-0,ValueFrom:nil,},EnvVar{Name:JAEGER_SAMPLER_MANAGER_HOST_PORT,Value:http://alloy-otlp.alloy-otlp.svc.cluster.local.:5778/sampling,ValueFrom:nil,},EnvVar{Name:GOOGLE_APPLICATION_CREDENTIALS,Value:/var/secrets/google/credentials.json,ValueFrom:nil,},EnvVar{Name:AM_TOKEN,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:ruler-alertmanager-token,},Key:token,Optional:nil,},},},EnvVar{Name:JAEGER_REPORTER_MAX_QUEUE_SIZE,Value:1000,ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{cpu: {{100 -3} {} 100m DecimalSI},memory: {{134217728 0} {} BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:gcs-credentials,ReadOnly:false,MountPath:/var/secrets/google/,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:config,ReadOnly:false,MountPath:/etc/mimir,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:license,ReadOnly:false,MountPath:/license,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:runtime-config,ReadOnly:false,MountPath:/var/mimir,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:storage,ReadOnly:false,MountPath:/data,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:active-queries,ReadOnly:false,MountPath:/active-query-tracker,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-jtnbs,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/ready,Port:{1 0 http-metrics},Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:45,TimeoutSeconds:1,PeriodSeconds:10,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:*true,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod gem-mimir-ruler-5f56f7846b-fgxdm_ge-metrics-federation(07c06e21-137b-4fdd-b7d3-703f0a567720): CreateContainerConfigError: secret "ruler-alertmanager-token" not found`, + `E0507 <_> 4731 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"overrides-exporter\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/kubernetes-dev/enterprise-logs:callum-shard-firstlast-08\\\"\"" pod="loki-dev-010/overrides-exporter-98c77fd66-6zj6m" podUID="1ff5bf3e-5856-4f6f-ae04-273f2dee170b"`, + `E0507 <_> 4733 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=bryan-prometheus-0_bryan-prometheus(6dadfe71-eb19-4231-a96e-c64bb5499a1e)\"" pod="bryan-prometheus/bryan-prometheus-0" podUID="6dadfe71-eb19-4231-a96e-c64bb5499a1e"`, + `E0507 <_> <_> kuberuntime_manager.go:1256] container &Container{Name:pdc,Image:us.gcr.io/hosted-grafana/pdc:0.1.415,Command:[],Args:[-proxy.auth.ca-keys-dir=/var/run/secrets/pdc-certs -proxy.socks-server.addr=:10443 -proxy.ssh-server.addr=:2222 -proxy.use-socks-username-for-routing -proxy.api.http-address=:9182 -proxy.check-connpool-address-in-ring -memberlist.join=dns+gossip-ring.pdc.svc.cluster.local:7946 -api.http-address=:11443 -distributor.enabled=true -distributor.addr=:10444 -distributor.use-socks-username-for-routing -gateway.enabled=true -gateway.addr=:2244 -log.level=debug -certs.ca-private-key-file=/var/run/secrets/pdc-certs/ca.key -certs.ca-cert-file=/var/run/secrets/pdc-certs/ca.crt -certs.ca-pub-file=/var/run/secrets/pdc-certs/ca.pub -certs.cluster=local-k8s -shard-size=3 -graceful-shutdown-period=30s -enable-multiple-networks],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:socks,HostPort:0,ContainerPort:10443,Protocol:TCP,HostIP:,},ContainerPort{Name:ssh,HostPort:0,ContainerPort:2222,Protocol:TCP,HostIP:,},ContainerPort{Name:distributor,HostPort:0,ContainerPort:10444,Protocol:TCP,HostIP:,},ContainerPort{Name:gateway,HostPort:0,ContainerPort:2244,Protocol:TCP,HostIP:,},ContainerPort{Name:api,HostPort:0,ContainerPort:11443,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:POD_NAME,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.name,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},},Resources:ResourceRequirements{Limits:ResourceList{cpu: {{500 -3} {} 500m DecimalSI},memory: {{134217728 0} {} BinarySI},},Requests:ResourceList{cpu: {{250 -3} {} 250m DecimalSI},memory: {{67108864 0} {} <_> 11443 },Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:40,TimeoutSeconds:1,PeriodSeconds:5,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:&Lifecycle{PostStart:nil,PreStop:&LifecycleHandler{Exec:&ExecAction{Command:[/bin/sleep 5],},HTTPGet:nil,TCPSocket:nil,},},TerminationMessagePath:/dev/termination-log,ImagePullPolicy:Never,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod <_> ErrImageNeverPull: Container image "us.gcr.io/hosted-grafana/pdc:0.1.415" is not present with pull policy of Never`, + `E0507 <_> <_> kuberuntime_manager.go:1256] container &Container{Name:ruler,Image:grafana/enterprise-metrics:v2.11.1,Command:[],Args:[-target=ruler -config.expand-env=true <_> {{100 -3} {} 100m DecimalSI},memory: {{134217728 0} {} <_> 0 http-metrics},Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:45,TimeoutSeconds:1,PeriodSeconds:10,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:*true,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod <_> CreateContainerConfigError: secret "ruler-alertmanager-token" not found`, + `E0507 <_> <_> kuberuntime_manager.go:1256] container <_> set -e; while [ "$(pidof hgrun-pause)" = "" ]; do sleep 0.5; done;`, + `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"agent\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=agent <_> <_> <_>`, + `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"cortex-gw\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=cortex-gw <_> <_> <_>`, + `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"gcom-sync\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/kubernetes-dev/frontend-monitoring:6a8eb5a\\\"\"" <_> <_>`, + `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"goldpinger\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=goldpinger <_> <_> <_>`, + `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off <_> restarting failed container=grafana <_> <_> <_>`, + `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ErrImagePull: \"[rpc error: code = NotFound desc = failed to pull and unpack image <_> failed to resolve reference <_> <_> not found, failed to pull and unpack image <_> failed to resolve reference <_> unexpected status from HEAD request to <_> 403 Forbidden]\"" <_> <_>`, + `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image <_> <_> <_>`, + `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"pdc\" with ErrImageNeverPull: \"Container image \\\"us.gcr.io/hosted-grafana/pdc:0.1.415\\\" is not present with pull policy of Never\"" <_> <_>`, + `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"ruler\" with CreateContainerConfigError: \"secret \\\"ruler-alertmanager-token\\\" not found\"" <_> <_>`, + `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"support-agent\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=support-agent <_> <_> <_>`, + `E0507 <_> <_> prober.go:104] "Probe errored" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task <_> not found: not found" probeType="Readiness" <_> <_> containerName="grafana"`, + `E0507 <_> <_> prober.go:239] "Unable to write all bytes from execInContainer" err="short write" <_> actualBytes=10240`, + `E0507 <_> <_> remote_image.go:180] "PullImage from image service failed" err="rpc error: code = NotFound desc = failed to pull and unpack image <_> failed to resolve reference <_> <_> not found" <_>`, + `E0507 <_> <_> remote_image.go:180] "PullImage from image service failed" err="rpc error: code = Unknown desc = failed to pull and unpack image <_> failed to resolve reference <_> unexpected status from HEAD request to <_> 403 Forbidden" <_>`, + `E0507 <_> <_> remote_runtime.go:432] "ContainerStatus from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container <_> not found" <_>`, + `E0507 <_> <_> remote_runtime.go:496] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task <_> not found: not found" <_> cmd=["/bin/hgrun","check"]`, + `I0507 11:59:29.320184 1537502 kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." pod="logs-endpoint-dev-005/kafka-controller-0" secret="" err="secret \"not-needed\" not found"`, + `I0507 11:59:31.815514 2791 azure_credentials.go:220] image(us.gcr.io/hosted-grafana/hosted-grafana-pro) is not from ACR, return empty authentication`, + `I0507 11:59:32.409568 581823 cache.go:40] re-using cached key and certificate`, + `I0507 11:59:33.422254 1537502 kubelet_getters.go:187] "Pod status updated" pod="kube-system/kube-proxy-gke-dev-us-central-0-main-n2s16-3-1dd-9b502d96-x28r" status="Running"`, + `I0507 11:59:34.518822 3224 kuberuntime_container.go:745] "Killing container with a grace period" pod="hosted-grafana/hosted-grafana-api-7b6bd9b949-9csb4" podUID="25cb986c-3d6c-4ed0-abf3-ee59ed6175f9" containerName="hgapi" containerID="containerd://c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e" gracePeriod=30`, + `I0507 11:59:34.834734 3224 reconciler_common.go:172] "operationExecutor.UnmountVolume started for volume \"kube-api-access-95j2t\" (UniqueName: \"kubernetes.io/projected/25cb986c-3d6c-4ed0-abf3-ee59ed6175f9-kube-api-access-95j2t\") pod \"25cb986c-3d6c-4ed0-abf3-ee59ed6175f9\" (UID: \"25cb986c-3d6c-4ed0-abf3-ee59ed6175f9\") "`, + `I0507 11:59:34.834794 3224 reconciler_common.go:172] "operationExecutor.UnmountVolume started for volume \"pdc-certs\" (UniqueName: \"kubernetes.io/secret/25cb986c-3d6c-4ed0-abf3-ee59ed6175f9-pdc-certs\") pod \"25cb986c-3d6c-4ed0-abf3-ee59ed6175f9\" (UID: \"25cb986c-3d6c-4ed0-abf3-ee59ed6175f9\") "`, + `I0507 11:59:34.834835 3224 reconciler_common.go:172] "operationExecutor.UnmountVolume started for volume \"gcs-serviceaccount\" (UniqueName: \"kubernetes.io/secret/25cb986c-3d6c-4ed0-abf3-ee59ed6175f9-gcs-serviceaccount\") pod \"25cb986c-3d6c-4ed0-abf3-ee59ed6175f9\" (UID: \"25cb986c-3d6c-4ed0-abf3-ee59ed6175f9\") "`, + `I0507 11:59:34.836955 3224 operation_generator.go:888] UnmountVolume.TearDown succeeded for volume "kubernetes.io/secret/25cb986c-3d6c-4ed0-abf3-ee59ed6175f9-pdc-certs" (OuterVolumeSpecName: "pdc-certs") pod "25cb986c-3d6c-4ed0-abf3-ee59ed6175f9" (UID: "25cb986c-3d6c-4ed0-abf3-ee59ed6175f9"). InnerVolumeSpecName "pdc-certs". PluginName "kubernetes.io/secret", VolumeGidValue ""`, + `I0507 11:59:34.841404 3224 operation_generator.go:888] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/25cb986c-3d6c-4ed0-abf3-ee59ed6175f9-kube-api-access-95j2t" (OuterVolumeSpecName: "kube-api-access-95j2t") pod "25cb986c-3d6c-4ed0-abf3-ee59ed6175f9" (UID: "25cb986c-3d6c-4ed0-abf3-ee59ed6175f9"). InnerVolumeSpecName "kube-api-access-95j2t". PluginName "kubernetes.io/projected", VolumeGidValue ""`, + `I0507 11:59:34.841447 3224 operation_generator.go:888] UnmountVolume.TearDown succeeded for volume "kubernetes.io/secret/25cb986c-3d6c-4ed0-abf3-ee59ed6175f9-gcs-serviceaccount" (OuterVolumeSpecName: "gcs-serviceaccount") pod "25cb986c-3d6c-4ed0-abf3-ee59ed6175f9" (UID: "25cb986c-3d6c-4ed0-abf3-ee59ed6175f9"). InnerVolumeSpecName "gcs-serviceaccount". PluginName "kubernetes.io/secret", VolumeGidValue ""`, + `I0507 11:59:34.854084 4727 kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." pod="integration/grafana-render-service-cbff479fc-cj9tp" secret="" err="secret \"us-gcr-io-hosted-grafana\" not found"`, + `I0507 11:59:34.936025 3224 reconciler_common.go:300] "Volume detached for volume \"pdc-certs\" (UniqueName: \"kubernetes.io/secret/25cb986c-3d6c-4ed0-abf3-ee59ed6175f9-pdc-certs\") on node \"ip-10-60-2-58.us-east-2.compute.internal\" DevicePath \"\""`, + `I0507 11:59:37.133005 3782 prober.go:107] "Probe failed" probeType="Readiness" pod="loki-dev-014/loki-dev-014-rollout-operator-58fc68b876-2qhmp" podUID="e6504036-2514-4ecc-b78c-c47061f60c9f" containerName="rollout-operator" probeResult="failure" output="HTTP probe failed with statuscode: 500"`, + `I0507 11:59:37.915108 4726 prober.go:107] "Probe failed" probeType="Readiness" pod="agent-management-dev-002/agent-management-api-7ff7b9b9-k9nft" podUID="9893f9ac-f3e4-41fb-8da7-592061d2386c" containerName="agent-management-api" probeResult="failure" output="HTTP probe failed with statuscode: 400"`, + `I0507 11:59:38.116658 2791 azure_credentials.go:220] image(us.gcr.io/hosted-grafana/hg-plugins) is not from ACR, return empty authentication`, + `I0507 11:59:39.168633 2776 kubelet.go:2493] "SyncLoop (probe)" probe="readiness" status="" pod="hosted-grafana/dafdeveuwest2-grafana-7845d969b5-f8h5q"`, + `I0507 11:59:39.560605 4739 kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." pod="logs-endpoint-dev-005/kafka-exporter-766c6757b5-bggf6" secret="" err="secret \"not-needed\" not found"`, + `I0507 <_> 2791 azure_credentials.go:220] image(us.gcr.io/hosted-grafana/hgrun) is not from ACR, return empty authentication`, + `I0507 <_> 3224 reconciler_common.go:300] "Volume detached for volume <_> (UniqueName: <_> on node \"ip-10-60-2-58.us-east-2.compute.internal\" DevicePath \"\""`, + `I0507 <_> 6247 prober.go:107] "Probe failed" probeType="Readiness" pod="grafana-agent/grafana-agent-helm-4" podUID="c36c5200-1cd6-4093-893c-c022f91af996" containerName="grafana-agent" probeResult="failure" output="Get \"http://10.0.99.125:3090/-/ready\": dial tcp 10.0.99.125:3090: connect: connection refused"`, + `I0507 <_> <_> <_> "Cleaned up orphaned pod volumes dir" <_> <_>`, + `I0507 <_> <_> <_> "SyncLoop (PLEG): event for pod" <_> <_>`, + `I0507 <_> <_> <_> "SyncLoop DELETE" source="api" <_>`, + `I0507 <_> <_> <_> "SyncLoop REMOVE" source="api" <_>`, + `I0507 <_> <_> generic.go:334] "Generic (PLEG): container finished" <_> <_> exitCode=1`, + `I0507 <_> <_> kubelet.go:2498] "SyncLoop (probe)" probe="liveness" status="unhealthy" <_>`, + `I0507 <_> <_> kubelet.go:2498] "SyncLoop (probe)" probe="readiness" status="ready" <_>`, + `I0507 <_> <_> kubelet_getters.go:187] "Pod status updated" <_> status="Running"`, + `I0507 <_> <_> kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." <_> secret="" err="secret \"dockerhub\" not found"`, + `I0507 <_> <_> kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." <_> secret="" err="secret \"gcr\" not found"`, + `I0507 <_> <_> pod_container_deletor.go:53] "DeleteContainer returned error" <_> err="failed to get container status <_> rpc error: code = NotFound desc = an error occurred when try to find container <_> not found"`, + `I0507 <_> <_> prober.go:107] "Probe failed" probeType="Readiness" <_> <_> containerName="grafana" probeResult="failure" output=<`, + `I0507 <_> <_> scope.go:117] "RemoveContainer" <_>`, + `I0507 <_> <_> cache.go:40] re-using cached key and certificate`, + `IPv4: martian source <_> from <_> on dev eth0`, + `PRC: Renewing lease on eth0.`, + `RCV: Reply message on eth0 from fe80::e9:7eff:fedf:3d37.`, + `Removed slice libcontainer container kubepods-burstable-pod25cb986c_3d6c_4ed0_abf3_ee59ed6175f9.slice.`, + `Started cri-containerd-95bf586cd79d43120ff44582d4dbd2476de61744411f8515b9b2c527a41fd5d9.scope.`, + `Started libcontainer container <_>`, + `XMT: Renew on eth0, interval 9700ms.`, + `XMT: Solicit on eth0, interval <_>`, + `audit: type=1400 <_> apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" <_> comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined"`, + `kauditd_printk_skb: <_> callbacks suppressed`, + `ll header: 00000000: 42 01 0a 80 00 <_> 42 01 0a 80 00 01 08 00`, + `net_ratelimit: 2 callbacks suppressed`, + `time="2024-05-07T11:59:34.519591759Z" level=info msg="StopContainer for \"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\" with timeout 30 (s)"`, + `time="2024-05-07T11:59:34.520032214Z" level=info msg="Stop container \"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\" with signal terminated"`, + `time="2024-05-07T11:59:34.591282703Z" level=info msg="StopContainer for \"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\" returns successfully"`, + `time="2024-05-07T11:59:34.592005066Z" level=info msg="StopPodSandbox for \"c605ad2cdc74c6b5288f2532ad71cce81a28ef6965f97a89ff6609deb825553a\""`, + `time="2024-05-07T11:59:34.592084495Z" level=info msg="Container to stop \"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\" must be in running or unknown state, current state \"CONTAINER_EXITED\""`, + `time="2024-05-07T11:59:34.706960850Z" level=info msg="TearDown network for sandbox \"c605ad2cdc74c6b5288f2532ad71cce81a28ef6965f97a89ff6609deb825553a\" successfully"`, + `time="2024-05-07T11:59:34.707025668Z" level=info msg="StopPodSandbox for \"c605ad2cdc74c6b5288f2532ad71cce81a28ef6965f97a89ff6609deb825553a\" returns successfully"`, + `time="2024-05-07T11:59:38.484586527Z" level=error msg="Failed to delete exec process \"d9e0a1867ce73695ad859f2b0a76fe8f5053db8a5e49142d747e53a445729bd4\" for container \"6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408\"" error="ttrpc: closed: unknown"`, }, }, { @@ -198,23 +255,23 @@ func TestDrain_TrainExtractsPatterns(t *testing.T) { drain: New(DefaultConfig(), nil), inputFile: "testdata/kafka.txt", patterns: []string{ - `[2024-05-07 <_> INFO [LocalLog partition=mimir-dev-09-aggregations-offsets-0, dir=/bitnami/kafka/data] Deleting segment files <_> size=948, <_> <_> (kafka.log.LocalLog$)`, - `[2024-05-07 <_> INFO [UnifiedLog <_> dir=/bitnami/kafka/data] Deleting segments due to log start offset <_> breach: <_> size=948, <_> <_> size=948, <_> <_> (kafka.log.UnifiedLog)`, - `[2024-05-07 <_> INFO [UnifiedLog <_> dir=/bitnami/kafka/data] Incremented log start offset to <_> due to leader offset increment (kafka.log.UnifiedLog)`, - `[2024-05-07 <_> INFO [UnifiedLog <_> dir=/bitnami/kafka/data] Deleting segments due to log start offset <_> breach: <_> <_> <_> <_> (kafka.log.UnifiedLog)`, - `[2024-05-07 <_> INFO [UnifiedLog <_> dir=/bitnami/kafka/data] Incremented log start offset to <_> due to segment deletion (kafka.log.UnifiedLog)`, - `[2024-05-07 <_> INFO [UnifiedLog <_> dir=/bitnami/kafka/data] Deleting segment <_> <_> <_> <_> due to retention size <_> breach. Log size after deletion will be <_> (kafka.log.UnifiedLog)`, - `[2024-05-07 <_> INFO [ProducerStateManager <_> Wrote producer snapshot at offset <_> with 0 producer ids in <_> ms. (kafka.log.ProducerStateManager)`, - `[2024-05-07 10:55:53,038] INFO [LocalLog partition=mimir-dev-09-aggregations-offsets-1, dir=/bitnami/kafka/data] Deleting segment files LogSegment(baseOffset=447957, size=948, lastModifiedTime=1715059232052, largestRecordTimestamp=Some(1715059232002)),LogSegment(baseOffset=447969, size=948, lastModifiedTime=1715059424352, largestRecordTimestamp=Some(1715059424301)) (kafka.log.LocalLog$)`, - `[2024-05-07 <_> INFO [LocalLog <_> dir=/bitnami/kafka/data] Rolled new log segment at offset <_> in <_> ms. (kafka.log.LocalLog)`, - `[2024-05-07 10:55:40,638] INFO [LocalLog partition=ingest-6, dir=/bitnami/kafka/data] Deleting segment files LogSegment(baseOffset=180400817, size=16997594, lastModifiedTime=1715075775780, largestRecordTimestamp=Some(1715075775771)),LogSegment(baseOffset=180403261, size=16992344, lastModifiedTime=1715075781053, largestRecordTimestamp=Some(1715075781021)),LogSegment(baseOffset=180405723, size=16989895, lastModifiedTime=1715075786205, largestRecordTimestamp=Some(1715075786174)),LogSegment(baseOffset=180408118, size=16998698, lastModifiedTime=1715075791681, largestRecordTimestamp=Some(1715075791673)),LogSegment(baseOffset=180410608, size=16995676, lastModifiedTime=1715075796438, largestRecordTimestamp=Some(1715075796430)),LogSegment(baseOffset=180412733, size=16963278, lastModifiedTime=1715075800534, largestRecordTimestamp=Some(1715075800511)),LogSegment(baseOffset=180414883, size=16984328, lastModifiedTime=1715075805272, largestRecordTimestamp=Some(1715075805230)),LogSegment(baseOffset=180417063, size=16989109, lastModifiedTime=1715075810381, largestRecordTimestamp=Some(1715075810372)),LogSegment(baseOffset=180419267, size=16996871, lastModifiedTime=1715075815153, largestRecordTimestamp=Some(1715075815125)),LogSegment(baseOffset=180421560, size=16988558, lastModifiedTime=1715075819785, largestRecordTimestamp=Some(1715075819763)),LogSegment(baseOffset=180424008, size=16999292, lastModifiedTime=1715075825336, largestRecordTimestamp=Some(1715075825303)),LogSegment(baseOffset=180426459, size=16990595, lastModifiedTime=1715075830839, largestRecordTimestamp=Some(1715075830827)),LogSegment(baseOffset=180428944, size=16995859, lastModifiedTime=1715075835942, largestRecordTimestamp=Some(1715075835904)),LogSegment(baseOffset=180431327, size=16992294, lastModifiedTime=1715075841219, largestRecordTimestamp=Some(1715075841214)),LogSegment(baseOffset=180433867, size=16966736, lastModifiedTime=1715075846443, largestRecordTimestamp=Some(1715075846401)),LogSegment(baseOffset=180436204, size=16894731, lastModifiedTime=1715075853273, largestRecordTimestamp=Some(1715075853244)),LogSegment(baseOffset=180438984, size=16983529, lastModifiedTime=1715075858911, largestRecordTimestamp=Some(1715075858891)),LogSegment(baseOffset=180441466, size=16996933, lastModifiedTime=1715075863566, largestRecordTimestamp=Some(1715075863554)),LogSegment(baseOffset=180443778, size=16999841, lastModifiedTime=1715075866199, largestRecordTimestamp=Some(1715075866185)),LogSegment(baseOffset=180445367, size=16992471, lastModifiedTime=1715075870385, largestRecordTimestamp=Some(1715075870347)),LogSegment(baseOffset=180447366, size=16999996, lastModifiedTime=1715075875102, largestRecordTimestamp=Some(1715075875091)),LogSegment(baseOffset=180449601, size=16994426, lastModifiedTime=1715075879927, largestRecordTimestamp=Some(1715075879926)),LogSegment(baseOffset=180452079, size=16998020, lastModifiedTime=1715075885293, largestRecordTimestamp=Some(1715075885263)),LogSegment(baseOffset=180454546, size=16992231, lastModifiedTime=1715075890424, largestRecordTimestamp=Some(1715075890409)),LogSegment(baseOffset=180456986, size=16970315, lastModifiedTime=1715075895719, largestRecordTimestamp=Some(1715075895690)),LogSegment(baseOffset=180459366, size=16990785, lastModifiedTime=1715075900996, largestRecordTimestamp=Some(1715075900985)),LogSegment(baseOffset=180461885, size=16996655, lastModifiedTime=1715075905847, largestRecordTimestamp=Some(1715075905841)),LogSegment(baseOffset=180464299, size=16982181, lastModifiedTime=1715075911052, largestRecordTimestamp=Some(1715075911028)),LogSegment(baseOffset=180466821, size=16997630, lastModifiedTime=1715075915962, largestRecordTimestamp=Some(1715075915953)),LogSegment(baseOffset=180468968, size=16995723, lastModifiedTime=1715075920325, largestRecordTimestamp=Some(1715075920308)),LogSegment(baseOffset=180471046, size=16979316, lastModifiedTime=1715075924724, largestRecordTimestamp=Some(1715075924697)),LogSegment(baseOffset=180473259, size=16995238, lastModifiedTime=1715075929645, largestRecordTimestamp=Some(1715075929624)),LogSegment(baseOffset=180475486, size=16988461, lastModifiedTime=1715075934288, largestRecordTimestamp=Some(1715075934283)),LogSegment(baseOffset=180477735, size=16993767, lastModifiedTime=1715075939277, largestRecordTimestamp=Some(1715075939270)),LogSegment(baseOffset=180480095, size=16995409, lastModifiedTime=1715075944639, largestRecordTimestamp=Some(1715075944635)),LogSegment(baseOffset=180482560, size=16992784, lastModifiedTime=1715075949760, largestRecordTimestamp=Some(1715075949760)),LogSegment(baseOffset=180484967, size=16990838, lastModifiedTime=1715075954937, largestRecordTimestamp=Some(1715075954929)),LogSegment(baseOffset=180487377, size=16976794, lastModifiedTime=1715075960151, largestRecordTimestamp=Some(1715075960119)),LogSegment(baseOffset=180489919, size=16997379, lastModifiedTime=1715075965116, largestRecordTimestamp=Some(1715075965085)),LogSegment(baseOffset=180492304, size=16956613, lastModifiedTime=1715075970448, largestRecordTimestamp=Some(1715075970424)),LogSegment(baseOffset=180494832, size=16895640, lastModifiedTime=1715075975354, largestRecordTimestamp=Some(1715075975341)),LogSegment(baseOffset=180496930, size=16998328, lastModifiedTime=1715075979813, largestRecordTimestamp=Some(1715075979796)),LogSegment(baseOffset=180499079, size=16995699, lastModifiedTime=1715075984309, largestRecordTimestamp=Some(1715075984285)),LogSegment(baseOffset=180501183, size=16993785, lastModifiedTime=1715075989086, largestRecordTimestamp=Some(1715075989064)),LogSegment(baseOffset=180503431, size=16989600, lastModifiedTime=1715075993713, largestRecordTimestamp=Some(1715075993683)),LogSegment(baseOffset=180505674, size=16984790, lastModifiedTime=1715075998337, largestRecordTimestamp=Some(1715075998318)),LogSegment(baseOffset=180508022, size=16982630, lastModifiedTime=1715076003671, largestRecordTimestamp=Some(1715076003660)),LogSegment(baseOffset=180510439, size=16999488, lastModifiedTime=1715076009000, largestRecordTimestamp=Some(1715076008996)),LogSegment(baseOffset=180512848, size=16997845, lastModifiedTime=1715076014033, largestRecordTimestamp=Some(1715076014032)),LogSegment(baseOffset=180515281, size=16990661, lastModifiedTime=1715076019245, largestRecordTimestamp=Some(1715076019216)),LogSegment(baseOffset=180517815, size=16996244, lastModifiedTime=1715076023989, largestRecordTimestamp=Some(1715076023963)),LogSegment(baseOffset=180520112, size=16992012, lastModifiedTime=1715076029243, largestRecordTimestamp=Some(1715076029231)) (kafka.log.LocalLog$)`, - `[2024-05-07 10:55:40,626] INFO [LocalLog partition=ingest-6, dir=/bitnami/kafka/data] Deleting segment files LogSegment(baseOffset=180391157, size=16991045, lastModifiedTime=1715075754780, largestRecordTimestamp=Some(1715075754774)),LogSegment(baseOffset=180393429, size=16997692, lastModifiedTime=1715075760206, largestRecordTimestamp=Some(1715075760186)),LogSegment(baseOffset=180395889, size=16998200, lastModifiedTime=1715075765542, largestRecordTimestamp=Some(1715075765526)),LogSegment(baseOffset=180398373, size=16977347, lastModifiedTime=1715075770515, largestRecordTimestamp=Some(1715075770504)) (kafka.log.LocalLog$)`, `[2024-05-07 10:55:40,559] INFO [LocalLog partition=ingest-7, dir=/bitnami/kafka/data] Deleting segment files LogSegment(baseOffset=179133378, size=16987985, lastModifiedTime=1715075760072, largestRecordTimestamp=Some(1715075760047)),LogSegment(baseOffset=179135832, size=16999459, lastModifiedTime=1715075765431, largestRecordTimestamp=Some(1715075765398)),LogSegment(baseOffset=179138321, size=16994485, lastModifiedTime=1715075770425, largestRecordTimestamp=Some(1715075770404)),LogSegment(baseOffset=179140761, size=16996810, lastModifiedTime=1715075775622, largestRecordTimestamp=Some(1715075775619)),LogSegment(baseOffset=179143198, size=16998520, lastModifiedTime=1715075780912, largestRecordTimestamp=Some(1715075780889)),LogSegment(baseOffset=179145674, size=16988474, lastModifiedTime=1715075786051, largestRecordTimestamp=Some(1715075786030)),LogSegment(baseOffset=179148084, size=16956099, lastModifiedTime=1715075791514, largestRecordTimestamp=Some(1715075791486)),LogSegment(baseOffset=179150568, size=16995476, lastModifiedTime=1715075796360, largestRecordTimestamp=Some(1715075796329)),LogSegment(baseOffset=179152727, size=16993313, lastModifiedTime=1715075800440, largestRecordTimestamp=Some(1715075800430)),LogSegment(baseOffset=179154861, size=16992142, lastModifiedTime=1715075805147, largestRecordTimestamp=Some(1715075805135)),LogSegment(baseOffset=179157056, size=16999919, lastModifiedTime=1715075810155, largestRecordTimestamp=Some(1715075810153)),LogSegment(baseOffset=179159230, size=16995021, lastModifiedTime=1715075815018, largestRecordTimestamp=Some(1715075815016)),LogSegment(baseOffset=179161550, size=16966526, lastModifiedTime=1715075819528, largestRecordTimestamp=Some(1715075819521)),LogSegment(baseOffset=179163962, size=16990848, lastModifiedTime=1715075825066, largestRecordTimestamp=Some(1715075825042)),LogSegment(baseOffset=179166414, size=16997833, lastModifiedTime=1715075830662, largestRecordTimestamp=Some(1715075830656)),LogSegment(baseOffset=179168915, size=16992619, lastModifiedTime=1715075835771, largestRecordTimestamp=Some(1715075835741)),LogSegment(baseOffset=179171302, size=16999091, lastModifiedTime=1715075841031, largestRecordTimestamp=Some(1715075841022)),LogSegment(baseOffset=179173853, size=16993953, lastModifiedTime=1715075846197, largestRecordTimestamp=Some(1715075846181)),LogSegment(baseOffset=179176191, size=16997479, lastModifiedTime=1715075853192, largestRecordTimestamp=Some(1715075853172)),LogSegment(baseOffset=179179037, size=16997174, lastModifiedTime=1715075858693, largestRecordTimestamp=Some(1715075858682)),LogSegment(baseOffset=179181478, size=16986004, lastModifiedTime=1715075863400, largestRecordTimestamp=Some(1715075863396)),LogSegment(baseOffset=179183786, size=16995316, lastModifiedTime=1715075866123, largestRecordTimestamp=Some(1715075866112)),LogSegment(baseOffset=179185434, size=16990492, lastModifiedTime=1715075870154, largestRecordTimestamp=Some(1715075870146)),LogSegment(baseOffset=179187398, size=16999541, lastModifiedTime=1715075874980, largestRecordTimestamp=Some(1715075874961)),LogSegment(baseOffset=179189664, size=16987383, lastModifiedTime=1715075879670, largestRecordTimestamp=Some(1715075879639)),LogSegment(baseOffset=179192076, size=16991701, lastModifiedTime=1715075885010, largestRecordTimestamp=Some(1715075884995)),LogSegment(baseOffset=179194546, size=16989109, lastModifiedTime=1715075890220, largestRecordTimestamp=Some(1715075890208)),LogSegment(baseOffset=179197009, size=16962782, lastModifiedTime=1715075895466, largestRecordTimestamp=Some(1715075895456)),LogSegment(baseOffset=179199373, size=16974715, lastModifiedTime=1715075900757, largestRecordTimestamp=Some(1715075900746)),LogSegment(baseOffset=179201897, size=16993973, lastModifiedTime=1715075905639, largestRecordTimestamp=Some(1715075905638)),LogSegment(baseOffset=179204346, size=16979828, lastModifiedTime=1715075910798, largestRecordTimestamp=Some(1715075910782)),LogSegment(baseOffset=179206836, size=16992092, lastModifiedTime=1715075915638, largestRecordTimestamp=Some(1715075915632)),LogSegment(baseOffset=179208986, size=16988849, lastModifiedTime=1715075920193, largestRecordTimestamp=Some(1715075920176)),LogSegment(baseOffset=179211133, size=16989206, lastModifiedTime=1715075924352, largestRecordTimestamp=Some(1715075924338)),LogSegment(baseOffset=179213268, size=16989737, lastModifiedTime=1715075929343, largestRecordTimestamp=Some(1715075929332)),LogSegment(baseOffset=179215514, size=16997903, lastModifiedTime=1715075934074, largestRecordTimestamp=Some(1715075934056)),LogSegment(baseOffset=179217793, size=16995100, lastModifiedTime=1715075938937, largestRecordTimestamp=Some(1715075938925)),LogSegment(baseOffset=179220122, size=16981574, lastModifiedTime=1715075944296, largestRecordTimestamp=Some(1715075944288)),LogSegment(baseOffset=179222600, size=16999794, lastModifiedTime=1715075949454, largestRecordTimestamp=Some(1715075949432)),LogSegment(baseOffset=179224988, size=16998870, lastModifiedTime=1715075954567, largestRecordTimestamp=Some(1715075954544)),LogSegment(baseOffset=179227402, size=16986053, lastModifiedTime=1715075959815, largestRecordTimestamp=Some(1715075959813)),LogSegment(baseOffset=179229948, size=16999937, lastModifiedTime=1715075964787, largestRecordTimestamp=Some(1715075964779)),LogSegment(baseOffset=179232368, size=16992995, lastModifiedTime=1715075970109, largestRecordTimestamp=Some(1715075970096)),LogSegment(baseOffset=179234885, size=16995271, lastModifiedTime=1715075975078, largestRecordTimestamp=Some(1715075975066)),LogSegment(baseOffset=179237038, size=16987833, lastModifiedTime=1715075979534, largestRecordTimestamp=Some(1715075979499)),LogSegment(baseOffset=179239147, size=16844618, lastModifiedTime=1715075984150, largestRecordTimestamp=Some(1715075984139)),LogSegment(baseOffset=179241334, size=16968482, lastModifiedTime=1715075988727, largestRecordTimestamp=Some(1715075988700)),LogSegment(baseOffset=179243472, size=16991395, lastModifiedTime=1715075993359, largestRecordTimestamp=Some(1715075993333)),LogSegment(baseOffset=179245756, size=16985926, lastModifiedTime=1715075998010, largestRecordTimestamp=Some(1715075998005)),LogSegment(baseOffset=179248096, size=16948574, lastModifiedTime=1715076003328, largestRecordTimestamp=Some(1715076003298)),LogSegment(baseOffset=179250530, size=16986047, lastModifiedTime=1715076008650, largestRecordTimestamp=Some(1715076008628)),LogSegment(baseOffset=179252915, size=16998875, lastModifiedTime=1715076013551, largestRecordTimestamp=Some(1715076013516)),LogSegment(baseOffset=179255312, size=16997990, lastModifiedTime=1715076018832, largestRecordTimestamp=Some(1715076018797)),LogSegment(baseOffset=179257861, size=16999525, lastModifiedTime=1715076023621, largestRecordTimestamp=Some(1715076023601)),LogSegment(baseOffset=179260226, size=16997755, lastModifiedTime=1715076028814, largestRecordTimestamp=Some(1715076028800)),LogSegment(baseOffset=179262715, size=16981492, lastModifiedTime=1715076034150, largestRecordTimestamp=Some(1715076034140)),LogSegment(baseOffset=179265040, size=16998332, lastModifiedTime=1715076038676, largestRecordTimestamp=Some(1715076038657)) (kafka.log.LocalLog$)`, + `[2024-05-07 10:55:40,626] INFO [LocalLog partition=ingest-6, dir=/bitnami/kafka/data] Deleting segment files LogSegment(baseOffset=180391157, size=16991045, lastModifiedTime=1715075754780, largestRecordTimestamp=Some(1715075754774)),LogSegment(baseOffset=180393429, size=16997692, lastModifiedTime=1715075760206, largestRecordTimestamp=Some(1715075760186)),LogSegment(baseOffset=180395889, size=16998200, lastModifiedTime=1715075765542, largestRecordTimestamp=Some(1715075765526)),LogSegment(baseOffset=180398373, size=16977347, lastModifiedTime=1715075770515, largestRecordTimestamp=Some(1715075770504)) (kafka.log.LocalLog$)`, + `[2024-05-07 10:55:40,638] INFO [LocalLog partition=ingest-6, dir=/bitnami/kafka/data] Deleting segment files LogSegment(baseOffset=180400817, size=16997594, lastModifiedTime=1715075775780, largestRecordTimestamp=Some(1715075775771)),LogSegment(baseOffset=180403261, size=16992344, lastModifiedTime=1715075781053, largestRecordTimestamp=Some(1715075781021)),LogSegment(baseOffset=180405723, size=16989895, lastModifiedTime=1715075786205, largestRecordTimestamp=Some(1715075786174)),LogSegment(baseOffset=180408118, size=16998698, lastModifiedTime=1715075791681, largestRecordTimestamp=Some(1715075791673)),LogSegment(baseOffset=180410608, size=16995676, lastModifiedTime=1715075796438, largestRecordTimestamp=Some(1715075796430)),LogSegment(baseOffset=180412733, size=16963278, lastModifiedTime=1715075800534, largestRecordTimestamp=Some(1715075800511)),LogSegment(baseOffset=180414883, size=16984328, lastModifiedTime=1715075805272, largestRecordTimestamp=Some(1715075805230)),LogSegment(baseOffset=180417063, size=16989109, lastModifiedTime=1715075810381, largestRecordTimestamp=Some(1715075810372)),LogSegment(baseOffset=180419267, size=16996871, lastModifiedTime=1715075815153, largestRecordTimestamp=Some(1715075815125)),LogSegment(baseOffset=180421560, size=16988558, lastModifiedTime=1715075819785, largestRecordTimestamp=Some(1715075819763)),LogSegment(baseOffset=180424008, size=16999292, lastModifiedTime=1715075825336, largestRecordTimestamp=Some(1715075825303)),LogSegment(baseOffset=180426459, size=16990595, lastModifiedTime=1715075830839, largestRecordTimestamp=Some(1715075830827)),LogSegment(baseOffset=180428944, size=16995859, lastModifiedTime=1715075835942, largestRecordTimestamp=Some(1715075835904)),LogSegment(baseOffset=180431327, size=16992294, lastModifiedTime=1715075841219, largestRecordTimestamp=Some(1715075841214)),LogSegment(baseOffset=180433867, size=16966736, lastModifiedTime=1715075846443, largestRecordTimestamp=Some(1715075846401)),LogSegment(baseOffset=180436204, size=16894731, lastModifiedTime=1715075853273, largestRecordTimestamp=Some(1715075853244)),LogSegment(baseOffset=180438984, size=16983529, lastModifiedTime=1715075858911, largestRecordTimestamp=Some(1715075858891)),LogSegment(baseOffset=180441466, size=16996933, lastModifiedTime=1715075863566, largestRecordTimestamp=Some(1715075863554)),LogSegment(baseOffset=180443778, size=16999841, lastModifiedTime=1715075866199, largestRecordTimestamp=Some(1715075866185)),LogSegment(baseOffset=180445367, size=16992471, lastModifiedTime=1715075870385, largestRecordTimestamp=Some(1715075870347)),LogSegment(baseOffset=180447366, size=16999996, lastModifiedTime=1715075875102, largestRecordTimestamp=Some(1715075875091)),LogSegment(baseOffset=180449601, size=16994426, lastModifiedTime=1715075879927, largestRecordTimestamp=Some(1715075879926)),LogSegment(baseOffset=180452079, size=16998020, lastModifiedTime=1715075885293, largestRecordTimestamp=Some(1715075885263)),LogSegment(baseOffset=180454546, size=16992231, lastModifiedTime=1715075890424, largestRecordTimestamp=Some(1715075890409)),LogSegment(baseOffset=180456986, size=16970315, lastModifiedTime=1715075895719, largestRecordTimestamp=Some(1715075895690)),LogSegment(baseOffset=180459366, size=16990785, lastModifiedTime=1715075900996, largestRecordTimestamp=Some(1715075900985)),LogSegment(baseOffset=180461885, size=16996655, lastModifiedTime=1715075905847, largestRecordTimestamp=Some(1715075905841)),LogSegment(baseOffset=180464299, size=16982181, lastModifiedTime=1715075911052, largestRecordTimestamp=Some(1715075911028)),LogSegment(baseOffset=180466821, size=16997630, lastModifiedTime=1715075915962, largestRecordTimestamp=Some(1715075915953)),LogSegment(baseOffset=180468968, size=16995723, lastModifiedTime=1715075920325, largestRecordTimestamp=Some(1715075920308)),LogSegment(baseOffset=180471046, size=16979316, lastModifiedTime=1715075924724, largestRecordTimestamp=Some(1715075924697)),LogSegment(baseOffset=180473259, size=16995238, lastModifiedTime=1715075929645, largestRecordTimestamp=Some(1715075929624)),LogSegment(baseOffset=180475486, size=16988461, lastModifiedTime=1715075934288, largestRecordTimestamp=Some(1715075934283)),LogSegment(baseOffset=180477735, size=16993767, lastModifiedTime=1715075939277, largestRecordTimestamp=Some(1715075939270)),LogSegment(baseOffset=180480095, size=16995409, lastModifiedTime=1715075944639, largestRecordTimestamp=Some(1715075944635)),LogSegment(baseOffset=180482560, size=16992784, lastModifiedTime=1715075949760, largestRecordTimestamp=Some(1715075949760)),LogSegment(baseOffset=180484967, size=16990838, lastModifiedTime=1715075954937, largestRecordTimestamp=Some(1715075954929)),LogSegment(baseOffset=180487377, size=16976794, lastModifiedTime=1715075960151, largestRecordTimestamp=Some(1715075960119)),LogSegment(baseOffset=180489919, size=16997379, lastModifiedTime=1715075965116, largestRecordTimestamp=Some(1715075965085)),LogSegment(baseOffset=180492304, size=16956613, lastModifiedTime=1715075970448, largestRecordTimestamp=Some(1715075970424)),LogSegment(baseOffset=180494832, size=16895640, lastModifiedTime=1715075975354, largestRecordTimestamp=Some(1715075975341)),LogSegment(baseOffset=180496930, size=16998328, lastModifiedTime=1715075979813, largestRecordTimestamp=Some(1715075979796)),LogSegment(baseOffset=180499079, size=16995699, lastModifiedTime=1715075984309, largestRecordTimestamp=Some(1715075984285)),LogSegment(baseOffset=180501183, size=16993785, lastModifiedTime=1715075989086, largestRecordTimestamp=Some(1715075989064)),LogSegment(baseOffset=180503431, size=16989600, lastModifiedTime=1715075993713, largestRecordTimestamp=Some(1715075993683)),LogSegment(baseOffset=180505674, size=16984790, lastModifiedTime=1715075998337, largestRecordTimestamp=Some(1715075998318)),LogSegment(baseOffset=180508022, size=16982630, lastModifiedTime=1715076003671, largestRecordTimestamp=Some(1715076003660)),LogSegment(baseOffset=180510439, size=16999488, lastModifiedTime=1715076009000, largestRecordTimestamp=Some(1715076008996)),LogSegment(baseOffset=180512848, size=16997845, lastModifiedTime=1715076014033, largestRecordTimestamp=Some(1715076014032)),LogSegment(baseOffset=180515281, size=16990661, lastModifiedTime=1715076019245, largestRecordTimestamp=Some(1715076019216)),LogSegment(baseOffset=180517815, size=16996244, lastModifiedTime=1715076023989, largestRecordTimestamp=Some(1715076023963)),LogSegment(baseOffset=180520112, size=16992012, lastModifiedTime=1715076029243, largestRecordTimestamp=Some(1715076029231)) (kafka.log.LocalLog$)`, `[2024-05-07 10:55:40,713] INFO [LocalLog partition=ingest-3, dir=/bitnami/kafka/data] Deleting segment files LogSegment(baseOffset=182526165, size=16998661, lastModifiedTime=1715075758062, largestRecordTimestamp=Some(1715075758061)),LogSegment(baseOffset=182528560, size=16999718, lastModifiedTime=1715075763583, largestRecordTimestamp=Some(1715075763577)),LogSegment(baseOffset=182531056, size=16994792, lastModifiedTime=1715075768711, largestRecordTimestamp=Some(1715075768697)),LogSegment(baseOffset=182533514, size=16987578, lastModifiedTime=1715075773552, largestRecordTimestamp=Some(1715075773536)),LogSegment(baseOffset=182535953, size=16987705, lastModifiedTime=1715075779055, largestRecordTimestamp=Some(1715075779046)),LogSegment(baseOffset=182538482, size=16997466, lastModifiedTime=1715075784005, largestRecordTimestamp=Some(1715075784004)),LogSegment(baseOffset=182540856, size=16981250, lastModifiedTime=1715075789523, largestRecordTimestamp=Some(1715075789487)),LogSegment(baseOffset=182543386, size=16980484, lastModifiedTime=1715075794637, largestRecordTimestamp=Some(1715075794632)),LogSegment(baseOffset=182545622, size=16999738, lastModifiedTime=1715075799008, largestRecordTimestamp=Some(1715075799000)),LogSegment(baseOffset=182547827, size=16872695, lastModifiedTime=1715075803273, largestRecordTimestamp=Some(1715075803251)),LogSegment(baseOffset=182550001, size=16999890, lastModifiedTime=1715075808368, largestRecordTimestamp=Some(1715075808355)),LogSegment(baseOffset=182552113, size=16959982, lastModifiedTime=1715075813294, largestRecordTimestamp=Some(1715075813293)),LogSegment(baseOffset=182554415, size=16988073, lastModifiedTime=1715075817816, largestRecordTimestamp=Some(1715075817783)),LogSegment(baseOffset=182556814, size=16974731, lastModifiedTime=1715075823018, largestRecordTimestamp=Some(1715075823016)),LogSegment(baseOffset=182559282, size=16996090, lastModifiedTime=1715075828672, largestRecordTimestamp=Some(1715075828632)),LogSegment(baseOffset=182561708, size=16999327, lastModifiedTime=1715075833742, largestRecordTimestamp=Some(1715075833709)),LogSegment(baseOffset=182564173, size=16992947, lastModifiedTime=1715075839121, largestRecordTimestamp=Some(1715075839114)),LogSegment(baseOffset=182566740, size=16982572, lastModifiedTime=1715075844268, largestRecordTimestamp=Some(1715075844254)),LogSegment(baseOffset=182569086, size=16994786, lastModifiedTime=1715075850659, largestRecordTimestamp=Some(1715075850642)),LogSegment(baseOffset=182571815, size=16998391, lastModifiedTime=1715075856704, largestRecordTimestamp=Some(1715075856684)),LogSegment(baseOffset=182574372, size=16994403, lastModifiedTime=1715075861956, largestRecordTimestamp=Some(1715075861922)),LogSegment(baseOffset=182576828, size=16984546, lastModifiedTime=1715075865194, largestRecordTimestamp=Some(1715075865180)),LogSegment(baseOffset=182578716, size=16987846, lastModifiedTime=1715075868470, largestRecordTimestamp=Some(1715075868460)),LogSegment(baseOffset=182580437, size=16958237, lastModifiedTime=1715075873168, largestRecordTimestamp=Some(1715075873151)),LogSegment(baseOffset=182582637, size=16999432, lastModifiedTime=1715075877858, largestRecordTimestamp=Some(1715075877850)),LogSegment(baseOffset=182585006, size=16938567, lastModifiedTime=1715075882952, largestRecordTimestamp=Some(1715075882938)),LogSegment(baseOffset=182587493, size=16998214, lastModifiedTime=1715075888306, largestRecordTimestamp=Some(1715075888285)),LogSegment(baseOffset=182589965, size=16996264, lastModifiedTime=1715075893370, largestRecordTimestamp=Some(1715075893365)),LogSegment(baseOffset=182592327, size=16991650, lastModifiedTime=1715075898806, largestRecordTimestamp=Some(1715075898802)),LogSegment(baseOffset=182594863, size=16998234, lastModifiedTime=1715075903737, largestRecordTimestamp=Some(1715075903733)),LogSegment(baseOffset=182597289, size=16996241, lastModifiedTime=1715075908805, largestRecordTimestamp=Some(1715075908797)),LogSegment(baseOffset=182599811, size=16993657, lastModifiedTime=1715075913918, largestRecordTimestamp=Some(1715075913915)),LogSegment(baseOffset=182602171, size=16993112, lastModifiedTime=1715075918570, largestRecordTimestamp=Some(1715075918570)),LogSegment(baseOffset=182604245, size=16959963, lastModifiedTime=1715075922720, largestRecordTimestamp=Some(1715075922714)),LogSegment(baseOffset=182606451, size=16998518, lastModifiedTime=1715075927490, largestRecordTimestamp=Some(1715075927484)),LogSegment(baseOffset=182608616, size=16999103, lastModifiedTime=1715075932207, largestRecordTimestamp=Some(1715075932188)),LogSegment(baseOffset=182610888, size=16999389, lastModifiedTime=1715075937118, largestRecordTimestamp=Some(1715075937103)),LogSegment(baseOffset=182613221, size=16982597, lastModifiedTime=1715075942170, largestRecordTimestamp=Some(1715075942153)),LogSegment(baseOffset=182615634, size=16986904, lastModifiedTime=1715075947544, largestRecordTimestamp=Some(1715075947541)),LogSegment(baseOffset=182618074, size=16998820, lastModifiedTime=1715075952370, largestRecordTimestamp=Some(1715075952351)),LogSegment(baseOffset=182620446, size=16985066, lastModifiedTime=1715075957884, largestRecordTimestamp=Some(1715075957865)),LogSegment(baseOffset=182623007, size=16998235, lastModifiedTime=1715075963030, largestRecordTimestamp=Some(1715075963008)),LogSegment(baseOffset=182625520, size=16987568, lastModifiedTime=1715075967944, largestRecordTimestamp=Some(1715075967934)),LogSegment(baseOffset=182627921, size=16997118, lastModifiedTime=1715075973216, largestRecordTimestamp=Some(1715075973204)),LogSegment(baseOffset=182630290, size=16978465, lastModifiedTime=1715075978064, largestRecordTimestamp=Some(1715075978053)),LogSegment(baseOffset=182632463, size=16901644, lastModifiedTime=1715075982228, largestRecordTimestamp=Some(1715075982211)),LogSegment(baseOffset=182634546, size=16992477, lastModifiedTime=1715075986935, largestRecordTimestamp=Some(1715075986914)),LogSegment(baseOffset=182636738, size=16951087, lastModifiedTime=1715075991658, largestRecordTimestamp=Some(1715075991636)),LogSegment(baseOffset=182639001, size=16994471, lastModifiedTime=1715075996281, largestRecordTimestamp=Some(1715075996266)),LogSegment(baseOffset=182641298, size=16995754, lastModifiedTime=1715076001319, largestRecordTimestamp=Some(1715076001269)),LogSegment(baseOffset=182643712, size=16992752, lastModifiedTime=1715076006604, largestRecordTimestamp=Some(1715076006583)),LogSegment(baseOffset=182646095, size=16992944, lastModifiedTime=1715076011511, largestRecordTimestamp=Some(1715076011470)),LogSegment(baseOffset=182648504, size=16998993, lastModifiedTime=1715076016908, largestRecordTimestamp=Some(1715076016908)),LogSegment(baseOffset=182651018, size=16996765, lastModifiedTime=1715076021971, largestRecordTimestamp=Some(1715076021968)),LogSegment(baseOffset=182653526, size=16995808, lastModifiedTime=1715076026767, largestRecordTimestamp=Some(1715076026752)),LogSegment(baseOffset=182655860, size=16993535, lastModifiedTime=1715076032181, largestRecordTimestamp=Some(1715076032131)),LogSegment(baseOffset=182658341, size=16971926, lastModifiedTime=1715076037067, largestRecordTimestamp=Some(1715076037053)) (kafka.log.LocalLog$)`, - `[2024-05-07 <_> INFO Deleted producer state snapshot <_> (kafka.log.SnapshotFile)`, - `[2024-05-07 <_> INFO Deleted offset index <_> (kafka.log.LogSegment)`, + `[2024-05-07 10:55:53,038] INFO [LocalLog partition=mimir-dev-09-aggregations-offsets-1, dir=/bitnami/kafka/data] Deleting segment files LogSegment(baseOffset=447957, size=948, lastModifiedTime=1715059232052, largestRecordTimestamp=Some(1715059232002)),LogSegment(baseOffset=447969, size=948, lastModifiedTime=1715059424352, largestRecordTimestamp=Some(1715059424301)) (kafka.log.LocalLog$)`, `[2024-05-07 <_> INFO Deleted log <_> (kafka.log.LogSegment)`, + `[2024-05-07 <_> INFO Deleted offset index <_> (kafka.log.LogSegment)`, + `[2024-05-07 <_> INFO Deleted producer state snapshot <_> (kafka.log.SnapshotFile)`, `[2024-05-07 <_> INFO Deleted time index <_> (kafka.log.LogSegment)`, + `[2024-05-07 <_> INFO [LocalLog <_> dir=/bitnami/kafka/data] Rolled new log segment at offset <_> in <_> ms. (kafka.log.LocalLog)`, + `[2024-05-07 <_> INFO [LocalLog partition=mimir-dev-09-aggregations-offsets-0, dir=/bitnami/kafka/data] Deleting segment files <_> size=948, <_> <_> (kafka.log.LocalLog$)`, + `[2024-05-07 <_> INFO [ProducerStateManager <_> Wrote producer snapshot at offset <_> with 0 producer ids in <_> ms. (kafka.log.ProducerStateManager)`, + `[2024-05-07 <_> INFO [UnifiedLog <_> dir=/bitnami/kafka/data] Deleting segment <_> <_> <_> <_> due to retention size <_> breach. Log size after deletion will be <_> (kafka.log.UnifiedLog)`, + `[2024-05-07 <_> INFO [UnifiedLog <_> dir=/bitnami/kafka/data] Deleting segments due to log start offset <_> breach: <_> <_> <_> <_> (kafka.log.UnifiedLog)`, + `[2024-05-07 <_> INFO [UnifiedLog <_> dir=/bitnami/kafka/data] Deleting segments due to log start offset <_> breach: <_> size=948, <_> <_> size=948, <_> <_> (kafka.log.UnifiedLog)`, + `[2024-05-07 <_> INFO [UnifiedLog <_> dir=/bitnami/kafka/data] Incremented log start offset to <_> due to leader offset increment (kafka.log.UnifiedLog)`, + `[2024-05-07 <_> INFO [UnifiedLog <_> dir=/bitnami/kafka/data] Incremented log start offset to <_> due to segment deletion (kafka.log.UnifiedLog)`, }, }, { @@ -222,32 +279,42 @@ func TestDrain_TrainExtractsPatterns(t *testing.T) { drain: New(DefaultConfig(), nil), inputFile: "testdata/kubernetes.txt", patterns: []string{ - "I0507 12:04:17.596484 1 highnodeutilization.go:107] \"Criteria for a node below target utilization\" CPU=50 Mem=50 Pods=100", - "I0507 12:04:17.595169 1 descheduler.go:155] Building a pod evictor", - "I0507 <_> 1 <_> \"Number of <_> <_> <_>", - "I0507 <_> 1 <_> \"Total <_> <_> <_> <_> <_> <_> <_>", - "I0507 <_> 1 <_> <_> <_> <_> <_> <_> <_> <_> <_> <_>", - "I0507 <_> 1 node.go:157] \"Pod does not fit on any other node\" pod:=\"loki-dev-005/querier-burst-6b5f6db455-5zvkm\" <_> error:=\"[insufficient <_> insufficient <_>", - "I0507 <_> 1 node.go:157] \"Pod does not fit on any other node\" pod:=\"loki-dev-005/querier-burst-6b5f6db455-5zvkm\" <_> error:=\"pod node selector does not match the node label\"", - "I0507 <_> 1 <_> \"Pods on node\" <_> <_> <_> <_>", - "I0507 12:02:27.947830 1 nodeutilization.go:274] \"Evicting pods based on priority, if they have same priority, they'll be evicted based on QoS tiers\"", - "I0507 <_> 1 defaultevictor.go:202] \"Pod fails the following checks\" <_> checks=\"pod has local storage and descheduler is not configured with evictLocalStoragePods\"", - "I0507 <_> 1 defaultevictor.go:202] \"Pod fails the following checks\" <_> checks=\"pod is a DaemonSet pod\"", - "I0507 <_> 1 defaultevictor.go:202] \"Pod fails the following checks\" <_> checks=\"[pod is a DaemonSet pod, pod has local storage and descheduler is not configured with evictLocalStoragePods]\"", - "I0507 <_> 1 defaultevictor.go:202] \"Pod fails the following checks\" <_> checks=\"[pod is a mirror pod, pod is a static pod, pod has system critical priority, pod has higher priority than specified priority class threshold, pod has local storage and descheduler is not configured with evictLocalStoragePods]\"", - "I0507 <_> 1 defaultevictor.go:202] \"Pod fails the following checks\" <_> checks=\"[pod is a DaemonSet pod, pod has system critical priority, pod has higher priority than specified priority class threshold, pod has local storage and descheduler is not configured with evictLocalStoragePods]\"", - "I0507 <_> 1 defaultevictor.go:202] \"Pod fails the following checks\" <_> checks=\"[pod <_> <_> <_> <_> pod has higher priority than specified priority class threshold]\"", - "I0507 <_> 1 defaultevictor.go:163] \"pod does not fit on any other node because of nodeSelector(s), Taint(s), or nodes marked as unschedulable\" <_>", - "I0507 <_> 1 node.go:157] \"Pod does not fit on any other node\" <_> <_> error:=\"[pod node selector does not match the node label, pod does not tolerate taints on the node]\"", - "I0507 <_> 1 defaultevictor.go:202] \"Pod fails the following checks\" <_> checks=\"[pod is a DaemonSet pod, pod has system critical priority, pod has higher priority than specified priority class threshold]\"", - "I0507 <_> 1 node.go:157] \"Pod does not fit on any other node\" <_> <_> error:=\"[pod node selector does not match the node label, insufficient <_> insufficient <_> insufficient pods]\"", - "I0507 <_> 1 defaultevictor.go:202] \"Pod fails the following checks\" <_> checks=\"[pod <_> <_> <_> <_> pod has higher priority than specified priority class threshold, pod has local storage and descheduler is not configured with evictLocalStoragePods]\"", - "I0507 <_> 1 node.go:157] \"Pod does not fit on any other node\" <_> <_> error:=\"[pod node selector does not match the node label, pod does not tolerate taints on the node, insufficient <_> insufficient <_>", - "I0507 <_> 1 node.go:157] \"Pod does not fit on any other node\" <_> <_> error:=\"insufficient cpu\"", - "I0507 <_> 1 node.go:157] \"Pod does not fit on any other node\" <_> <_> error:=\"[pod node selector does not match the node label, pod does not tolerate taints on the node, insufficient <_>", - "I0507 <_> 1 node.go:157] \"Pod does not fit on any other node\" <_> <_> error:=\"[pod node selector does not match the node label, insufficient <_> insufficient <_>", - "I0507 <_> 1 node.go:157] \"Pod does not fit on any other node\" <_> <_> error:=\"[pod node selector does not match the node label, insufficient <_>", - "I0507 <_> 1 <_> <_> <_> <_> <_> <_> <_>", + `I0507 12:02:27.947830 1 nodeutilization.go:274] "Evicting pods based on priority, if they have same priority, they'll be evicted based on QoS tiers"`, + `I0507 12:04:17.595169 1 descheduler.go:155] Building a pod evictor`, + `I0507 12:04:17.596431 1 nodeutilization.go:204] "Node is underutilized" node="gke-dev-eu-west-3-main-n2s8-1-1dd39c-d1c92061-4z2l" usage={"cpu":"984m","memory":"611Mi","pods":"16"} usagePercentage={"cpu":12.44,"memory":2.15,"pods":25}`, + `I0507 12:04:17.596484 1 highnodeutilization.go:107] "Criteria for a node below target utilization" CPU=50 Mem=50 Pods=100`, + `I0507 12:04:17.596504 1 highnodeutilization.go:108] "Number of underutilized nodes" totalNumber=1`, + `I0507 12:04:17.596528 1 nodeutilization.go:260] "Total capacity to be moved" CPU=5060 Mem=112216292800 Pods=163`, + `I0507 12:04:17.596651 1 defaultevictor.go:202] "Pod fails the following checks" pod="kube-system/metrics-server-v0.6.3-68f5b7c4d5-t5mz8" checks="[pod has system critical priority, pod has higher priority than specified priority class threshold, pod has local storage and descheduler is not configured with evictLocalStoragePods]"`, + `I0507 12:04:17.596803 1 defaultevictor.go:202] "Pod fails the following checks" pod="gadget/gadget-zjjts" checks="[pod is a DaemonSet pod, pod has local storage and descheduler is not configured with evictLocalStoragePods]"`, + `I0507 <_> 1 <_> "Evicting pods from node" <_> <_>`, + `I0507 <_> 1 <_> "No removable pods on node, try next node" <_>`, + `I0507 <_> 1 <_> "Number of evicted pods" <_>`, + `I0507 <_> 1 <_> "Pods on node" <_> <_> <_> <_>`, + `I0507 <_> 1 <_> "Total number of pods evicted" extension point="Balance" <_>`, + `I0507 <_> 1 <_> <_> Watch close - <_> total <_> items received`, + `I0507 <_> 1 defaultevictor.go:163] "pod does not fit on any other node because of nodeSelector(s), Taint(s), or nodes marked as unschedulable" <_>`, + `I0507 <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod has system critical priority, pod has higher priority than specified priority class threshold]"`, + `I0507 <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod is a DaemonSet pod, pod has higher priority than specified priority class threshold, pod has local storage and descheduler is not configured with evictLocalStoragePods]"`, + `I0507 <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod is a DaemonSet pod, pod has higher priority than specified priority class threshold]"`, + `I0507 <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod is a DaemonSet pod, pod has local storage and descheduler is not configured with evictLocalStoragePods]"`, + `I0507 <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod is a DaemonSet pod, pod has system critical priority, pod has higher priority than specified priority class threshold, pod has local storage and descheduler is not configured with evictLocalStoragePods]"`, + `I0507 <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod is a DaemonSet pod, pod has system critical priority, pod has higher priority than specified priority class threshold]"`, + `I0507 <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod is a mirror pod, pod is a static pod, pod has system critical priority, pod has higher priority than specified priority class threshold, pod has local storage and descheduler is not configured with evictLocalStoragePods]"`, + `I0507 <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="pod has local storage and descheduler is not configured with evictLocalStoragePods"`, + `I0507 <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="pod is a DaemonSet pod"`, + `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" <_> <_> error:="[pod node selector does not match the node label, insufficient <_>`, + `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" <_> <_> error:="[pod node selector does not match the node label, insufficient <_> insufficient <_>`, + `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" <_> <_> error:="[pod node selector does not match the node label, insufficient <_> insufficient <_> insufficient pods]"`, + `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" <_> <_> error:="[pod node selector does not match the node label, pod does not tolerate taints on the node, insufficient <_>`, + `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" <_> <_> error:="[pod node selector does not match the node label, pod does not tolerate taints on the node, insufficient <_> insufficient <_>`, + `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" <_> <_> error:="[pod node selector does not match the node label, pod does not tolerate taints on the node]"`, + `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" <_> <_> error:="insufficient cpu"`, + `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" pod:="loki-dev-005/querier-burst-6b5f6db455-5zvkm" <_> error:="[insufficient cpu, insufficient memory]"`, + `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" pod:="loki-dev-005/querier-burst-6b5f6db455-5zvkm" <_> error:="[insufficient memory, insufficient cpu]"`, + `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" pod:="loki-dev-005/querier-burst-6b5f6db455-5zvkm" <_> error:="pod node selector does not match the node label"`, + `I0507 <_> 1 node.go:339] "no Pod antiaffinity rule found" <_>`, + `I0507 <_> 1 nodeutilization.go:207] "Node is overutilized" <_> <_> <_>`, }, }, { @@ -263,91 +330,87 @@ func TestDrain_TrainExtractsPatterns(t *testing.T) { drain: New(DefaultConfig(), nil), inputFile: "testdata/calico.txt", patterns: []string{ - `2024-05-08 <_> [DEBUG][216945] felix/table.go 870: Found forward-reference <_> ipVersion=0x4 <_> <_> [0:0]" table="nat"`, - `2024-05-08 15:23:58.716 [DEBUG][216945] felix/table.go 851: Parsing line ipVersion=0x4 line="*nat" table="nat"`, - `2024-05-08 15:23:58.716 [DEBUG][216945] felix/table.go 881: Not an append, skipping ipVersion=0x4 line="# Generated by iptables-nft-save v1.8.4 on Wed May 8 15:23:58 2024" table="nat"`, - `2024-05-08 15:23:58.715 [DEBUG][216945] felix/table.go 851: Parsing line ipVersion=0x4 line="# Generated by iptables-nft-save v1.8.4 on Wed May 8 15:23:58 2024" table="nat"`, - `2024-05-08 15:23:58.684 [DEBUG][216945] felix/versionparse.go 118: Parsed kernel version version=5.15.0-1057`, - `2024-05-08 15:23:58.684 [DEBUG][216945] felix/versionparse.go 110: Raw kernel version rawVersion="Linux version 5.15.0-1057-azure (buildd@lcy02-amd64-033) (gcc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0, GNU ld (GNU Binutils for Ubuntu) 2.38) #65-Ubuntu SMP Fri Feb 9 18:39:24 UTC 2024\n"`, - `2024-05-08 15:23:58.684 [DEBUG][216945] felix/feature_detect.go 242: Ran iptables --version rawVersion="iptables v1.8.4 (legacy)\n"`, - `2024-05-08 <_> [DEBUG][216945] felix/feature_detect.go <_> <_> <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][216945] felix/table.go <_> <_> <_> <_> <_> <_> <_> table="nat"`, - `2024-05-08 <_> [DEBUG][3576126] felix/int_dataplane.go <_> <_> <_> for MTU <_> <_> <_>`, - `2024-05-08 15:23:57.969 [WARNING][56] felix/table.go 654: Detected out-of-sync inserts, marking for resync actualRuleIDs=[]string{"", "", "", "", "Cz_u1IQiXIMmKD4c", "", "", "", "", "", "", "", "", "", "", "", ""} chainName="INPUT" expectedRuleIDs=[]string{"Cz_u1IQiXIMmKD4c", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""} ipVersion=0x4 table="filter"`, - `2024-05-08 15:23:57.969 [WARNING][56] felix/table.go 654: Detected out-of-sync inserts, marking for resync actualRuleIDs=[]string{"", "", "", "", "tVnHkvAo15HuiPy0", "", "", "", "", ""} chainName="OUTPUT" expectedRuleIDs=[]string{"tVnHkvAo15HuiPy0", "", "", "", "", "", "", "", "", ""} ipVersion=0x4 table="filter"`, - `2024-05-08 15:23:57.942 [WARNING][56] felix/table.go 654: Detected out-of-sync inserts, marking for resync actualRuleIDs=[]string{"", "", "", "", "", "", "", "", "", "", "", "", "tVnHkvAo15HuiPy0", "", ""} chainName="OUTPUT" expectedRuleIDs=[]string{"tVnHkvAo15HuiPy0", "", "", "", "", "", "", "", "", "", "", "", "", "", ""} ipVersion=0x4 table="raw"`, - `2024-05-08 15:23:57.942 [WARNING][56] felix/table.go 654: Detected out-of-sync inserts, marking for resync actualRuleIDs=[]string{"", "", "", "", "6gwbT8clXdHdC1b1"} chainName="PREROUTING" expectedRuleIDs=[]string{"6gwbT8clXdHdC1b1", "", "", "", ""} ipVersion=0x4 table="raw"`, - `2024-05-08 <_> <_> felix/ipsets.go 607: Skipping expected Calico IP set. family="inet" <_>`, - `2024-05-08 15:23:58.604 [DEBUG][65] felix/ipsets.go <_> <_> <_> <_> <_> <_> <_>`, - `2024-05-08 <_> <_> felix/ipsets.go 366: Finished IPSets resync family="inet" numInconsistenciesFound=0 <_>`, - `2024-05-08 <_> <_> felix/ipsets.go 589: Whitelisting IP sets. <_> family="inet" <_>`, - `2024-05-08 <_> <_> felix/ipsets.go 467: Found member in dataplane <_> family="inet" <_> setID="this-host"`, - `2024-05-08 <_> <_> felix/ipsets.go 234: Asked to resync with the dataplane on next update. family="inet"`, - `bird: Netlink: No route to host`, - `2024-05-08 <_> [DEBUG][76] felix/route_table.go 661: Syncing interface routes <_> <_> ipVersion=0x4 <_>`, - `2024-05-08 15:23:56.619 [DEBUG][76] felix/route_table.go 584: Flag no OIF for full re-sync`, - `2024-05-08 <_> [DEBUG][76] felix/route_table.go <_> <_> <_> <_> <_> <_> <_> ipVersion=0x4 <_>`, - `2024-05-08 15:23:56.617 [DEBUG][76] felix/wireguard.go 1503: Wireguard is disabled and does not exist ifaceName="wireguard.cali" ipVersion=0x4`, - `2024-05-08 15:23:56.615 [DEBUG][76] felix/route_table.go 533: Check interfaces matching regex`, - `2024-05-08 15:23:56.615 [DEBUG][76] felix/wireguard.go 654: Wireguard is not in-sync - verifying wireguard configuration is removed ipVersion=0x4`, - `2024-05-08 <_> <_> felix/ipsets.go 426: Parsing IP set. family="inet" <_>`, - `2024-05-08 <_> <_> felix/ipsets.go <_> <_> <_> <_> <_> family="inet"`, + `2024-05-08 15:23:56.403 [DEBUG][615489] felix/table.go 699: Finished loading iptables state ipVersion=0x4 table="filter"`, + `2024-05-08 15:23:56.614 [DEBUG][76] felix/int_dataplane.go 1777: Refreshing routes`, `2024-05-08 15:23:56.615 [DEBUG][76] felix/route_rule.go 179: Queueing a resync of routing rules. ipVersion=4`, + `2024-05-08 15:23:56.615 [DEBUG][76] felix/route_table.go 480: Queueing a resync of routing table. ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, + `2024-05-08 15:23:56.615 [DEBUG][76] felix/route_table.go 480: Queueing a resync of routing table. ifaceRegex="^wireguard.cali$" ipVersion=0x4 tableIndex=1`, + `2024-05-08 15:23:56.615 [DEBUG][76] felix/route_table.go 533: Check interfaces matching regex`, `2024-05-08 15:23:56.615 [DEBUG][76] felix/wireguard.go 605: Queueing a resync of wireguard configuration ipVersion=0x4`, - `2024-05-08 <_> <_> felix/wireguard.go 652: Wireguard is not enabled, skipping sync ipVersion=0x4`, - `2024-05-08 <_> [DEBUG][76] felix/route_table.go 880: Processing route: 254 <_> <_> <_> ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, - `2024-05-08 15:23:56.615 [DEBUG][76] felix/route_table.go 480: Queueing a resync of routing table. <_> ipVersion=0x4 <_>`, - `2024-05-08 <_> [DEBUG][216945] felix/xdp_state.go <_> <_> <_> <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][65] felix/xdp_state.go <_> <_> <_> <_> <_> <_>`, - `2024-05-08 15:23:58.035 [DEBUG][501368] felix/xdp_state.go <_> <_> <_> <_> <_> <_>`, - `2024-05-08 15:23:57.886 [DEBUG][3503680] felix/xdp_state.go <_> <_> <_> <_> <_> <_>`, - `2024-05-08 15:23:57.239 [DEBUG][732993] felix/xdp_state.go <_> <_> <_> <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][76] felix/xdp_state.go <_> <_> <_> <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][216945] felix/xdp_state.go <_> <_> <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][65] felix/xdp_state.go <_> <_> <_> <_> <_>`, - `2024-05-08 15:23:58.035 [DEBUG][501368] felix/xdp_state.go <_> <_> <_> <_> <_>`, - `2024-05-08 15:23:57.886 [DEBUG][3503680] felix/xdp_state.go <_> <_> <_> <_> <_>`, - `2024-05-08 15:23:57.239 [DEBUG][732993] felix/xdp_state.go <_> <_> <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][76] felix/xdp_state.go <_> <_> <_> <_> <_>`, - `2024-05-08 <_> <_> felix/xdp_state.go 1270: Finished processing pending diff state. bpfActions=intdataplane.xdpBPFActions{CreateMap:set.Typed[string]{}, RemoveMap:set.Typed[string]{}, AddToMap:map[string]map[string]uint32{}, RemoveFromMap:map[string]map[string]uint32{}, InstallXDP:set.Typed[string]{}, UninstallXDP:set.Typed[string]{}, MembersToDrop:map[string]map[string]uint32{}, MembersToAdd:map[string]map[string]uint32{}} family=4 newCS=&intdataplane.xdpSystemState{IfaceNameToData:map[string]intdataplane.xdpIfaceData{}, XDPEligiblePolicies:map[proto.PolicyID]intdataplane.xdpRules{}}`, - `2024-05-08 <_> <_> felix/xdp_state.go 1043: Processing pending diff state. cs=&intdataplane.xdpSystemState{IfaceNameToData:map[string]intdataplane.xdpIfaceData{}, XDPEligiblePolicies:map[proto.PolicyID]intdataplane.xdpRules{}} family=4`, + `2024-05-08 15:23:56.615 [DEBUG][76] felix/wireguard.go 654: Wireguard is not in-sync - verifying wireguard configuration is removed ipVersion=0x4`, + `2024-05-08 15:23:56.617 [DEBUG][76] felix/wireguard.go 1503: Wireguard is disabled and does not exist ifaceName="wireguard.cali" ipVersion=0x4`, + `2024-05-08 15:23:56.619 [DEBUG][76] felix/route_table.go 584: Flag no OIF for full re-sync`, + `2024-05-08 15:23:56.619 [DEBUG][76] felix/route_table.go 614: Synchronised routes on interface ifaceName="*NoOIF*" ifaceRegex="^wireguard.cali$" ipVersion=0x4 tableIndex=1`, + `2024-05-08 15:23:56.619 [DEBUG][76] felix/route_table.go 661: Syncing interface routes ifaceName="*NoOIF*" ifaceRegex="^wireguard.cali$" ipVersion=0x4 tableIndex=1`, + `2024-05-08 15:23:56.619 [DEBUG][76] felix/route_table.go 686: Reconcile against kernel programming ifaceName="*NoOIF*" ifaceRegex="^wireguard.cali$" ipVersion=0x4 tableIndex=1`, + `2024-05-08 15:23:57.942 [WARNING][56] felix/table.go 654: Detected out-of-sync inserts, marking for resync actualRuleIDs=[]string{"", "", "", "", "", "", "", "", "", "", "", "", "tVnHkvAo15HuiPy0", "", ""} chainName="OUTPUT" expectedRuleIDs=[]string{"tVnHkvAo15HuiPy0", "", "", "", "", "", "", "", "", "", "", "", "", "", ""} ipVersion=0x4 table="raw"`, + `2024-05-08 15:23:57.942 [WARNING][56] felix/table.go 654: Detected out-of-sync inserts, marking for resync actualRuleIDs=[]string{"", "", "", "", "6gwbT8clXdHdC1b1"} chainName="PREROUTING" expectedRuleIDs=[]string{"6gwbT8clXdHdC1b1", "", "", "", ""} ipVersion=0x4 table="raw"`, + `2024-05-08 15:23:57.969 [WARNING][56] felix/table.go 654: Detected out-of-sync inserts, marking for resync actualRuleIDs=[]string{"", "", "", "", "Cz_u1IQiXIMmKD4c", "", "", "", "", "", "", "", "", "", "", "", ""} chainName="INPUT" expectedRuleIDs=[]string{"Cz_u1IQiXIMmKD4c", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""} ipVersion=0x4 table="filter"`, + `2024-05-08 15:23:57.969 [WARNING][56] felix/table.go 654: Detected out-of-sync inserts, marking for resync actualRuleIDs=[]string{"", "", "", "", "tVnHkvAo15HuiPy0", "", "", "", "", ""} chainName="OUTPUT" expectedRuleIDs=[]string{"tVnHkvAo15HuiPy0", "", "", "", "", "", "", "", "", ""} ipVersion=0x4 table="filter"`, + `2024-05-08 15:23:58.566 [DEBUG][3576126] felix/int_dataplane.go 957: Examining link for MTU calculation mtu=1500 name="eth0"`, + `2024-05-08 15:23:58.680 [DEBUG][216945] felix/int_dataplane.go 1785: Reschedule kick received`, + `2024-05-08 15:23:58.681 [DEBUG][216945] felix/feature_detect.go 112: Refreshing detected iptables features`, + `2024-05-08 15:23:58.681 [DEBUG][216945] felix/table.go 944: Invalidating dataplane cache ipVersion=0x4 reason="refresh timer" table="nat"`, + `2024-05-08 15:23:58.684 [DEBUG][216945] felix/feature_detect.go 242: Ran iptables --version rawVersion="iptables v1.8.4 (legacy)\n"`, + `2024-05-08 15:23:58.684 [DEBUG][216945] felix/feature_detect.go 255: Parsed iptables version version=1.8.4`, + `2024-05-08 15:23:58.684 [DEBUG][216945] felix/table.go 604: Loading current iptables state and checking it is correct. ipVersion=0x4 table="nat"`, + `2024-05-08 15:23:58.684 [DEBUG][216945] felix/versionparse.go 110: Raw kernel version rawVersion="Linux version 5.15.0-1057-azure (buildd@lcy02-amd64-033) (gcc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0, GNU ld (GNU Binutils for Ubuntu) 2.38) #65-Ubuntu SMP Fri Feb 9 18:39:24 UTC 2024\n"`, + `2024-05-08 15:23:58.684 [DEBUG][216945] felix/versionparse.go 118: Parsed kernel version version=5.15.0-1057`, + `2024-05-08 15:23:58.715 [DEBUG][216945] felix/table.go 851: Parsing line ipVersion=0x4 line="# Generated by iptables-nft-save v1.8.4 on Wed May 8 15:23:58 2024" table="nat"`, + `2024-05-08 15:23:58.716 [DEBUG][216945] felix/table.go 851: Parsing line ipVersion=0x4 line="*nat" table="nat"`, + `2024-05-08 15:23:58.716 [DEBUG][216945] felix/table.go 881: Not an append, skipping ipVersion=0x4 line="# Generated by iptables-nft-save v1.8.4 on Wed May 8 15:23:58 2024" table="nat"`, + `2024-05-08 15:23:58.716 [DEBUG][216945] felix/table.go 881: Not an append, skipping ipVersion=0x4 line="*nat" table="nat"`, + `2024-05-08 15:23:58.717 [DEBUG][216945] felix/table.go 851: Parsing line ipVersion=0x4 line=":POSTROUTING ACCEPT [0:0]" table="nat"`, + `2024-05-08 15:23:58.717 [DEBUG][216945] felix/table.go 870: Found forward-reference chainName="POSTROUTING" ipVersion=0x4 line=":POSTROUTING ACCEPT [0:0]" table="nat"`, + `2024-05-08 15:23:58.718 [DEBUG][216945] felix/table.go 851: Parsing line ipVersion=0x4 line=":OUTPUT ACCEPT [0:0]" table="nat"`, + `2024-05-08 15:23:58.718 [DEBUG][216945] felix/table.go 851: Parsing line ipVersion=0x4 line=":PREROUTING ACCEPT [0:0]" table="nat"`, + `2024-05-08 15:23:58.718 [DEBUG][216945] felix/table.go 870: Found forward-reference chainName="OUTPUT" ipVersion=0x4 line=":OUTPUT ACCEPT [0:0]" table="nat"`, + `2024-05-08 15:23:58.718 [DEBUG][216945] felix/table.go 870: Found forward-reference chainName="PREROUTING" ipVersion=0x4 line=":PREROUTING ACCEPT [0:0]" table="nat"`, `2024-05-08 <_> <_> felix/endpoint_mgr.go 443: Reporting endpoint status. dirtyEndpoints=set.Set{}`, - `2024-05-08 15:23:58.680 [DEBUG][216945] felix/int_dataplane.go <_> <_> <_> <_>`, - `2024-05-08 <_> <_> felix/int_dataplane.go 1807: Applying dataplane updates`, - `2024-05-08 15:23:56.614 [DEBUG][76] felix/int_dataplane.go 1777: Refreshing routes`, - `2024-05-08 <_> <_> felix/sync_client.go <_> <_> <_> <_> Typha connID=0x0 connection=&discovery.Typha{Addr:"", IP:"", NodeName:(*string)(nil)} type=""`, - `2024-05-08 <_> <_> felix/sync_client.go 434: New message from Typha. connID=0x0 connection=&discovery.Typha{Addr:"", IP:"", NodeName:(*string)(nil)} envelope=syncproto.Envelope{Message:syncproto.MsgPing{Timestamp:time.Date(2024, time.May, 8, 15, 23, <_> <_> time.Local)}} type=""`, + `2024-05-08 <_> <_> felix/health.go 167: Health: <_>`, + `2024-05-08 <_> <_> felix/health.go 196: Checking state of reporter reporter=&health.reporterState{name:"async_calc_graph", reports:health.HealthReport{Live:true, Ready:true, Detail:""}, timeout:20000000000, latest:health.HealthReport{Live:true, Ready:true, Detail:""}, <_> <_> loc:(*time.Location)(0x4ce3aa0)}}`, + `2024-05-08 <_> <_> felix/health.go 196: Checking state of reporter reporter=&health.reporterState{name:"felix-startup", reports:health.HealthReport{Live:true, Ready:true, Detail:""}, timeout:0, latest:health.HealthReport{Live:true, Ready:true, Detail:""}, <_> <_> loc:(*time.Location)(0x4ce3aa0)}}`, + `2024-05-08 <_> <_> felix/health.go 196: Checking state of reporter reporter=&health.reporterState{name:"int_dataplane", reports:health.HealthReport{Live:true, Ready:true, Detail:""}, timeout:90000000000, latest:health.HealthReport{Live:true, Ready:true, Detail:""}, <_> <_> loc:(*time.Location)(0x4ce3aa0)}}`, `2024-05-08 <_> <_> felix/health.go 245: Calculated health summary healthResult=&health.HealthReport{Live:true, Ready:true, Detail:"+------------------+---------+----------------+-----------------+--------+\n| COMPONENT | TIMEOUT | LIVENESS | READINESS | DETAIL |\n+------------------+---------+----------------+-----------------+--------+\n| async_calc_graph | 20s | reporting live | reporting ready | |\n| felix-startup | 0s | reporting live | reporting ready | |\n| int_dataplane | 1m30s | reporting live | reporting ready | |\n+------------------+---------+----------------+-----------------+--------+"}`, - `2024-05-08 <_> <_> felix/health.go 196: Checking state of reporter <_> reports:health.HealthReport{Live:true, Ready:true, Detail:""}, <_> latest:health.HealthReport{Live:true, Ready:true, Detail:""}, <_> <_> loc:(*time.Location)(0x4ce3aa0)}}`, - `2024-05-08 <_> [DEBUG][501368] felix/health.go <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][76] felix/health.go <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][3576126] felix/health.go <_> <_> <_>`, - `2024-05-08 15:23:57.701 [DEBUG][216945] felix/health.go <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][3583983] felix/health.go <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][3596528] felix/health.go <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][65] felix/health.go <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][3383360] felix/health.go <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][3435880] felix/health.go <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][3794357] felix/health.go <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][88347] felix/health.go <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][990568] felix/health.go <_> <_> <_>`, - `2024-05-08 15:23:56.615 [DEBUG][2460733] felix/health.go <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][3503680] felix/health.go <_> <_> <_>`, - `2024-05-08 <_> <_> felix/summary.go 100: Summarising <_> dataplane reconciliation loops over <_> <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][65] felix/int_dataplane.go <_> <_> <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][501368] felix/int_dataplane.go <_> <_> <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][3503680] felix/int_dataplane.go <_> <_> <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][732993] felix/int_dataplane.go <_> <_> <_> <_> <_>`, - `2024-05-08 <_> [DEBUG][216945] felix/int_dataplane.go <_> <_> <_> <_> <_>`, + `2024-05-08 <_> <_> felix/health.go <_> GET <_>`, + `2024-05-08 <_> <_> felix/int_dataplane.go 1773: Refreshing IP sets state`, + `2024-05-08 <_> <_> felix/int_dataplane.go 1807: Applying dataplane updates`, `2024-05-08 <_> <_> felix/int_dataplane.go 2080: Asked to reschedule. <_>`, - `2024-05-08 15:23:58.684 [DEBUG][216945] felix/table.go 604: Loading current iptables state and checking it is correct. ipVersion=0x4 table="nat"`, - `2024-05-08 15:23:58.681 [DEBUG][216945] felix/table.go <_> <_> <_> <_> <_> <_> <_> <_> <_> <_> ipVersion=0x4 <_>`, - `2024-05-08 15:23:58.605 [DEBUG][65] felix/table.go <_> <_> <_> <_> <_> <_> <_> <_> <_> <_> ipVersion=0x4 table="filter"`, - `2024-05-08 15:23:58.604 [DEBUG][65] felix/table.go <_> <_> <_> <_> <_> <_> <_> <_> <_> <_> ipVersion=0x4 <_>`, - `2024-05-08 15:23:58.038 [DEBUG][501368] felix/table.go <_> <_> <_> <_> <_> <_> <_> <_> <_> <_> ipVersion=0x4 <_>`, - `2024-05-08 <_> <_> felix/table.go 1263: Update ended up being no-op, skipping call to ip(6)tables-restore. ipVersion=0x4 <_>`, + `2024-05-08 <_> <_> felix/ipsets.go 234: Asked to resync with the dataplane on next update. family="inet"`, + `2024-05-08 <_> <_> felix/ipsets.go 314: Resyncing ipsets with dataplane. family="inet"`, + `2024-05-08 <_> <_> felix/ipsets.go 366: Finished IPSets resync family="inet" numInconsistenciesFound=0 <_>`, + `2024-05-08 <_> <_> felix/ipsets.go 426: Parsing IP set. family="inet" <_>`, + `2024-05-08 <_> <_> felix/ipsets.go 467: Found member in dataplane <_> family="inet" <_> setID="this-host"`, + `2024-05-08 <_> <_> felix/ipsets.go 589: Whitelisting IP sets. ID="all-ipam-pools" family="inet" mainName="cali40all-ipam-pools"`, + `2024-05-08 <_> <_> felix/ipsets.go 589: Whitelisting IP sets. ID="masq-ipam-pools" family="inet" mainName="cali40masq-ipam-pools"`, + `2024-05-08 <_> <_> felix/ipsets.go 589: Whitelisting IP sets. ID="this-host" family="inet" mainName="cali40this-host"`, + `2024-05-08 <_> <_> felix/ipsets.go 607: Skipping expected Calico IP set. family="inet" <_>`, + `2024-05-08 <_> <_> felix/ipsets.go 643: No dirty IP sets. family="inet"`, + `2024-05-08 <_> <_> felix/summary.go 100: Summarising <_> dataplane reconciliation loops over <_> <_> <_> <_>`, + `2024-05-08 <_> <_> felix/sync_client.go 347: Ping received from Typha connID=0x0 connection=&discovery.Typha{Addr:"", IP:"", NodeName:(*string)(nil)} type=""`, + `2024-05-08 <_> <_> felix/sync_client.go 356: Pong sent to Typha connID=0x0 connection=&discovery.Typha{Addr:"", IP:"", NodeName:(*string)(nil)} type=""`, + `2024-05-08 <_> <_> felix/sync_client.go 434: New message from Typha. connID=0x0 connection=&discovery.Typha{Addr:"", IP:"", NodeName:(*string)(nil)} envelope=syncproto.Envelope{Message:syncproto.MsgPing{Timestamp:time.Date(2024, time.May, 8, 15, 23, <_> <_> time.Local)}} type=""`, `2024-05-08 <_> <_> felix/table.go 1233: In nftables mode, restarting transaction between updates and deletions. ipVersion=0x4 <_>`, - `2024-05-08 <_> [DEBUG][615489] felix/table.go <_> <_> <_> <_> <_> ipVersion=0x4 table="filter"`, + `2024-05-08 <_> <_> felix/table.go 1263: Update ended up being no-op, skipping call to ip(6)tables-restore. ipVersion=0x4 <_>`, + `2024-05-08 <_> <_> felix/wireguard.go 652: Wireguard is not enabled, skipping sync ipVersion=0x4`, + `2024-05-08 <_> <_> felix/xdp_state.go 1004: Updating ipsetIDsToMembers cache. family=4`, + `2024-05-08 <_> <_> felix/xdp_state.go 1043: Processing pending diff state. cs=&intdataplane.xdpSystemState{IfaceNameToData:map[string]intdataplane.xdpIfaceData{}, XDPEligiblePolicies:map[proto.PolicyID]intdataplane.xdpRules{}} family=4`, + `2024-05-08 <_> <_> felix/xdp_state.go 1270: Finished processing pending diff state. bpfActions=intdataplane.xdpBPFActions{CreateMap:set.Typed[string]{}, RemoveMap:set.Typed[string]{}, AddToMap:map[string]map[string]uint32{}, RemoveFromMap:map[string]map[string]uint32{}, InstallXDP:set.Typed[string]{}, UninstallXDP:set.Typed[string]{}, MembersToDrop:map[string]map[string]uint32{}, MembersToAdd:map[string]map[string]uint32{}} family=4 newCS=&intdataplane.xdpSystemState{IfaceNameToData:map[string]intdataplane.xdpIfaceData{}, XDPEligiblePolicies:map[proto.PolicyID]intdataplane.xdpRules{}}`, + `2024-05-08 <_> <_> felix/xdp_state.go 1605: Getting member changes. family=4 oldMembers=map[string]set.Set[string]{}`, + `2024-05-08 <_> <_> felix/xdp_state.go 1798: Processing BPF actions. family="ipv4"`, + `2024-05-08 <_> <_> felix/xdp_state.go 1932: Finished processing BPF actions. family="ipv4"`, + `2024-05-08 <_> <_> felix/xdp_state.go 968: Processing member updates. family=4`, + `2024-05-08 <_> [DEBUG][216945] felix/table.go 851: Parsing line ipVersion=0x4 <_> - [0:0]" table="nat"`, + `2024-05-08 <_> [DEBUG][216945] felix/table.go 870: Found forward-reference <_> ipVersion=0x4 <_> - [0:0]" table="nat"`, + `2024-05-08 <_> [DEBUG][3576126] felix/int_dataplane.go 954: Skipping interface for MTU detection <_> <_>`, + `2024-05-08 <_> [DEBUG][615489] felix/table.go 677: Skipping expected chain <_> ipVersion=0x4 table="filter"`, + `2024-05-08 <_> [DEBUG][76] felix/route_table.go 557: Resync: found calico-owned interface <_> ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, + `2024-05-08 <_> [DEBUG][76] felix/route_table.go 614: Synchronised routes on interface <_> ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, + `2024-05-08 <_> [DEBUG][76] felix/route_table.go 661: Syncing interface routes <_> ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, + `2024-05-08 <_> [DEBUG][76] felix/route_table.go 686: Reconcile against kernel programming <_> ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, + `2024-05-08 <_> [DEBUG][76] felix/route_table.go 880: Processing route: 254 <_> <_> <_> ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, + `2024-05-08 <_> [DEBUG][76] felix/route_table.go 915: Route is correct <_> <_> ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, + `bird: Netlink: No route to host`, }, }, } @@ -369,6 +432,13 @@ func TestDrain_TrainExtractsPatterns(t *testing.T) { for _, cluster := range clusters { output = append(output, cluster.String()) } + slices.Sort(output) + + if outputPatternsForTestUpdate { + for _, pattern := range output { + fmt.Printf("`%s`,\n", pattern) + } + } require.Equal(t, tt.patterns, output) }) From cc3694eecddaab579d08328cdab78a7d8a7bd720 Mon Sep 17 00:00:00 2001 From: George Robinson Date: Wed, 29 May 2024 12:12:51 +0100 Subject: [PATCH 32/41] feat: Add ingester_chunks_flush_failures_total (#12925) Signed-off-by: George Robinson --- pkg/ingester/flush.go | 1 + pkg/ingester/metrics.go | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/pkg/ingester/flush.go b/pkg/ingester/flush.go index f9904ca8409e..00aad0547549 100644 --- a/pkg/ingester/flush.go +++ b/pkg/ingester/flush.go @@ -372,6 +372,7 @@ func (i *Ingester) encodeChunk(ctx context.Context, ch *chunk.Chunk, desc *chunk // chunk to have another opportunity to be flushed. func (i *Ingester) flushChunk(ctx context.Context, ch *chunk.Chunk) error { if err := i.store.Put(ctx, []chunk.Chunk{*ch}); err != nil { + i.metrics.chunksFlushFailures.Inc() return fmt.Errorf("store put chunk: %w", err) } i.metrics.flushedChunksStats.Inc(1) diff --git a/pkg/ingester/metrics.go b/pkg/ingester/metrics.go index 8b005860555f..756eba0ebea7 100644 --- a/pkg/ingester/metrics.go +++ b/pkg/ingester/metrics.go @@ -47,6 +47,7 @@ type ingesterMetrics struct { chunkSizePerTenant *prometheus.CounterVec chunkAge prometheus.Histogram chunkEncodeTime prometheus.Histogram + chunksFlushFailures prometheus.Counter chunksFlushedPerReason *prometheus.CounterVec chunkLifespan prometheus.Histogram flushedChunksStats *analytics.Counter @@ -232,6 +233,11 @@ func newIngesterMetrics(r prometheus.Registerer, metricsNamespace string) *inges // 10ms to 10s. Buckets: prometheus.ExponentialBuckets(0.01, 4, 6), }), + chunksFlushFailures: promauto.With(r).NewCounter(prometheus.CounterOpts{ + Namespace: constants.Loki, + Name: "ingester_chunks_flush_failures_total", + Help: "Total number of flush failures.", + }), chunksFlushedPerReason: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ Namespace: constants.Loki, Name: "ingester_chunks_flushed_total", From ab7af0575dc84070a11d4b589fd5c8a223baf536 Mon Sep 17 00:00:00 2001 From: Salva Corts Date: Wed, 29 May 2024 14:18:44 +0200 Subject: [PATCH 33/41] refactor(blooms): Builder retrieves tasks from planner (#13046) --- docs/sources/shared/configuration.md | 10 ++ pkg/bloombuild/builder/builder.go | 137 +++++++++++++++++++++++-- pkg/bloombuild/builder/builder_test.go | 123 ++++++++++++++++++++++ pkg/bloombuild/builder/config.go | 23 ++++- pkg/bloombuild/builder/metrics.go | 33 ++++++ pkg/bloombuild/planner/planner.go | 10 +- pkg/bloombuild/protos/service.pb.go | 87 +++++++++++++--- pkg/bloombuild/protos/service.proto | 1 + 8 files changed, 392 insertions(+), 32 deletions(-) create mode 100644 pkg/bloombuild/builder/builder_test.go diff --git a/docs/sources/shared/configuration.md b/docs/sources/shared/configuration.md index c0e4bdeeca4d..a8a4a7df720e 100644 --- a/docs/sources/shared/configuration.md +++ b/docs/sources/shared/configuration.md @@ -356,6 +356,15 @@ bloom_build: [max_queued_tasks_per_tenant: | default = 30000] builder: + # The grpc_client block configures the gRPC client used to communicate + # between a client and server component in Loki. + # The CLI flags prefix for this block configuration is: + # bloom-build.builder.grpc + [grpc_config: ] + + # Hostname (and port) of the bloom planner + # CLI flag: -bloom-build.builder.planner-address + [planner_address: | default = ""] # Experimental: The bloom_gateway block configures the Loki bloom gateway # server, responsible for serving queries for filtering chunks based on filter @@ -2335,6 +2344,7 @@ The `gcs_storage_config` block configures the connection to Google Cloud Storage The `grpc_client` block configures the gRPC client used to communicate between a client and server component in Loki. The supported CLI flags `` used to reference this configuration block are: - `bigtable` +- `bloom-build.builder.grpc` - `bloom-gateway-client.grpc` - `boltdb.shipper.index-gateway-client.grpc` - `frontend.grpc-client-config` diff --git a/pkg/bloombuild/builder/builder.go b/pkg/bloombuild/builder/builder.go index 098e7d6d83f0..aa1455fc38c0 100644 --- a/pkg/bloombuild/builder/builder.go +++ b/pkg/bloombuild/builder/builder.go @@ -2,49 +2,164 @@ package builder import ( "context" + "fmt" + "time" "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/google/uuid" "github.com/grafana/dskit/services" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc" + "github.com/grafana/loki/v3/pkg/bloombuild/protos" utillog "github.com/grafana/loki/v3/pkg/util/log" ) -type Worker struct { +type Builder struct { services.Service + ID string + cfg Config metrics *Metrics logger log.Logger + + client protos.PlannerForBuilderClient } func New( cfg Config, logger log.Logger, r prometheus.Registerer, -) (*Worker, error) { +) (*Builder, error) { utillog.WarnExperimentalUse("Bloom Builder", logger) - w := &Worker{ + b := &Builder{ + ID: uuid.NewString(), cfg: cfg, metrics: NewMetrics(r), logger: logger, } - w.Service = services.NewBasicService(w.starting, w.running, w.stopping) - return w, nil + b.Service = services.NewBasicService(b.starting, b.running, b.stopping) + return b, nil +} + +func (b *Builder) starting(_ context.Context) error { + b.metrics.running.Set(1) + return nil } -func (w *Worker) starting(_ context.Context) (err error) { - w.metrics.running.Set(1) - return err +func (b *Builder) stopping(_ error) error { + if b.client != nil { + req := &protos.NotifyBuilderShutdownRequest{ + BuilderID: b.ID, + } + if _, err := b.client.NotifyBuilderShutdown(context.Background(), req); err != nil { + level.Error(b.logger).Log("msg", "failed to notify planner about builder shutdown", "err", err) + } + } + + b.metrics.running.Set(0) + return nil } -func (w *Worker) stopping(_ error) error { - w.metrics.running.Set(0) +func (b *Builder) running(ctx context.Context) error { + opts, err := b.cfg.GrpcConfig.DialOption(nil, nil) + if err != nil { + return fmt.Errorf("failed to create grpc dial options: %w", err) + } + + // TODO: Wrap hereafter in retry logic + conn, err := grpc.DialContext(ctx, b.cfg.PlannerAddress, opts...) + if err != nil { + return fmt.Errorf("failed to dial bloom planner: %w", err) + } + + b.client = protos.NewPlannerForBuilderClient(conn) + + c, err := b.client.BuilderLoop(ctx) + if err != nil { + return fmt.Errorf("failed to start builder loop: %w", err) + } + + // Start processing tasks from planner + if err := b.builderLoop(c); err != nil { + return fmt.Errorf("builder loop failed: %w", err) + } + return nil } -func (w *Worker) running(_ context.Context) error { +func (b *Builder) builderLoop(c protos.PlannerForBuilder_BuilderLoopClient) error { + // Send ready message to planner + if err := c.Send(&protos.BuilderToPlanner{BuilderID: b.ID}); err != nil { + return fmt.Errorf("failed to send ready message to planner: %w", err) + } + + for b.State() == services.Running { + // When the planner connection closes or the builder stops, the context + // will be canceled and the loop will exit. + protoTask, err := c.Recv() + if err != nil { + if errors.Is(c.Context().Err(), context.Canceled) { + level.Debug(b.logger).Log("msg", "builder loop context canceled") + return nil + } + + return fmt.Errorf("failed to receive task from planner: %w", err) + } + + b.metrics.taskStarted.Inc() + start := time.Now() + status := statusSuccess + + err = b.processTask(c.Context(), protoTask.Task) + if err != nil { + status = statusFailure + level.Error(b.logger).Log("msg", "failed to process task", "err", err) + } + + b.metrics.taskCompleted.WithLabelValues(status).Inc() + b.metrics.taskDuration.WithLabelValues(status).Observe(time.Since(start).Seconds()) + + // Acknowledge task completion to planner + if err = b.notifyTaskCompletedToPlanner(c, err); err != nil { + return fmt.Errorf("failed to notify task completion to planner: %w", err) + } + } + + level.Debug(b.logger).Log("msg", "builder loop stopped") + return nil +} + +func (b *Builder) notifyTaskCompletedToPlanner(c protos.PlannerForBuilder_BuilderLoopClient, err error) error { + var errMsg string + if err != nil { + errMsg = err.Error() + } + + // TODO: Implement retry + if err := c.Send(&protos.BuilderToPlanner{ + BuilderID: b.ID, + Error: errMsg, + }); err != nil { + return fmt.Errorf("failed to acknowledge task completion to planner: %w", err) + } + return nil +} + +func (b *Builder) processTask(_ context.Context, protoTask *protos.ProtoTask) error { + task, err := protos.FromProtoTask(protoTask) + if err != nil { + return fmt.Errorf("failed to convert proto task to task: %w", err) + } + + level.Debug(b.logger).Log("msg", "received task", "task", task.ID) + + // TODO: Implement task processing + return nil } diff --git a/pkg/bloombuild/builder/builder_test.go b/pkg/bloombuild/builder/builder_test.go new file mode 100644 index 000000000000..ba2c7eae30bb --- /dev/null +++ b/pkg/bloombuild/builder/builder_test.go @@ -0,0 +1,123 @@ +package builder + +import ( + "context" + "fmt" + "net" + "testing" + "time" + + "github.com/go-kit/log" + "github.com/grafana/dskit/flagext" + "github.com/grafana/dskit/services" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + "github.com/grafana/loki/v3/pkg/bloombuild/protos" +) + +func Test_BuilderLoop(t *testing.T) { + logger := log.NewNopLogger() + + tasks := make([]*protos.ProtoTask, 256) + for i := range tasks { + tasks[i] = &protos.ProtoTask{ + Id: fmt.Sprintf("task-%d", i), + } + } + + server, err := newFakePlannerServer(tasks) + require.NoError(t, err) + + cfg := Config{ + PlannerAddress: server.Addr(), + } + flagext.DefaultValues(&cfg.GrpcConfig) + + builder, err := New(cfg, logger, prometheus.DefaultRegisterer) + require.NoError(t, err) + t.Cleanup(func() { + err = services.StopAndAwaitTerminated(context.Background(), builder) + require.NoError(t, err) + + server.Stop() + }) + + err = services.StartAndAwaitRunning(context.Background(), builder) + require.NoError(t, err) + + require.Eventually(t, func() bool { + return server.completedTasks == len(tasks) + }, 5*time.Second, 100*time.Millisecond) + + err = services.StopAndAwaitTerminated(context.Background(), builder) + require.NoError(t, err) + + require.True(t, server.shutdownCalled) +} + +type fakePlannerServer struct { + tasks []*protos.ProtoTask + completedTasks int + shutdownCalled bool + + addr string + grpcServer *grpc.Server +} + +func newFakePlannerServer(tasks []*protos.ProtoTask) (*fakePlannerServer, error) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + return nil, err + } + + server := &fakePlannerServer{ + tasks: tasks, + addr: lis.Addr().String(), + grpcServer: grpc.NewServer(), + } + + protos.RegisterPlannerForBuilderServer(server.grpcServer, server) + go func() { + if err := server.grpcServer.Serve(lis); err != nil { + panic(err) + } + }() + + return server, nil +} + +func (f *fakePlannerServer) Addr() string { + return f.addr +} + +func (f *fakePlannerServer) Stop() { + f.grpcServer.Stop() +} + +func (f *fakePlannerServer) BuilderLoop(srv protos.PlannerForBuilder_BuilderLoopServer) error { + // Receive Ready + if _, err := srv.Recv(); err != nil { + return fmt.Errorf("failed to receive ready: %w", err) + } + + for _, task := range f.tasks { + if err := srv.Send(&protos.PlannerToBuilder{Task: task}); err != nil { + return fmt.Errorf("failed to send task: %w", err) + } + if _, err := srv.Recv(); err != nil { + return fmt.Errorf("failed to receive task response: %w", err) + } + f.completedTasks++ + } + + // No more tasks. Wait until shutdown. + <-srv.Context().Done() + return nil +} + +func (f *fakePlannerServer) NotifyBuilderShutdown(_ context.Context, _ *protos.NotifyBuilderShutdownRequest) (*protos.NotifyBuilderShutdownResponse, error) { + f.shutdownCalled = true + return &protos.NotifyBuilderShutdownResponse{}, nil +} diff --git a/pkg/bloombuild/builder/config.go b/pkg/bloombuild/builder/config.go index ac282ccf95eb..122288c12041 100644 --- a/pkg/bloombuild/builder/config.go +++ b/pkg/bloombuild/builder/config.go @@ -1,18 +1,33 @@ package builder -import "flag" +import ( + "flag" + "fmt" + + "github.com/grafana/dskit/grpcclient" +) // Config configures the bloom-builder component. type Config struct { - // TODO: Add config + GrpcConfig grpcclient.Config `yaml:"grpc_config"` + PlannerAddress string `yaml:"planner_address"` } // RegisterFlagsWithPrefix registers flags for the bloom-planner configuration. -func (cfg *Config) RegisterFlagsWithPrefix(_ string, _ *flag.FlagSet) { - // TODO: Register flags with flagsPrefix +func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + f.StringVar(&cfg.PlannerAddress, prefix+".planner-address", "", "Hostname (and port) of the bloom planner") + cfg.GrpcConfig.RegisterFlagsWithPrefix(prefix+".grpc", f) } func (cfg *Config) Validate() error { + if cfg.PlannerAddress == "" { + return fmt.Errorf("planner address is required") + } + + if err := cfg.GrpcConfig.Validate(); err != nil { + return fmt.Errorf("grpc config is invalid: %w", err) + } + return nil } diff --git a/pkg/bloombuild/builder/metrics.go b/pkg/bloombuild/builder/metrics.go index e8f46fa02508..90a59cd4d402 100644 --- a/pkg/bloombuild/builder/metrics.go +++ b/pkg/bloombuild/builder/metrics.go @@ -8,10 +8,17 @@ import ( const ( metricsNamespace = "loki" metricsSubsystem = "bloombuilder" + + statusSuccess = "success" + statusFailure = "failure" ) type Metrics struct { running prometheus.Gauge + + taskStarted prometheus.Counter + taskCompleted *prometheus.CounterVec + taskDuration *prometheus.HistogramVec } func NewMetrics(r prometheus.Registerer) *Metrics { @@ -22,5 +29,31 @@ func NewMetrics(r prometheus.Registerer) *Metrics { Name: "running", Help: "Value will be 1 if the bloom builder is currently running on this instance", }), + + taskStarted: promauto.With(r).NewCounter(prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "task_started_total", + Help: "Total number of task started", + }), + taskCompleted: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "task_completed_total", + Help: "Total number of task completed", + }, []string{"status"}), + taskDuration: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "task_duration_seconds", + Help: "Time spent processing a task.", + // Buckets in seconds: + Buckets: append( + // 1s --> 1h (steps of 10 minutes) + prometheus.LinearBuckets(1, 600, 6), + // 1h --> 24h (steps of 1 hour) + prometheus.LinearBuckets(3600, 3600, 24)..., + ), + }, []string{"status"}), } } diff --git a/pkg/bloombuild/planner/planner.go b/pkg/bloombuild/planner/planner.go index dfb6fea80cc3..7a7954c67e05 100644 --- a/pkg/bloombuild/planner/planner.go +++ b/pkg/bloombuild/planner/planner.go @@ -607,9 +607,15 @@ func (p *Planner) forwardTaskToBuilder( } // TODO(salvacorts): Implement timeout and retry for builder response. - _, err := builder.Recv() + res, err := builder.Recv() + if err != nil { + return fmt.Errorf("error receiving response from builder (%s): %w", builderID, err) + } + if res.GetError() != "" { + return fmt.Errorf("error processing task in builder (%s): %s", builderID, res.GetError()) + } - return err + return nil } func (p *Planner) isRunningOrStopping() bool { diff --git a/pkg/bloombuild/protos/service.pb.go b/pkg/bloombuild/protos/service.pb.go index 91684dd90ef8..257206f27582 100644 --- a/pkg/bloombuild/protos/service.pb.go +++ b/pkg/bloombuild/protos/service.pb.go @@ -31,6 +31,7 @@ const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package type BuilderToPlanner struct { BuilderID string `protobuf:"bytes,1,opt,name=builderID,proto3" json:"builderID,omitempty"` + Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` } func (m *BuilderToPlanner) Reset() { *m = BuilderToPlanner{} } @@ -72,6 +73,13 @@ func (m *BuilderToPlanner) GetBuilderID() string { return "" } +func (m *BuilderToPlanner) GetError() string { + if m != nil { + return m.Error + } + return "" +} + type PlannerToBuilder struct { Task *ProtoTask `protobuf:"bytes,1,opt,name=task,proto3" json:"task,omitempty"` } @@ -205,27 +213,28 @@ func init() { } var fileDescriptor_89de33e08b859356 = []byte{ - // 323 bytes of a gzipped FileDescriptorProto + // 339 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x2e, 0xc8, 0x4e, 0xd7, 0x4f, 0xca, 0xc9, 0xcf, 0xcf, 0x4d, 0x2a, 0xcd, 0xcc, 0x49, 0xd1, 0x2f, 0x28, 0xca, 0x2f, 0xc9, 0x2f, 0xd6, 0x2f, 0x4e, 0x2d, 0x2a, 0xcb, 0x4c, 0x4e, 0xd5, 0x03, 0x73, 0x85, 0xd8, 0x20, 0xa2, 0x52, 0x22, 0xe9, 0xf9, 0xe9, 0xf9, 0x60, 0xb6, 0x3e, 0x88, 0x05, 0x91, 0x95, 0x52, 0xc4, 0x6e, - 0x44, 0x49, 0x65, 0x41, 0x6a, 0x31, 0x44, 0x89, 0x92, 0x01, 0x97, 0x80, 0x13, 0x48, 0x2e, 0xb5, + 0x44, 0x49, 0x65, 0x41, 0x6a, 0x31, 0x44, 0x89, 0x92, 0x1b, 0x97, 0x80, 0x13, 0x48, 0x2e, 0xb5, 0x28, 0x24, 0x3f, 0x20, 0x27, 0x31, 0x2f, 0x2f, 0xb5, 0x48, 0x48, 0x86, 0x8b, 0x33, 0x09, 0x22, - 0xe6, 0xe9, 0x22, 0xc1, 0xa8, 0xc0, 0xa8, 0xc1, 0x19, 0x84, 0x10, 0x50, 0xb2, 0xe4, 0x12, 0x80, - 0x2a, 0x0c, 0xc9, 0x87, 0x6a, 0x15, 0x52, 0xe5, 0x62, 0x29, 0x49, 0x2c, 0xce, 0x06, 0x2b, 0xe6, - 0x36, 0x12, 0x84, 0x98, 0x5d, 0xac, 0x17, 0x00, 0xa2, 0x42, 0x12, 0x8b, 0xb3, 0x83, 0xc0, 0xd2, - 0x4a, 0x36, 0x5c, 0x32, 0x7e, 0xf9, 0x25, 0x99, 0x69, 0x95, 0x50, 0x7d, 0xc1, 0x19, 0xa5, 0x25, - 0x29, 0xf9, 0xe5, 0x79, 0x41, 0xa9, 0x85, 0xa5, 0xa9, 0xc5, 0x25, 0x04, 0x2c, 0x96, 0xe7, 0x92, + 0xe6, 0xe9, 0x22, 0xc1, 0xa8, 0xc0, 0xa8, 0xc1, 0x19, 0x84, 0x10, 0x10, 0x12, 0xe1, 0x62, 0x4d, + 0x2d, 0x2a, 0xca, 0x2f, 0x92, 0x60, 0x02, 0xcb, 0x40, 0x38, 0x4a, 0x96, 0x5c, 0x02, 0x50, 0xed, + 0x21, 0xf9, 0x50, 0x03, 0x85, 0x54, 0xb9, 0x58, 0x4a, 0x12, 0x8b, 0xb3, 0xc1, 0x46, 0x70, 0x1b, + 0x09, 0x42, 0x6c, 0x2c, 0xd6, 0x0b, 0x00, 0x51, 0x21, 0x89, 0xc5, 0xd9, 0x41, 0x60, 0x69, 0x25, + 0x1b, 0x2e, 0x19, 0xbf, 0xfc, 0x92, 0xcc, 0xb4, 0x4a, 0xa8, 0xbe, 0xe0, 0x8c, 0xd2, 0x92, 0x94, + 0xfc, 0xf2, 0xbc, 0xa0, 0xd4, 0xc2, 0xd2, 0xd4, 0xe2, 0x12, 0xfc, 0xce, 0x51, 0x92, 0xe7, 0x92, 0xc5, 0xa1, 0xbb, 0xb8, 0x20, 0x3f, 0xaf, 0x38, 0xd5, 0xe8, 0x08, 0x23, 0x97, 0x20, 0xd4, 0x69, 0x6e, 0xf9, 0x45, 0x30, 0xb7, 0xb9, 0x73, 0x71, 0x43, 0x99, 0x3e, 0xf9, 0xf9, 0x05, 0x42, 0x12, - 0x30, 0xc7, 0xa1, 0x7b, 0x5b, 0x0a, 0x2e, 0x83, 0xee, 0x3d, 0x25, 0x06, 0x0d, 0x46, 0x03, 0x46, - 0xa1, 0x34, 0x2e, 0x51, 0xac, 0xf6, 0x0b, 0xa9, 0xc0, 0x34, 0xe2, 0xf3, 0x9c, 0x94, 0x2a, 0x01, - 0x55, 0x10, 0x4f, 0x28, 0x31, 0x38, 0xd9, 0x5c, 0x78, 0x28, 0xc7, 0x70, 0xe3, 0xa1, 0x1c, 0xc3, - 0x87, 0x87, 0x72, 0x8c, 0x0d, 0x8f, 0xe4, 0x18, 0x57, 0x3c, 0x92, 0x63, 0x3c, 0xf1, 0x48, 0x8e, - 0xf1, 0xc2, 0x23, 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x5f, 0x3c, 0x92, 0x63, 0xf8, 0xf0, 0x48, - 0x8e, 0x71, 0xc2, 0x63, 0x39, 0x86, 0x0b, 0x8f, 0xe5, 0x18, 0x6e, 0x3c, 0x96, 0x63, 0x88, 0x82, - 0xa6, 0x84, 0x24, 0x08, 0x6d, 0x0c, 0x08, 0x00, 0x00, 0xff, 0xff, 0x28, 0x86, 0x3f, 0xfe, 0x3f, + 0x30, 0xc7, 0xa1, 0x07, 0x86, 0x14, 0x5c, 0x06, 0xdd, 0x7b, 0x4a, 0x0c, 0x1a, 0x8c, 0x06, 0x8c, + 0x42, 0x69, 0x5c, 0xa2, 0x58, 0xed, 0x17, 0x52, 0x81, 0x69, 0xc4, 0xe7, 0x39, 0x29, 0x55, 0x02, + 0xaa, 0x20, 0x9e, 0x50, 0x62, 0x70, 0xb2, 0xb9, 0xf0, 0x50, 0x8e, 0xe1, 0xc6, 0x43, 0x39, 0x86, + 0x0f, 0x0f, 0xe5, 0x18, 0x1b, 0x1e, 0xc9, 0x31, 0xae, 0x78, 0x24, 0xc7, 0x78, 0xe2, 0x91, 0x1c, + 0xe3, 0x85, 0x47, 0x72, 0x8c, 0x0f, 0x1e, 0xc9, 0x31, 0xbe, 0x78, 0x24, 0xc7, 0xf0, 0xe1, 0x91, + 0x1c, 0xe3, 0x84, 0xc7, 0x72, 0x0c, 0x17, 0x1e, 0xcb, 0x31, 0xdc, 0x78, 0x2c, 0xc7, 0x10, 0x05, + 0x4d, 0x1f, 0x49, 0x10, 0xda, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff, 0x9f, 0x41, 0x32, 0x50, 0x55, 0x02, 0x00, 0x00, } @@ -251,6 +260,9 @@ func (this *BuilderToPlanner) Equal(that interface{}) bool { if this.BuilderID != that1.BuilderID { return false } + if this.Error != that1.Error { + return false + } return true } func (this *PlannerToBuilder) Equal(that interface{}) bool { @@ -326,9 +338,10 @@ func (this *BuilderToPlanner) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 5) + s := make([]string, 0, 6) s = append(s, "&protos.BuilderToPlanner{") s = append(s, "BuilderID: "+fmt.Sprintf("%#v", this.BuilderID)+",\n") + s = append(s, "Error: "+fmt.Sprintf("%#v", this.Error)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -541,6 +554,13 @@ func (m *BuilderToPlanner) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.Error) > 0 { + i -= len(m.Error) + copy(dAtA[i:], m.Error) + i = encodeVarintService(dAtA, i, uint64(len(m.Error))) + i-- + dAtA[i] = 0x12 + } if len(m.BuilderID) > 0 { i -= len(m.BuilderID) copy(dAtA[i:], m.BuilderID) @@ -660,6 +680,10 @@ func (m *BuilderToPlanner) Size() (n int) { if l > 0 { n += 1 + l + sovService(uint64(l)) } + l = len(m.Error) + if l > 0 { + n += 1 + l + sovService(uint64(l)) + } return n } @@ -710,6 +734,7 @@ func (this *BuilderToPlanner) String() string { } s := strings.Join([]string{`&BuilderToPlanner{`, `BuilderID:` + fmt.Sprintf("%v", this.BuilderID) + `,`, + `Error:` + fmt.Sprintf("%v", this.Error) + `,`, `}`, }, "") return s @@ -812,6 +837,38 @@ func (m *BuilderToPlanner) Unmarshal(dAtA []byte) error { } m.BuilderID = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Error", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowService + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthService + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthService + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Error = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipService(dAtA[iNdEx:]) diff --git a/pkg/bloombuild/protos/service.proto b/pkg/bloombuild/protos/service.proto index e061684c41be..1d172f31ec9b 100644 --- a/pkg/bloombuild/protos/service.proto +++ b/pkg/bloombuild/protos/service.proto @@ -17,6 +17,7 @@ service PlannerForBuilder { message BuilderToPlanner { string builderID = 1; + string error = 2; } message PlannerToBuilder { From 670cd89aa8ffb8b852bca05fd0adb554e93ce796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B4=AA=E9=98=BF=E5=8D=97?= Date: Wed, 29 May 2024 21:08:44 +0800 Subject: [PATCH 34/41] fix: Fix incorrect sorting of chunks in bloom-filtered response since `ChunkRef.Cmp` method is used in reverse (#12999) The `v1.ChunkRef.Cmp()` method is used in reverse, and this can lead to chunks in bloom filtered result be sorted in descending but not ascending. --- pkg/storage/bloom/v1/index.go | 6 +++--- pkg/storage/bloom/v1/index_test.go | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/storage/bloom/v1/index.go b/pkg/storage/bloom/v1/index.go index a4bcd450650c..caadfa26ddf7 100644 --- a/pkg/storage/bloom/v1/index.go +++ b/pkg/storage/bloom/v1/index.go @@ -427,14 +427,14 @@ func (r *ChunkRef) Less(other ChunkRef) bool { func (r *ChunkRef) Cmp(other ChunkRef) int { if r.From != other.From { - return int(other.From) - int(r.From) + return int(r.From) - int(other.From) } if r.Through != other.Through { - return int(other.Through) - int(r.Through) + return int(r.Through) - int(other.Through) } - return int(other.Checksum) - int(r.Checksum) + return int(r.Checksum) - int(other.Checksum) } func (r *ChunkRef) Encode(enc *encoding.Encbuf, previousEnd model.Time) model.Time { diff --git a/pkg/storage/bloom/v1/index_test.go b/pkg/storage/bloom/v1/index_test.go index 95e35617b568..45d967fb1191 100644 --- a/pkg/storage/bloom/v1/index_test.go +++ b/pkg/storage/bloom/v1/index_test.go @@ -74,42 +74,42 @@ func TestChunkRefCmpLess(t *testing.T) { desc: "From is before", left: ChunkRef{0, 1, 0}, right: ChunkRef{1, 1, 0}, - expCmp: 1, + expCmp: -1, expLess: true, }, { desc: "From is after", left: ChunkRef{1, 1, 0}, right: ChunkRef{0, 1, 0}, - expCmp: -1, + expCmp: 1, expLess: false, }, { desc: "Through is before", left: ChunkRef{0, 1, 0}, right: ChunkRef{0, 2, 0}, - expCmp: 1, + expCmp: -1, expLess: true, }, { desc: "Through is after", left: ChunkRef{0, 2, 0}, right: ChunkRef{0, 1, 0}, - expCmp: -1, + expCmp: 1, expLess: false, }, { desc: "Checksum is smaller", left: ChunkRef{0, 1, 0}, right: ChunkRef{0, 1, 1}, - expCmp: 1, + expCmp: -1, expLess: true, }, { desc: "Checksum is bigger", left: ChunkRef{0, 0, 1}, right: ChunkRef{0, 0, 0}, - expCmp: -1, + expCmp: 1, expLess: false, }, } { From 6e4555010eab5a2b12caf9af2df5f0991362d754 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Wed, 29 May 2024 17:37:09 +0200 Subject: [PATCH 35/41] fix: Introduce feature flag for [last|first]_over_time sharding. (#13067) --- docs/sources/shared/configuration.md | 3 +- pkg/logql/downstream_test.go | 55 +++++++++---------- pkg/logql/shardmapper.go | 23 +++++++- .../queryrange/queryrangebase/roundtrip.go | 2 +- 4 files changed, 51 insertions(+), 32 deletions(-) diff --git a/docs/sources/shared/configuration.md b/docs/sources/shared/configuration.md index a8a4a7df720e..aa045ce1f977 100644 --- a/docs/sources/shared/configuration.md +++ b/docs/sources/shared/configuration.md @@ -3889,7 +3889,8 @@ results_cache: [parallelise_shardable_queries: | default = true] # A comma-separated list of LogQL vector and range aggregations that should be -# sharded +# sharded. Possible values 'quantile_over_time', 'last_over_time', +# 'first_over_time'. # CLI flag: -querier.shard-aggregations [shard_aggregations: | default = ""] diff --git a/pkg/logql/downstream_test.go b/pkg/logql/downstream_test.go index 0777822dbbb6..948aef03876b 100644 --- a/pkg/logql/downstream_test.go +++ b/pkg/logql/downstream_test.go @@ -37,26 +37,27 @@ func TestMappingEquivalence(t *testing.T) { for _, tc := range []struct { query string approximate bool + shardAgg []string }{ - {`1`, false}, - {`1 + 1`, false}, - {`{a="1"}`, false}, - {`{a="1"} |= "number: 10"`, false}, - {`rate({a=~".+"}[1s])`, false}, - {`sum by (a) (rate({a=~".+"}[1s]))`, false}, - {`sum(rate({a=~".+"}[1s]))`, false}, - {`max without (a) (rate({a=~".+"}[1s]))`, false}, - {`count(rate({a=~".+"}[1s]))`, false}, - {`avg(rate({a=~".+"}[1s]))`, true}, - {`avg(rate({a=~".+"}[1s])) by (a)`, true}, - {`1 + sum by (cluster) (rate({a=~".+"}[1s]))`, false}, - {`sum(max(rate({a=~".+"}[1s])))`, false}, - {`max(count(rate({a=~".+"}[1s])))`, false}, - {`max(sum by (cluster) (rate({a=~".+"}[1s]))) / count(rate({a=~".+"}[1s]))`, false}, - {`sum(rate({a=~".+"} |= "foo" != "foo"[1s]) or vector(1))`, false}, - {`avg_over_time({a=~".+"} | logfmt | unwrap value [1s])`, false}, - {`avg_over_time({a=~".+"} | logfmt | unwrap value [1s]) by (a)`, true}, - {`quantile_over_time(0.99, {a=~".+"} | logfmt | unwrap value [1s])`, true}, + {`1`, false, nil}, + {`1 + 1`, false, nil}, + {`{a="1"}`, false, nil}, + {`{a="1"} |= "number: 10"`, false, nil}, + {`rate({a=~".+"}[1s])`, false, nil}, + {`sum by (a) (rate({a=~".+"}[1s]))`, false, nil}, + {`sum(rate({a=~".+"}[1s]))`, false, nil}, + {`max without (a) (rate({a=~".+"}[1s]))`, false, nil}, + {`count(rate({a=~".+"}[1s]))`, false, nil}, + {`avg(rate({a=~".+"}[1s]))`, true, nil}, + {`avg(rate({a=~".+"}[1s])) by (a)`, true, nil}, + {`1 + sum by (cluster) (rate({a=~".+"}[1s]))`, false, nil}, + {`sum(max(rate({a=~".+"}[1s])))`, false, nil}, + {`max(count(rate({a=~".+"}[1s])))`, false, nil}, + {`max(sum by (cluster) (rate({a=~".+"}[1s]))) / count(rate({a=~".+"}[1s]))`, false, nil}, + {`sum(rate({a=~".+"} |= "foo" != "foo"[1s]) or vector(1))`, false, nil}, + {`avg_over_time({a=~".+"} | logfmt | unwrap value [1s])`, false, nil}, + {`avg_over_time({a=~".+"} | logfmt | unwrap value [1s]) by (a)`, true, nil}, + {`quantile_over_time(0.99, {a=~".+"} | logfmt | unwrap value [1s])`, true, []string{ShardQuantileOverTime}}, { ` (quantile_over_time(0.99, {a=~".+"} | logfmt | unwrap value [1s]) by (a) > 1) @@ -64,11 +65,12 @@ func TestMappingEquivalence(t *testing.T) { avg by (a) (rate({a=~".+"}[1s])) `, false, + nil, }, - {`first_over_time({a=~".+"} | logfmt | unwrap value [1s])`, false}, - {`first_over_time({a=~".+"} | logfmt | unwrap value [1s]) by (a)`, false}, - {`last_over_time({a=~".+"} | logfmt | unwrap value [1s])`, false}, - {`last_over_time({a=~".+"} | logfmt | unwrap value [1s]) by (a)`, false}, + {`first_over_time({a=~".+"} | logfmt | unwrap value [1s])`, false, []string{ShardFirstOverTime}}, + {`first_over_time({a=~".+"} | logfmt | unwrap value [1s]) by (a)`, false, []string{ShardFirstOverTime}}, + {`last_over_time({a=~".+"} | logfmt | unwrap value [1s])`, false, []string{ShardLastOverTime}}, + {`last_over_time({a=~".+"} | logfmt | unwrap value [1s]) by (a)`, false, []string{ShardLastOverTime}}, // topk prefers already-seen values in tiebreakers. Since the test data generates // the same log lines for each series & the resulting promql.Vectors aren't deterministically // sorted by labels, we don't expect this to pass. @@ -102,12 +104,7 @@ func TestMappingEquivalence(t *testing.T) { ctx := user.InjectOrgID(context.Background(), "fake") strategy := NewPowerOfTwoStrategy(ConstantShards(shards)) - mapper := NewShardMapper(strategy, nilShardMetrics, []string{}) - // TODO (callum) refactor this test so that we won't need to set every - // possible sharding config option to true when we have multiple in the future - if tc.approximate { - mapper.quantileOverTimeSharding = true - } + mapper := NewShardMapper(strategy, nilShardMetrics, tc.shardAgg) _, _, mapped, err := mapper.Parse(params.GetExpression()) require.NoError(t, err) diff --git a/pkg/logql/shardmapper.go b/pkg/logql/shardmapper.go index 67b35b809df1..e55b01504537 100644 --- a/pkg/logql/shardmapper.go +++ b/pkg/logql/shardmapper.go @@ -13,6 +13,8 @@ import ( ) const ( + ShardLastOverTime = "last_over_time" + ShardFirstOverTime = "first_over_time" ShardQuantileOverTime = "quantile_over_time" ) @@ -20,19 +22,30 @@ type ShardMapper struct { shards ShardingStrategy metrics *MapperMetrics quantileOverTimeSharding bool + lastOverTimeSharding bool + firstOverTimeSharding bool } func NewShardMapper(strategy ShardingStrategy, metrics *MapperMetrics, shardAggregation []string) ShardMapper { quantileOverTimeSharding := false + lastOverTimeSharding := false + firstOverTimeSharding := false for _, a := range shardAggregation { - if a == ShardQuantileOverTime { + switch a { + case ShardQuantileOverTime: quantileOverTimeSharding = true + case ShardLastOverTime: + lastOverTimeSharding = true + case ShardFirstOverTime: + firstOverTimeSharding = true } } return ShardMapper{ shards: strategy, metrics: metrics, quantileOverTimeSharding: quantileOverTimeSharding, + firstOverTimeSharding: firstOverTimeSharding, + lastOverTimeSharding: lastOverTimeSharding, } } @@ -472,6 +485,10 @@ func (m ShardMapper) mapRangeAggregationExpr(expr *syntax.RangeAggregationExpr, }, bytesPerShard, nil case syntax.OpRangeTypeFirst: + if !m.firstOverTimeSharding { + return noOp(expr, m.shards.Resolver()) + } + potentialConflict := syntax.ReducesLabels(expr) if !potentialConflict && (expr.Grouping == nil || expr.Grouping.Noop()) { return m.mapSampleExpr(expr, r) @@ -499,6 +516,10 @@ func (m ShardMapper) mapRangeAggregationExpr(expr *syntax.RangeAggregationExpr, downstreams: downstreams, }, bytesPerShard, nil case syntax.OpRangeTypeLast: + if !m.lastOverTimeSharding { + return noOp(expr, m.shards.Resolver()) + } + potentialConflict := syntax.ReducesLabels(expr) if !potentialConflict && (expr.Grouping == nil || expr.Grouping.Noop()) { return m.mapSampleExpr(expr, r) diff --git a/pkg/querier/queryrange/queryrangebase/roundtrip.go b/pkg/querier/queryrange/queryrangebase/roundtrip.go index 1e0fe625f24d..d8b666f6888c 100644 --- a/pkg/querier/queryrange/queryrangebase/roundtrip.go +++ b/pkg/querier/queryrange/queryrangebase/roundtrip.go @@ -52,7 +52,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { cfg.ShardAggregations = []string{} f.Var(&cfg.ShardAggregations, "querier.shard-aggregations", - "A comma-separated list of LogQL vector and range aggregations that should be sharded") + "A comma-separated list of LogQL vector and range aggregations that should be sharded. Possible values 'quantile_over_time', 'last_over_time', 'first_over_time'.") cfg.ResultsCacheConfig.RegisterFlags(f) } From 797bb641736a2355b4f8503c147fc0c8a814f19a Mon Sep 17 00:00:00 2001 From: benclive Date: Wed, 29 May 2024 17:55:08 +0100 Subject: [PATCH 36/41] feat: Add tokenizer interface for Drain Training (#13069) --- pkg/pattern/drain/drain.go | 25 +- pkg/pattern/drain/drain_benchmark_test.go | 49 +- pkg/pattern/drain/drain_test.go | 518 ++++++++++------------ pkg/pattern/drain/line_tokenizer.go | 55 +++ pkg/pattern/drain/line_tokenizer_test.go | 60 +++ 5 files changed, 375 insertions(+), 332 deletions(-) create mode 100644 pkg/pattern/drain/line_tokenizer.go create mode 100644 pkg/pattern/drain/line_tokenizer_test.go diff --git a/pkg/pattern/drain/drain.go b/pkg/pattern/drain/drain.go index c3076386c4d6..5d7b4a3c0a83 100644 --- a/pkg/pattern/drain/drain.go +++ b/pkg/pattern/drain/drain.go @@ -25,7 +25,6 @@ package drain import ( "math" "strconv" - "strings" "unicode" "github.com/hashicorp/golang-lru/v2/simplelru" @@ -161,6 +160,7 @@ func New(config *Config, metrics *Metrics) *Drain { rootNode: createNode(), idToCluster: createLogClusterCache(config.MaxClusters, evictFn), metrics: metrics, + tokenizer: splittingTokenizer{}, // Default to this for now } return d } @@ -171,6 +171,7 @@ type Drain struct { idToCluster *LogClusterCache clustersCounter int metrics *Metrics + tokenizer LineTokenizer } func (d *Drain) Clusters() []*LogCluster { @@ -182,10 +183,13 @@ func (d *Drain) TrainTokens(tokens []string, stringer func([]string) string, ts } func (d *Drain) Train(content string, ts int64) *LogCluster { - return d.train(d.getContentAsTokens(content), nil, ts) + return d.train(d.tokenizer.Tokenize(content), d.tokenizer.Join, ts) } func (d *Drain) train(tokens []string, stringer func([]string) string, ts int64) *LogCluster { + if len(tokens) < 4 { + return nil + } matchCluster := d.treeSearch(d.rootNode, tokens, d.config.SimTh, false) // Match no existing log cluster if matchCluster == nil { @@ -215,7 +219,7 @@ func (d *Drain) train(tokens []string, stringer func([]string) string, ts int64) } func (d *Drain) TrainPattern(content string, samples []*logproto.PatternSample) *LogCluster { - tokens := tokenizePattern(content, d.config.ParamString) + tokens := deduplicatePlaceholders(d.tokenizer.Tokenize(content), d.config.ParamString) matchCluster := d.treeSearch(d.rootNode, tokens, d.config.SimTh, false) // Match no existing log cluster if matchCluster == nil { @@ -237,10 +241,6 @@ func (d *Drain) TrainPattern(content string, samples []*logproto.PatternSample) return matchCluster } -func tokenizePattern(content, param string) []string { - return deduplicatePlaceholders(strings.Split(content, " "), param) -} - func deduplicatePlaceholders(tokens []string, param string) []string { if len(tokens) < 2 { return tokens @@ -258,7 +258,7 @@ func deduplicatePlaceholders(tokens []string, param string) []string { } func (d *Drain) PatternString(c *LogCluster) string { - s := strings.Join(deduplicatePlaceholders(c.Tokens, d.config.ParamString), " ") + s := d.tokenizer.Join(deduplicatePlaceholders(c.Tokens, d.config.ParamString)) if s == d.config.ParamString { return "" } @@ -271,18 +271,11 @@ func (d *Drain) Delete(cluster *LogCluster) { // Match against an already existing cluster. Match shall be perfect (sim_th=1.0). New cluster will not be created as a result of this call, nor any cluster modifications. func (d *Drain) Match(content string) *LogCluster { - contentTokens := d.getContentAsTokens(content) + contentTokens := d.tokenizer.Tokenize(content) matchCluster := d.treeSearch(d.rootNode, contentTokens, 1.0, true) return matchCluster } -func (d *Drain) getContentAsTokens(content string) []string { - for _, extraDelimiter := range d.config.ExtraDelimiters { - content = strings.Replace(content, extraDelimiter, " ", -1) - } - return strings.Split(content, " ") -} - func (d *Drain) treeSearch(rootNode *Node, tokens []string, simTh float64, includeParams bool) *LogCluster { tokenCount := len(tokens) diff --git a/pkg/pattern/drain/drain_benchmark_test.go b/pkg/pattern/drain/drain_benchmark_test.go index cf9378025102..e03770f613c0 100644 --- a/pkg/pattern/drain/drain_benchmark_test.go +++ b/pkg/pattern/drain/drain_benchmark_test.go @@ -10,50 +10,21 @@ import ( func BenchmarkDrain_TrainExtractsPatterns(b *testing.B) { tests := []struct { - name string - drain *Drain inputFile string }{ - { - name: `Patterns for agent logfmt logs`, - inputFile: `testdata/agent-logfmt.txt`, - }, - { - name: `Patterns for ingester logfmt logs`, - inputFile: `testdata/ingester-logfmt.txt`, - }, - { - name: `Patterns for Drone json logs`, - inputFile: `testdata/drone-json.txt`, - }, - { - name: "Patterns for distributor logfmt logs", - inputFile: "testdata/distributor-logfmt.txt", - }, - { - name: "Patterns for journald logs", - inputFile: "testdata/journald.txt", - }, - { - name: "Patterns for kafka logs", - inputFile: "testdata/kafka.txt", - }, - { - name: "Patterns for kubernetes logs", - inputFile: "testdata/kubernetes.txt", - }, - { - name: "Patterns for vault logs", - inputFile: "testdata/vault.txt", - }, - { - name: "Patterns for calico logs", - inputFile: "testdata/calico.txt", - }, + {inputFile: `testdata/agent-logfmt.txt`}, + {inputFile: `testdata/ingester-logfmt.txt`}, + {inputFile: `testdata/drone-json.txt`}, + {inputFile: "testdata/distributor-logfmt.txt"}, + {inputFile: "testdata/journald.txt"}, + {inputFile: "testdata/kafka.txt"}, + {inputFile: "testdata/kubernetes.txt"}, + {inputFile: "testdata/vault.txt"}, + {inputFile: "testdata/calico.txt"}, } for _, tt := range tests { - b.Run(tt.name, func(b *testing.B) { + b.Run(tt.inputFile, func(b *testing.B) { file, err := os.Open(tt.inputFile) require.NoError(b, err) defer file.Close() diff --git a/pkg/pattern/drain/drain_test.go b/pkg/pattern/drain/drain_test.go index 754ac54b3fb7..7c502a895e7e 100644 --- a/pkg/pattern/drain/drain_test.go +++ b/pkg/pattern/drain/drain_test.go @@ -19,180 +19,121 @@ func TestDrain_TrainExtractsPatterns(t *testing.T) { outputPatternsForTestUpdate := false tests := []struct { - name string drain *Drain inputFile string patterns []string }{ { - // High variation leads to many patterns including some that are too generic (many tokens matched) and some that are too specific (too few matchers) - name: `Generate patterns on high variation logfmt logs`, drain: New(DefaultConfig(), nil), inputFile: `testdata/agent-logfmt.txt`, patterns: []string{ - `<_> caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" <_>`, - `<_> caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" <_>`, - `<_> caller=filetarget.go:326 level=info component=logs logs_config=default msg="removing directory from watcher" <_>`, - `<_> caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" <_> op=CREATE`, - `<_> caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" <_> container=\"kube-proxy\", <_> namespace=\"kube-system\", pod=\"kube-proxy-gke-ops-us-east-0-main-n2s32-1-1dd39c-32ae1dde-hmhw\", tier=\"node\"}"`, - `<_> caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" <_> conprof=\"true\", container=\"grafana\", <_> job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", <_> plan=\"free\", <_> <_> <_> <_> <_>`, - `<_> caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" <_> conprof=\"true\", container=\"hg-plugins\", <_> job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", <_> plan=\"free\", <_> <_> <_> <_> <_>`, - `<_> caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" <_> conprof=\"true\", container=\"hgrun\", <_> job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", <_> plan=\"free\", <_> <_> <_> <_> <_>`, - `<_> caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" <_> conprof=\"true\", container=\"hosted-grafana-security\", <_> job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", <_> plan=\"free\", <_> <_> <_> <_> <_>`, - `<_> caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file <_> ..."`, - `<_> caller=log.go:168 component=logs logs_config=default level=info msg="Seeked <_> - &{Offset:0 Whence:0}"`, - `<_> caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened <_>`, - `<_> caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for <_> to appear..."`, - `<_> caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"`, - `<_> caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos <_> on line 1: unexpected '\"'"`, - `<_> caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" <_>`, - `<_> caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" <_>`, - `<_> caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" <_>`, - `<_> caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" <_> reason=null`, - `<_> caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" <_>`, - `<_> caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" <_>`, - `<_> level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines <_> <_>`, `ts=2024-04-16T15:10:42.556278698Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*b92ee988-5c26-4c64-bba3-ff6a01723759/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i1111\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"orgnamehere\", plan=\"free\", pod=\"orgnamehere-grafana-7c65678f86-9zhlb\", pod_template_hash=\"7c65678f86\", resource_version=\"143638246\", slug=\"orgnamehere\", stackId=\"866772\"}"`, `ts=2024-04-16T15:10:42.556706613Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*b92ee988-5c26-4c64-bba3-ff6a01723759/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i1111\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"orgnamehere\", plan=\"free\", pod=\"orgnamehere-grafana-7c65678f86-9zhlb\", pod_template_hash=\"7c65678f86\", resource_version=\"143638246\", slug=\"orgnamehere\", stackId=\"866772\"}"`, `ts=2024-04-16T15:10:42.556930066Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*b92ee988-5c26-4c64-bba3-ff6a01723759/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i1111\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"orgnamehere\", plan=\"free\", pod=\"orgnamehere-grafana-7c65678f86-9zhlb\", pod_template_hash=\"7c65678f86\", resource_version=\"143638246\", slug=\"orgnamehere\", stackId=\"866772\"}"`, `ts=2024-04-16T15:10:42.557102408Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*b92ee988-5c26-4c64-bba3-ff6a01723759/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i1111\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"orgnamehere\", plan=\"free\", pod=\"orgnamehere-grafana-7c65678f86-9zhlb\", pod_template_hash=\"7c65678f86\", resource_version=\"143638246\", slug=\"orgnamehere\", stackId=\"866772\"}"`, `ts=2024-04-16T15:10:43.192290389Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*19a1cce8-5f04-46e0-a124-292b0dd9b343/testcoordinator/*.log:{batch_kubernetes_io_controller_uid=\"25ec5edf-f78e-468b-b6f3-3b9685f0cc8f\", batch_kubernetes_io_job_name=\"testcoordinator-job-2665838\", container=\"testcoordinator\", controller_uid=\"25ec5edf-f78e-468b-b6f3-3b9685f0cc8f\", job=\"k6-cloud/testcoordinator\", job_name=\"testcoordinator-job-2665838\", name=\"testcoordinator\", namespace=\"k6-cloud\", pod=\"testcoordinator-job-2665838-9g8ds\"}"`, `ts=2024-04-16T15:10:43.551543875Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*35649bfd-52ff-4281-9294-5f65fd5a89fc/marketplaces-api/*.log:{container=\"marketplaces-api\", job=\"grafana-com/marketplaces-api\", name=\"marketplaces-api\", namespace=\"grafana-com\", pod=\"marketplaces-api-f67ff7567-gqrvb\", pod_template_hash=\"f67ff7567\"}"`, - `ts=2024-04-16T15:10:43.869370539Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i2222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"someorg\", plan=\"free\", pod=\"someorg-grafana-666bd48cf9-7zrtv\", pod_template_hash=\"666bd48cf9\", resource_version=\"167212086\", slug=\"someorg\", stackId=\"444444\"}"`, - `ts=2024-04-16T15:10:43.869672113Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i2222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"someorg\", plan=\"free\", pod=\"someorg-grafana-666bd48cf9-7zrtv\", pod_template_hash=\"666bd48cf9\", resource_version=\"167212086\", slug=\"someorg\", stackId=\"444444\"}"`, - `ts=2024-04-16T15:10:43.869833185Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i2222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"someorg\", plan=\"free\", pod=\"someorg-grafana-666bd48cf9-7zrtv\", pod_template_hash=\"666bd48cf9\", resource_version=\"167212086\", slug=\"someorg\", stackId=\"444444\"}"`, - `ts=2024-04-16T15:10:43.870016638Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i2222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"someorg\", plan=\"free\", pod=\"someorg-grafana-666bd48cf9-7zrtv\", pod_template_hash=\"666bd48cf9\", resource_version=\"167212086\", slug=\"someorg\", stackId=\"444444\"}"`, + `ts=<_> caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget:watcher closed, tailer stopped, positions saved" path=<_>`, + `ts=<_> caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=<_>`, + `ts=<_> caller=filetarget.go:326 level=info component=logs logs_config=default msg="removing directory from watcher" directory=<_>`, + `ts=<_> caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=<_> op=CREATE`, + `ts=<_> caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key=<_> \"kube-proxy\", container=\"kube-proxy\", job=<_> namespace=\"kube-system\", pod=\"kube-proxy-gke-ops-us-east-0-main-n2s32-1-1dd39c-32ae1dde-hmhw\", tier=\"node\"}"`, + `ts=<_> caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key=<_> \"grafana\", conprof=\"true\", container=\"grafana\", instanceId=<_> job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=<_> plan=\"free\", pod=<_> pod_template_hash=<_> resource_version=<_> slug=<_> stackId=<_>`, + `ts=<_> caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key=<_> \"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=<_> job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=<_> plan=\"free\", pod=<_> pod_template_hash=<_> resource_version=<_> slug=<_> stackId=<_>`, + `ts=<_> caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key=<_> \"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=<_> job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=<_> plan=\"free\", pod=<_> pod_template_hash=<_> resource_version=<_> slug=<_> stackId=<_>`, + `ts=<_> caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key=<_> \"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=<_> job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=<_> plan=\"free\", pod=<_> pod_template_hash=<_> resource_version=<_> slug=<_> stackId=<_>`, + `ts=<_> caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file <_> ..."`, + `ts=<_> caller=log.go:168 component=logs logs_config=default level=info msg="Seeked <_> - &{Offset:0 Whence:0}"`, + `ts=<_> caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened <_>`, + `ts=<_> caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for <_> to appear..."`, + `ts=<_> caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner:token too long"`, + `ts=<_> caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos <_> on line 1:unexpected '\"'"`, + `ts=<_> caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer:exited" path=<_>`, + `ts=<_> caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine:started" path=<_>`, + `ts=<_> caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine:exited" path=<_>`, + `ts=<_> caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine:tail channel closed, stopping tailer" path=<_> reason=null`, + `ts=<_> caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=<_>`, + `ts=<_> caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=<_>`, + `ts=<_> level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=<_> duration=<_>`, }, }, { - // Lower variation leads to fewer patterns including some with limited value (single lines, no matchers) - name: `Generate patterns on low variation logfmt logs`, drain: New(DefaultConfig(), nil), inputFile: `testdata/ingester-logfmt.txt`, patterns: []string{ - `<_> caller=head.go:216 level=debug tenant=987678 msg="profile is empty after delta computation" metricName=memory`, - `<_> caller=http.go:194 level=debug <_> <_> msg="POST /ingester.v1.IngesterService/Push (200) <_>`, `ts=2024-04-17T09:52:46.363974185Z caller=http.go:194 level=debug traceID=1b48f5156a61ca69 msg="GET /debug/pprof/delta_mutex (200) 1.161082ms"`, + `ts=<_> caller=head.go:216 level=debug tenant=987678 msg="profile is empty after delta computation" metricName=memory`, + `ts=<_> caller=http.go:194 level=debug traceID=<_> orgID=<_> msg="POST /ingester.v1.IngesterService/Push (200) <_>`, }, }, { - // Lower variation logs in json leads to a high number of patterns with very few matchers - name: `Generate patterns on json formatted logs`, drain: New(DefaultConfig(), nil), inputFile: `testdata/drone-json.txt`, patterns: []string{ - `<_> <_> (flow; linux; helm)"}`, - `<_> capacity <_>`, - `<_> capacity changes <_>`, - `<_> server <_>`, - `<_> unfinished <_>`, - `{"id":"15eSzaEG0enf86Kl","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:00:57Z"}`, - `{"id":"1Lu90T1fWzsWOKlc","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:58:27Z"}`, - `{"id":"3K61Yf6ImKYexoFx","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:54:56Z"}`, - `{"id":"4G6Srn6lSwzYrx19","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:58:57Z"}`, - `{"id":"4XjwwNoOwZFaWePQ","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:57:57Z"}`, - `{"id":"4rNxIlhDKxGgzBHe","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:49:22Z"}`, - `{"id":"5qEIzErDfiALVPAN","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:50:22Z"}`, - `{"id":"96TvvsMzSkkaW8oW","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:53:53Z"}`, - `{"id":"9eA72xOtx8kzMhXn","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:01:58Z"}`, - `{"id":"C7aYn8cb4NCrkkYI","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:53:23Z"}`, - `{"id":"CMG7ZwwYqNPBonAn","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:52:53Z"}`, - `{"id":"D4Oh1ivB6cdLWa08","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:48:52Z"}`, - `{"id":"FQ8wCQfaR9W387cH","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:56:56Z"}`, - `{"id":"Hhwn7ecXjxF67DG6","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:56:26Z"}`, - `{"id":"HphRnJOM8uYohf1p","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:51:53Z"}`, - `{"id":"IQy23J3NON0BV10V","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:57:26Z"}`, - `{"id":"JO1OT5ADoNA8NYqr","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:00:27Z"}`, - `{"id":"SmbOO0l5aADX9BaQ","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:54:23Z"}`, - `{"id":"T0I8Dsnw3uSi3Gal","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:02:28Z"}`, - `{"id":"Xz2OCJhgeBSRFyoN","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:59:57Z"}`, - `{"id":"focV9BzODwRbWwKE","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:52:23Z"}`, - `{"id":"luflyGZvZnLzhQEH","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:55:56Z"}`, - `{"id":"m3n8GndhG45uGIQA","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:51:23Z"}`, - `{"id":"m6SpYHzdXrDAFqDR","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:02:58Z"}`, - `{"id":"nTO38tWtnvRWRl1G","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:50:52Z"}`, - `{"id":"pPc2ORUhHAhFgBg3","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:59:27Z"}`, - `{"id":"pet7QVfO1yE8fk56","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:01:28Z"}`, - `{"id":"q20GZcvyzMwrTGx5","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:55:26Z"}`, - `{"id":"q61oHTtF4MMiQVGH","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:49:52Z"}`, - `{"id":"q62wCcIkEOueqFKF","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:03:28Z"}`, + `{"duration":<_> "debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":<_> <_> <_> "GrafanaAgent/v0.40.3 (flow; linux; helm)"}`, + `{"id":<_> "debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":<_> <_> <_>`, + `{"id":<_> "debug","msg":"calculate server capacity","time":<_> <_> <_>`, + `{"id":<_> "debug","msg":"calculate unfinished jobs","time":<_> <_> <_>`, + `{"id":<_> "debug","msg":"check capacity complete","time":<_> <_> <_>`, + `{"id":<_> "debug","msg":"no capacity changes required","time":<_> <_> <_>`, }, }, { - name: "Patterns for distributor logs", drain: New(DefaultConfig(), nil), inputFile: "testdata/distributor-logfmt.txt", patterns: []string{ - `<_> caller=http.go:194 level=debug <_> <_> msg="POST <_> <_> <_>`, + `ts=2024-05-02T12:17:22.115385619Z caller=http.go:194 level=debug traceID=7836a12bb7f1964e orgID=75 msg="POST /ingest?aggregationType=sum&from=1714652227107641016&name=checkoutservice%7B__session_id__%3D294b9729f5a7de95%2Cnamespace%3Dotel-demo%7D&sampleRate=100&spyName=gospy&units=samples&until=1714652242109516917 (200) 1.562143ms"`, + `ts=2024-05-02T12:17:22.242343806Z caller=http.go:194 level=debug traceID=404c6a83a18e66a4 orgID=75 msg="POST /ingest?aggregationType=average&from=1714652227232613927&name=checkoutservice%7B__session_id__%3D294b9729f5a7de95%2Cnamespace%3Dotel-demo%7D&sampleRate=0&spyName=gospy&units=goroutines&until=1714652242232506798 (200) 2.902485ms"`, + `ts=<_> caller=http.go:194 level=debug traceID=<_> orgID=1819 msg="POST /pyroscope/ingest?aggregationType=sum&from=1714652230&name=<_> 0&spyName=scrape&units=samples&until=1714652240 (200) <_>`, + `ts=<_> caller=http.go:194 level=debug traceID=<_> orgID=75 msg="POST /ingest?aggregationType=&from=1714652227232613927&name=checkoutservice%7B__session_id__%3D294b9729f5a7de95%2Cnamespace%3Dotel-demo%7D&sampleRate=<_> gospy&units=&until=1714652242232506798 (200) <_>`, + `ts=<_> caller=http.go:194 level=debug traceID=<_> orgID=<_> msg="POST /push.v1.PusherService/Push <_> <_>`, }, }, { - name: "Patterns for journald logs", drain: New(DefaultConfig(), nil), inputFile: "testdata/journald.txt", patterns: []string{ - ` exec /bin/hgrun -log.level=debug launch -bundledPluginsManifest /proc/$(pidof plugins-pause)/root/manifest.json -bundledPluginsDir /proc/$(pidof <_> -profile-port=6060 <_> {{536870912 0} {} BinarySI},},Requests:ResourceList{cpu: {{26 -3} {} 26m DecimalSI},memory: {{293601280 0} {} BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{},LivenessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/api/health,Port:{0 80 },Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:300,TimeoutSeconds:10,PeriodSeconds:30,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:&ExecAction{Command:[/bin/hgrun check],},HTTPGet:nil,TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:0,TimeoutSeconds:30,PeriodSeconds:30,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:&Lifecycle{PostStart:nil,PreStop:&LifecycleHandler{Exec:&ExecAction{Command:[/bin/hgrun drain -timeout 1m0s -waitTime 55s],},HTTPGet:nil,TCPSocket:nil,},},TerminationMessagePath:/dev/termination-log,ImagePullPolicy:Always,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[SYS_PTRACE],Drop:[],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod <_> ErrImagePull: [rpc error: code = NotFound desc = failed to pull and unpack image <_> failed to resolve reference <_> <_> not found, failed to pull and unpack image <_> failed to resolve reference <_> unexpected status from HEAD request to <_> 403 Forbidden]`, + ` exec /bin/hgrun -log.level=debug launch -bundledPluginsManifest /proc/$(pidof plugins-pause)/root/manifest.json -bundledPluginsDir /proc/$(pidof plugins-pause)/root/plugins],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:http-metrics,HostPort:0,ContainerPort:80,Protocol:TCP,HostIP:,},ContainerPort{Name:grpc,HostPort:0,ContainerPort:10000,Protocol:TCP,HostIP:,},ContainerPort{Name:profiling,HostPort:0,ContainerPort:6060,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:HG_API,Value:http://hosted-grafana-api,ValueFrom:nil,},EnvVar{Name:HG_INSTANCE_SLUG,Value:<_> nil,},EnvVar{Name:HG_INSTANCE_SECRET,Value:<_> nil,},EnvVar{Name:EXTRA_OPTIONS,Value:-profile -profile-port=6060 -profile-addr=0.0.0.0,ValueFrom:nil,},EnvVar{Name:HG_CREATE_TIME_MS,Value:<_> nil,},EnvVar{Name:HG_PULL_POLICY,Value:Always,ValueFrom:nil,},EnvVar{Name:HG_START_REASON,Value:active,ValueFrom:nil,},EnvVar{Name:HGRUN_SECURE_PLUGINS,Value:false,ValueFrom:nil,},EnvVar{Name:HGRUN_PLUGIN_RUNNER_ROOT_CA,Value:false,ValueFrom:nil,},EnvVar{Name:OTEL_EXPORTER_OTLP_TRACES_ENDPOINT,Value:http://jaeger-agent.jaeger.svc.cluster.local:4317,ValueFrom:nil,},EnvVar{Name:JAEGER_SAMPLER_PARAM,Value:1,ValueFrom:nil,},EnvVar{Name:OTEL_RESOURCE_ATTRIBUTES,Value:cluster=dev-us-central-0,namespace=hosted-grafana,ValueFrom:nil,},EnvVar{Name:HG_PROBE_PATH,Value:/api/health,ValueFrom:nil,},EnvVar{Name:HGRUN_EXIT_ON_PLUGIN_FAIL,Value:true,ValueFrom:nil,},EnvVar{Name:HGRUN_PLUGIN_INSTALL_RETRIES,Value:2,ValueFrom:nil,},EnvVar{Name:HGRUN_PLUGIN_INSTALL_CONCURRENCY,Value:1,ValueFrom:nil,},EnvVar{Name:HGRUN_LAUNCH_TIMEOUT,Value:3m0s,ValueFrom:nil,},EnvVar{Name:GOMEMLIMIT,Value:429496730,ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{memory: {{536870912 0} {} BinarySI},},Requests:ResourceList{cpu: {{26 -3} {} 26m DecimalSI},memory: {{293601280 0} {} BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{},LivenessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/api/health,Port:{0 80 },Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:300,TimeoutSeconds:10,PeriodSeconds:30,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:&ExecAction{Command:[/bin/hgrun check],},HTTPGet:nil,TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:0,TimeoutSeconds:30,PeriodSeconds:30,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:&Lifecycle{PostStart:nil,PreStop:&LifecycleHandler{Exec:&ExecAction{Command:[/bin/hgrun drain -timeout 1m0s -waitTime 55s],},HTTPGet:nil,TCPSocket:nil,},},TerminationMessagePath:/dev/termination-log,ImagePullPolicy:Always,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[SYS_PTRACE],Drop:[],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod <_> ErrImagePull: [rpc error: code =NotFound desc =failed to pull and unpack image "us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> failed to resolve reference "us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> not found, failed to pull and unpack image "us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> failed to resolve reference "us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> unexpected status from HEAD request to https:<_> 403 Forbidden]`, ` ln --force -s /proc/$(pidof hgrun-pause)/root/bin/hgrun /bin/hgrun;`, ` while [ "$(pidof plugins-pause)" = "" ]; do sleep 0.5; done;`, ` ts=2024-05-07T11:59:32.025687537Z level=error caller=http_client.go:56 app=hgrun hgrun_version=0.1.453-59-gf3f63162a msg="request`, - ` >`, + ` ts=2024-05-07T11:59:<_> level=error caller=http_client.go:56 app=hgrun <_> msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health`, `2024-05-07T11:59:43.484606Z INFO ExtHandler ExtHandler Downloading agent manifest`, - `<_> Consumed <_> CPU time.`, - `<_> Deactivated successfully.`, - `<_> INFO TelemetryEventsCollector ExtHandler Collected 2 events for extension: Microsoft.Azure.Extensions.CustomScript`, - `<_> level=error caller=http_client.go:56 app=hgrun <_> msg="request failed" error="Get \"http://127.0.0.1:3000/api/health\": dial tcp 127.0.0.1:3000: connect: connection refused" method=GET url=http://127.0.0.1:3000/api/health`, - `<_> level=error msg="ContainerStatus for <_> failed" error="rpc error: code = NotFound desc = an error occurred when try to find container <_> not found"`, - `<_> level=error msg="ExecSync for <_> failed" error="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task <_> not found: not found"`, - `<_> level=error msg="PullImage <_> failed" error="failed to pull and unpack image <_> failed to resolve reference <_> unexpected status from HEAD request to <_> 403 Forbidden"`, - `<_> level=error msg="PullImage <_> failed" error="rpc error: code = NotFound desc = failed to pull and unpack image <_> failed to resolve reference <_> <_> not found"`, - `<_> level=info msg="CreateContainer within sandbox <_> for <_> returns container id <_>`, - `<_> level=info msg="CreateContainer within sandbox <_> for container <_>`, - `<_> level=info msg="ImageCreate event <_> labels:{key:\"io.cri-containerd.image\" value:\"managed\"}"`, - `<_> level=info msg="ImageUpdate event <_> labels:{key:\"io.cri-containerd.image\" value:\"managed\"}"`, - `<_> level=info msg="PullImage <_>`, - `<_> level=info msg="PullImage <_> returns image reference <_>`, - `<_> level=info msg="Pulled image <_> with image id <_> repo tag <_> repo digest <_> size <_> in <_>`, - `<_> level=info msg="RemoveContainer for <_>`, - `<_> level=info msg="RemoveContainer for <_> returns successfully"`, - `<_> level=info msg="StartContainer for <_>`, - `<_> level=info msg="StartContainer for <_> returns successfully"`, - `<_> level=info msg="cleaning up dead shim" namespace=k8s.io`, - `<_> level=info msg="shim disconnected" <_> namespace=k8s.io`, - `<_> level=info msg="stop pulling image <_> active requests=0, bytes <_>`, - `<_> level=info msg="trying next host - response was http.StatusNotFound" host=us.gcr.io`, - `<_> level=warning msg="cleaning up after shim disconnected" <_> namespace=k8s.io`, - `AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" <_> comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined"`, - `E0507 11:59:29.725681 3089 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"azure-resourcemanager-exporter\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=azure-resourcemanager-exporter pod=azure-resourcemanager-exporter-6b5b58c666-rsttd_infra-exporters(5a95f801-309c-4f33-864a-406262c6ece6)\"" pod="infra-exporters/azure-resourcemanager-exporter-6b5b58c666-rsttd" podUID="5a95f801-309c-4f33-864a-406262c6ece6"`, - `E0507 11:59:31.554203 4531 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"frontend\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=frontend pod=otel-demo-alt-dev-frontend-79ccf98858-mbj4x_otel-demo-alt(d08e620e-00d0-49f1-a195-820a62e8de8f)\"" pod="otel-demo-alt/otel-demo-alt-dev-frontend-79ccf98858-mbj4x" podUID="d08e620e-00d0-49f1-a195-820a62e8de8f"`, - `E0507 11:59:31.928148 4734 pod_workers.go:1300] "Error syncing pod, skipping" err="unmounted volumes=[terraform-drift-detector-data], unattached volumes=[terraform-drift-detector-data], failed to process volumes=[]: context deadline exceeded" pod="terraform-drift-detector/terraform-drift-detector-d68b4c545-jg2vj" podUID="6c607496-ef26-454e-b2f2-4cb75b233fa3"`, + `2024-05-07T11:59:<_> INFO TelemetryEventsCollector ExtHandler Collected 2 events for extension: Microsoft.Azure.Extensions.CustomScript`, + `<_> Consumed <_> CPU time.`, + `<_> Deactivated successfully.`, + `AVC apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=<_> comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined"`, + `E0507 11:59:29.725681 3089 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"azure-resourcemanager-exporter\" with CrashLoopBackOff:\"back-off 5m0s restarting failed container=azure-resourcemanager-exporter pod=azure-resourcemanager-exporter-6b5b58c666-rsttd_infra-exporters(5a95f801-309c-4f33-864a-406262c6ece6)\"" pod="infra-exporters/azure-resourcemanager-exporter-6b5b58c666-rsttd" podUID="5a95f801-309c-4f33-864a-406262c6ece6"`, + `E0507 11:59:31.554203 4531 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"frontend\" with CrashLoopBackOff:\"back-off 5m0s restarting failed container=frontend pod=otel-demo-alt-dev-frontend-79ccf98858-mbj4x_otel-demo-alt(d08e620e-00d0-49f1-a195-820a62e8de8f)\"" pod="otel-demo-alt/otel-demo-alt-dev-frontend-79ccf98858-mbj4x" podUID="d08e620e-00d0-49f1-a195-820a62e8de8f"`, + `E0507 11:59:31.928148 4734 pod_workers.go:1300] "Error syncing pod, skipping" err="unmounted volumes=[terraform-drift-detector-data], unattached volumes=[terraform-drift-detector-data], failed to process volumes=[]:context deadline exceeded" pod="terraform-drift-detector/terraform-drift-detector-d68b4c545-jg2vj" podUID="6c607496-ef26-454e-b2f2-4cb75b233fa3"`, `E0507 11:59:34.856101 4727 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana-render-security\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-security:0.1.181\\\"\"" pod="integration/grafana-render-service-cbff479fc-cj9tp" podUID="0e3114d1-2f3a-49d6-a71d-dbc75050d8e0"`, `E0507 11:59:34.923938 3027 kuberuntime_manager.go:1261] container &Container{Name:mysqld-exporter,Image:prom/mysqld-exporter:v0.13.0,Command:[],Args:[--collect.info_schema.innodb_metrics],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:http-metrics,HostPort:0,ContainerPort:9104,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:MYSQL_USER,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:testcrossplane-user-exporter,},Key:username,Optional:nil,},},},EnvVar{Name:MYSQL_PASSWORD,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:testcrossplane-user-exporter,},Key:password,Optional:nil,},},},EnvVar{Name:MYSQL_HOST,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:testcrossplane-user-exporter,},Key:endpoint,Optional:nil,},},},EnvVar{Name:MYSQL_PORT,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:testcrossplane-user-exporter,},Key:port,Optional:nil,},},},EnvVar{Name:MYSQL_TLS_MODE,Value:preferred,ValueFrom:nil,},EnvVar{Name:DATA_SOURCE_NAME,Value:$(MYSQL_USER):$(MYSQL_PASSWORD)@tcp($(MYSQL_HOST):$(MYSQL_PORT))/?tls=$(MYSQL_TLS_MODE),ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:kube-api-access-dzx7d,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod testcrossplane-exporter-c67cfc58f-vbzl4_crossplane-playground(3d49134d-3378-4ec3-824c-5ff4ea2590a5): CreateContainerConfigError: secret "testcrossplane-user-exporter" not found`, `E0507 11:59:34.923984 3027 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"mysqld-exporter\" with CreateContainerConfigError: \"secret \\\"testcrossplane-user-exporter\\\" not found\"" pod="crossplane-playground/testcrossplane-exporter-c67cfc58f-vbzl4" podUID="3d49134d-3378-4ec3-824c-5ff4ea2590a5"`, - `E0507 11:59:35.928465 4734 pod_workers.go:1300] "Error syncing pod, skipping" err="unmounted volumes=[custom-grafana-agent], unattached volumes=[], failed to process volumes=[]: context deadline exceeded" pod="loki-dev-010/custom-grafana-agent-856948968f-6jfks" podUID="17b244cc-ecb9-4fbc-beaa-8fa47fafe013"`, - `E0507 11:59:37.252214 4736 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"ksm\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=ksm pod=new-relic-nri-bundle-nrk8s-ksm-6c785668f5-jcxh2_integration(f7cc3cca-2ffb-4fde-a73e-a4ba8b0f6b3c)\"" pod="integration/new-relic-nri-bundle-nrk8s-ksm-6c785668f5-jcxh2" podUID="f7cc3cca-2ffb-4fde-a73e-a4ba8b0f6b3c"`, - `E0507 11:59:39.149450 4729 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"cluster-agent\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=cluster-agent pod=appdynamics-cluster-agent-appdynamics-cluster-agent-56667dmbnkv_integration(69bc5e6c-0451-443e-af8a-c831871afbb8)\"" pod="integration/appdynamics-cluster-agent-appdynamics-cluster-agent-56667dmbnkv" podUID="69bc5e6c-0451-443e-af8a-c831871afbb8"`, + `E0507 11:59:35.928465 4734 pod_workers.go:1300] "Error syncing pod, skipping" err="unmounted volumes=[custom-grafana-agent], unattached volumes=[], failed to process volumes=[]:context deadline exceeded" pod="loki-dev-010/custom-grafana-agent-856948968f-6jfks" podUID="17b244cc-ecb9-4fbc-beaa-8fa47fafe013"`, + `E0507 11:59:37.252214 4736 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"ksm\" with CrashLoopBackOff:\"back-off 5m0s restarting failed container=ksm pod=new-relic-nri-bundle-nrk8s-ksm-6c785668f5-jcxh2_integration(f7cc3cca-2ffb-4fde-a73e-a4ba8b0f6b3c)\"" pod="integration/new-relic-nri-bundle-nrk8s-ksm-6c785668f5-jcxh2" podUID="f7cc3cca-2ffb-4fde-a73e-a4ba8b0f6b3c"`, + `E0507 11:59:39.149450 4729 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"cluster-agent\" with CrashLoopBackOff:\"back-off 5m0s restarting failed container=cluster-agent pod=appdynamics-cluster-agent-appdynamics-cluster-agent-56667dmbnkv_integration(69bc5e6c-0451-443e-af8a-c831871afbb8)\"" pod="integration/appdynamics-cluster-agent-appdynamics-cluster-agent-56667dmbnkv" podUID="69bc5e6c-0451-443e-af8a-c831871afbb8"`, `E0507 11:59:41.375655 4736 kuberuntime_manager.go:1256] container &Container{Name:ruler,Image:grafana/enterprise-metrics:v2.12.0,Command:[],Args:[-target=ruler -config.expand-env=true -config.file=/etc/mimir/mimir.yaml -distributor.remote-timeout=10s],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:http-metrics,HostPort:0,ContainerPort:8080,Protocol:TCP,HostIP:,},ContainerPort{Name:grpc,HostPort:0,ContainerPort:9095,Protocol:TCP,HostIP:,},ContainerPort{Name:memberlist,HostPort:0,ContainerPort:7946,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:JAEGER_AGENT_HOST,Value:alloy-otlp.alloy-otlp.svc.cluster.local.,ValueFrom:nil,},EnvVar{Name:JAEGER_TAGS,Value:namespace=ge-metrics-federation,cluster=dev-us-central-0,ValueFrom:nil,},EnvVar{Name:JAEGER_SAMPLER_MANAGER_HOST_PORT,Value:http://alloy-otlp.alloy-otlp.svc.cluster.local.:5778/sampling,ValueFrom:nil,},EnvVar{Name:GOOGLE_APPLICATION_CREDENTIALS,Value:/var/secrets/google/credentials.json,ValueFrom:nil,},EnvVar{Name:AM_TOKEN,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:ruler-alertmanager-token,},Key:token,Optional:nil,},},},EnvVar{Name:JAEGER_REPORTER_MAX_QUEUE_SIZE,Value:1000,ValueFrom:nil,},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{cpu: {{100 -3} {} 100m DecimalSI},memory: {{134217728 0} {} BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:gcs-credentials,ReadOnly:false,MountPath:/var/secrets/google/,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:config,ReadOnly:false,MountPath:/etc/mimir,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:license,ReadOnly:false,MountPath:/license,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:runtime-config,ReadOnly:false,MountPath:/var/mimir,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:storage,ReadOnly:false,MountPath:/data,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:active-queries,ReadOnly:false,MountPath:/active-query-tracker,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-jtnbs,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/ready,Port:{1 0 http-metrics},Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:45,TimeoutSeconds:1,PeriodSeconds:10,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:*true,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod gem-mimir-ruler-5f56f7846b-fgxdm_ge-metrics-federation(07c06e21-137b-4fdd-b7d3-703f0a567720): CreateContainerConfigError: secret "ruler-alertmanager-token" not found`, - `E0507 <_> 4731 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"overrides-exporter\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/kubernetes-dev/enterprise-logs:callum-shard-firstlast-08\\\"\"" pod="loki-dev-010/overrides-exporter-98c77fd66-6zj6m" podUID="1ff5bf3e-5856-4f6f-ae04-273f2dee170b"`, - `E0507 <_> 4733 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=prometheus pod=bryan-prometheus-0_bryan-prometheus(6dadfe71-eb19-4231-a96e-c64bb5499a1e)\"" pod="bryan-prometheus/bryan-prometheus-0" podUID="6dadfe71-eb19-4231-a96e-c64bb5499a1e"`, - `E0507 <_> <_> kuberuntime_manager.go:1256] container &Container{Name:pdc,Image:us.gcr.io/hosted-grafana/pdc:0.1.415,Command:[],Args:[-proxy.auth.ca-keys-dir=/var/run/secrets/pdc-certs -proxy.socks-server.addr=:10443 -proxy.ssh-server.addr=:2222 -proxy.use-socks-username-for-routing -proxy.api.http-address=:9182 -proxy.check-connpool-address-in-ring -memberlist.join=dns+gossip-ring.pdc.svc.cluster.local:7946 -api.http-address=:11443 -distributor.enabled=true -distributor.addr=:10444 -distributor.use-socks-username-for-routing -gateway.enabled=true -gateway.addr=:2244 -log.level=debug -certs.ca-private-key-file=/var/run/secrets/pdc-certs/ca.key -certs.ca-cert-file=/var/run/secrets/pdc-certs/ca.crt -certs.ca-pub-file=/var/run/secrets/pdc-certs/ca.pub -certs.cluster=local-k8s -shard-size=3 -graceful-shutdown-period=30s -enable-multiple-networks],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:socks,HostPort:0,ContainerPort:10443,Protocol:TCP,HostIP:,},ContainerPort{Name:ssh,HostPort:0,ContainerPort:2222,Protocol:TCP,HostIP:,},ContainerPort{Name:distributor,HostPort:0,ContainerPort:10444,Protocol:TCP,HostIP:,},ContainerPort{Name:gateway,HostPort:0,ContainerPort:2244,Protocol:TCP,HostIP:,},ContainerPort{Name:api,HostPort:0,ContainerPort:11443,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:POD_NAME,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.name,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},},Resources:ResourceRequirements{Limits:ResourceList{cpu: {{500 -3} {} 500m DecimalSI},memory: {{134217728 0} {} BinarySI},},Requests:ResourceList{cpu: {{250 -3} {} 250m DecimalSI},memory: {{67108864 0} {} <_> 11443 },Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:40,TimeoutSeconds:1,PeriodSeconds:5,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:&Lifecycle{PostStart:nil,PreStop:&LifecycleHandler{Exec:&ExecAction{Command:[/bin/sleep 5],},HTTPGet:nil,TCPSocket:nil,},},TerminationMessagePath:/dev/termination-log,ImagePullPolicy:Never,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod <_> ErrImageNeverPull: Container image "us.gcr.io/hosted-grafana/pdc:0.1.415" is not present with pull policy of Never`, - `E0507 <_> <_> kuberuntime_manager.go:1256] container &Container{Name:ruler,Image:grafana/enterprise-metrics:v2.11.1,Command:[],Args:[-target=ruler -config.expand-env=true <_> {{100 -3} {} 100m DecimalSI},memory: {{134217728 0} {} <_> 0 http-metrics},Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:45,TimeoutSeconds:1,PeriodSeconds:10,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:*true,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod <_> CreateContainerConfigError: secret "ruler-alertmanager-token" not found`, - `E0507 <_> <_> kuberuntime_manager.go:1256] container <_> set -e; while [ "$(pidof hgrun-pause)" = "" ]; do sleep 0.5; done;`, - `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"agent\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=agent <_> <_> <_>`, - `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"cortex-gw\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=cortex-gw <_> <_> <_>`, - `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"gcom-sync\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/kubernetes-dev/frontend-monitoring:6a8eb5a\\\"\"" <_> <_>`, - `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"goldpinger\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=goldpinger <_> <_> <_>`, - `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff: \"back-off <_> restarting failed container=grafana <_> <_> <_>`, - `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ErrImagePull: \"[rpc error: code = NotFound desc = failed to pull and unpack image <_> failed to resolve reference <_> <_> not found, failed to pull and unpack image <_> failed to resolve reference <_> unexpected status from HEAD request to <_> 403 Forbidden]\"" <_> <_>`, - `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image <_> <_> <_>`, - `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"pdc\" with ErrImageNeverPull: \"Container image \\\"us.gcr.io/hosted-grafana/pdc:0.1.415\\\" is not present with pull policy of Never\"" <_> <_>`, - `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"ruler\" with CreateContainerConfigError: \"secret \\\"ruler-alertmanager-token\\\" not found\"" <_> <_>`, - `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"support-agent\" with CrashLoopBackOff: \"back-off 5m0s restarting failed container=support-agent <_> <_> <_>`, - `E0507 <_> <_> prober.go:104] "Probe errored" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task <_> not found: not found" probeType="Readiness" <_> <_> containerName="grafana"`, - `E0507 <_> <_> prober.go:239] "Unable to write all bytes from execInContainer" err="short write" <_> actualBytes=10240`, - `E0507 <_> <_> remote_image.go:180] "PullImage from image service failed" err="rpc error: code = NotFound desc = failed to pull and unpack image <_> failed to resolve reference <_> <_> not found" <_>`, - `E0507 <_> <_> remote_image.go:180] "PullImage from image service failed" err="rpc error: code = Unknown desc = failed to pull and unpack image <_> failed to resolve reference <_> unexpected status from HEAD request to <_> 403 Forbidden" <_>`, - `E0507 <_> <_> remote_runtime.go:432] "ContainerStatus from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container <_> not found" <_>`, - `E0507 <_> <_> remote_runtime.go:496] "ExecSync cmd from runtime service failed" err="rpc error: code = NotFound desc = failed to exec in container: failed to load task: no running task found: task <_> not found: not found" <_> cmd=["/bin/hgrun","check"]`, + `E0507 11:59:<_> 4731 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"overrides-exporter\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/kubernetes-dev/enterprise-logs:callum-shard-firstlast-08\\\"\"" pod="loki-dev-010/overrides-exporter-98c77fd66-6zj6m" podUID="1ff5bf3e-5856-4f6f-ae04-273f2dee170b"`, + `E0507 11:59:<_> <_> kuberuntime_manager.go:1256] container &Container{Name:grafana,Image:us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> [/bin/sh],Args:[-c set -e; while [ "$(pidof hgrun-pause)" ="" ]; do sleep 0.5; done;`, + `E0507 11:59:<_> <_> kuberuntime_manager.go:1256] container &Container{Name:pdc,Image:us.gcr.io/hosted-grafana/pdc:0.1.415,Command:[],Args:[-proxy.auth.ca-keys-dir=/var/run/secrets/pdc-certs -proxy.socks-server.addr=:10443 -proxy.ssh-server.addr=:2222 -proxy.use-socks-username-for-routing -proxy.api.http-address=:9182 -proxy.check-connpool-address-in-ring -memberlist.join=dns+gossip-ring.pdc.svc.cluster.local:7946 -api.http-address=:11443 -distributor.enabled=true -distributor.addr=:10444 -distributor.use-socks-username-for-routing -gateway.enabled=true -gateway.addr=:2244 -log.level=debug -certs.ca-private-key-file=/var/run/secrets/pdc-certs/ca.key -certs.ca-cert-file=/var/run/secrets/pdc-certs/ca.crt -certs.ca-pub-file=/var/run/secrets/pdc-certs/ca.pub -certs.cluster=local-k8s -shard-size=3 -graceful-shutdown-period=30s -enable-multiple-networks],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:socks,HostPort:0,ContainerPort:10443,Protocol:TCP,HostIP:,},ContainerPort{Name:ssh,HostPort:0,ContainerPort:2222,Protocol:TCP,HostIP:,},ContainerPort{Name:distributor,HostPort:0,ContainerPort:10444,Protocol:TCP,HostIP:,},ContainerPort{Name:gateway,HostPort:0,ContainerPort:2244,Protocol:TCP,HostIP:,},ContainerPort{Name:api,HostPort:0,ContainerPort:11443,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:POD_NAME,Value:,ValueFrom:&EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.name,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,},},},Resources:ResourceRequirements{Limits:ResourceList{cpu: {{500 -3} {} 500m DecimalSI},memory: {{134217728 0} {} BinarySI},},Requests:ResourceList{cpu: {{250 -3} {} 250m DecimalSI},memory: {{67108864 0} {} BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:pdc-certs,ReadOnly:true,MountPath:/var/run/secrets/pdc-certs,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:<_> true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/ready,Port:{0 11443 },Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:40,TimeoutSeconds:1,PeriodSeconds:5,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:&Lifecycle{PostStart:nil,PreStop:&LifecycleHandler{Exec:&ExecAction{Command:[/bin/sleep 5],},HTTPGet:nil,TCPSocket:nil,},},TerminationMessagePath:/dev/termination-log,ImagePullPolicy:Never,SecurityContext:nil,Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod <_> ErrImageNeverPull: Container image "us.gcr.io/hosted-grafana/pdc:0.1.415" is not present with pull policy of Never`, + `E0507 11:59:<_> <_> kuberuntime_manager.go:1256] container &Container{Name:ruler,Image:grafana/enterprise-metrics:v2.11.1,Command:[],Args:[-target=ruler -config.expand-env=true -config.file=/etc/mimir/mimir.yaml],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:http-metrics,HostPort:0,ContainerPort:8080,Protocol:TCP,HostIP:,},ContainerPort{Name:grpc,HostPort:0,ContainerPort:9095,Protocol:TCP,HostIP:,},ContainerPort{Name:memberlist,HostPort:0,ContainerPort:7946,Protocol:TCP,HostIP:,},},Env:[]EnvVar{EnvVar{Name:JAEGER_AGENT_HOST,Value:<_> nil,},EnvVar{Name:JAEGER_TAGS,Value:namespace=ge-metrics-federation,cluster=dev-us-central-0,ValueFrom:nil,},EnvVar{Name:JAEGER_SAMPLER_MANAGER_HOST_PORT,Value:http:<_> 5778/sampling,ValueFrom:nil,},EnvVar{Name:GOOGLE_APPLICATION_CREDENTIALS,Value:/var/secrets/google/credentials.json,ValueFrom:nil,},EnvVar{Name:AM_TOKEN,Value:,ValueFrom:&EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:ruler-alertmanager-token,},Key:token,Optional:nil,},},},},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{cpu: {{100 -3} {} 100m DecimalSI},memory: {{134217728 0} {} BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:gcs-credentials,ReadOnly:false,MountPath:/var/secrets/google/,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:config,ReadOnly:false,MountPath:/etc/mimir,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:license,ReadOnly:false,MountPath:/license,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:runtime-config,ReadOnly:false,MountPath:/var/mimir,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:storage,ReadOnly:false,MountPath:/data,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:active-queries,ReadOnly:false,MountPath:/active-query-tracker,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:<_> true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/ready,Port:{1 0 http-metrics},Host:,Scheme:HTTP,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:45,TimeoutSeconds:1,PeriodSeconds:10,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:*true,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,} start failed in pod <_> CreateContainerConfigError: secret "ruler-alertmanager-token" not found`, + `E0507 11:59:<_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"gcom-sync\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/kubernetes-dev/frontend-monitoring:6a8eb5a\\\"\"" <_> <_>`, + `E0507 11:59:<_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ErrImagePull: \"[rpc error: code =NotFound desc =failed to pull and unpack image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> failed to resolve reference \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> not found, failed to pull and unpack image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> failed to resolve reference \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> unexpected status from HEAD request to https:<_> 403 Forbidden]\"" <_> <_>`, + `E0507 11:59:<_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with ImagePullBackOff: \"Back-off pulling image \\\"us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> <_> <_>`, + `E0507 11:59:<_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"pdc\" with ErrImageNeverPull: \"Container image \\\"us.gcr.io/hosted-grafana/pdc:0.1.415\\\" is not present with pull policy of Never\"" <_> <_>`, + `E0507 11:59:<_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"ruler\" with CreateContainerConfigError: \"secret \\\"ruler-alertmanager-token\\\" not found\"" <_> <_>`, + `E0507 11:59:<_> <_> prober.go:104] "Probe errored" err="rpc error: code =NotFound desc =failed to exec in container: failed to load task: no running task found: task <_> not found: not found" probeType="Readiness" <_> <_> containerName="grafana"`, + `E0507 11:59:<_> <_> remote_image.go:180] "PullImage from image service failed" err="rpc error: code =NotFound desc =failed to pull and unpack image \"us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> failed to resolve reference \"us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> not found" image="us.gcr.io/hosted-grafana/hosted-grafana-pro:<_>`, + `E0507 11:59:<_> <_> remote_image.go:180] "PullImage from image service failed" err="rpc error: code =Unknown desc =failed to pull and unpack image \"us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> failed to resolve reference \"us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> unexpected status from HEAD request to https:<_> 403 Forbidden" image="us.gcr.io/hosted-grafana/hosted-grafana-pro:<_>`, + `E0507 11:59:<_> <_> remote_runtime.go:432] "ContainerStatus from runtime service failed" err="rpc error: code =NotFound desc =an error occurred when try to find container <_> not found" <_>`, + `E0507 11:59:<_> <_> remote_runtime.go:496] "ExecSync cmd from runtime service failed" err="rpc error: code =NotFound desc =failed to exec in container: failed to load task: no running task found: task <_> not found: not found" <_> cmd=["/bin/hgrun","check"]`, + `E0507 <_> 4733 pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"prometheus\" with CrashLoopBackOff:\"back-off 5m0s restarting failed container=prometheus pod=bryan-prometheus-0_bryan-prometheus(6dadfe71-eb19-4231-a96e-c64bb5499a1e)\"" pod="bryan-prometheus/bryan-prometheus-0" podUID="6dadfe71-eb19-4231-a96e-c64bb5499a1e"`, + `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"agent\" with CrashLoopBackOff:\"back-off 5m0s restarting failed container=agent pod=<_> pod=<_> podUID=<_>`, + `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"cortex-gw\" with CrashLoopBackOff:\"back-off 5m0s restarting failed container=cortex-gw pod=<_> pod=<_> podUID=<_>`, + `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"goldpinger\" with CrashLoopBackOff:\"back-off 5m0s restarting failed container=goldpinger pod=<_> pod=<_> podUID=<_>`, + `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"grafana\" with CrashLoopBackOff:\"back-off <_> restarting failed container=grafana pod=<_> pod=<_> podUID=<_>`, + `E0507 <_> <_> pod_workers.go:1300] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"support-agent\" with CrashLoopBackOff:\"back-off 5m0s restarting failed container=support-agent pod=<_> pod=<_> podUID=<_>`, + `E0507 <_> <_> prober.go:239] "Unable to write all bytes from execInContainer" err="short write" expectedBytes=<_> actualBytes=10240`, `I0507 11:59:29.320184 1537502 kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." pod="logs-endpoint-dev-005/kafka-controller-0" secret="" err="secret \"not-needed\" not found"`, `I0507 11:59:31.815514 2791 azure_credentials.go:220] image(us.gcr.io/hosted-grafana/hosted-grafana-pro) is not from ACR, return empty authentication`, `I0507 11:59:32.409568 581823 cache.go:40] re-using cached key and certificate`, @@ -206,40 +147,41 @@ func TestDrain_TrainExtractsPatterns(t *testing.T) { `I0507 11:59:34.841447 3224 operation_generator.go:888] UnmountVolume.TearDown succeeded for volume "kubernetes.io/secret/25cb986c-3d6c-4ed0-abf3-ee59ed6175f9-gcs-serviceaccount" (OuterVolumeSpecName: "gcs-serviceaccount") pod "25cb986c-3d6c-4ed0-abf3-ee59ed6175f9" (UID: "25cb986c-3d6c-4ed0-abf3-ee59ed6175f9"). InnerVolumeSpecName "gcs-serviceaccount". PluginName "kubernetes.io/secret", VolumeGidValue ""`, `I0507 11:59:34.854084 4727 kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." pod="integration/grafana-render-service-cbff479fc-cj9tp" secret="" err="secret \"us-gcr-io-hosted-grafana\" not found"`, `I0507 11:59:34.936025 3224 reconciler_common.go:300] "Volume detached for volume \"pdc-certs\" (UniqueName: \"kubernetes.io/secret/25cb986c-3d6c-4ed0-abf3-ee59ed6175f9-pdc-certs\") on node \"ip-10-60-2-58.us-east-2.compute.internal\" DevicePath \"\""`, - `I0507 11:59:37.133005 3782 prober.go:107] "Probe failed" probeType="Readiness" pod="loki-dev-014/loki-dev-014-rollout-operator-58fc68b876-2qhmp" podUID="e6504036-2514-4ecc-b78c-c47061f60c9f" containerName="rollout-operator" probeResult="failure" output="HTTP probe failed with statuscode: 500"`, - `I0507 11:59:37.915108 4726 prober.go:107] "Probe failed" probeType="Readiness" pod="agent-management-dev-002/agent-management-api-7ff7b9b9-k9nft" podUID="9893f9ac-f3e4-41fb-8da7-592061d2386c" containerName="agent-management-api" probeResult="failure" output="HTTP probe failed with statuscode: 400"`, + `I0507 11:59:37.133005 3782 prober.go:107] "Probe failed" probeType="Readiness" pod="loki-dev-014/loki-dev-014-rollout-operator-58fc68b876-2qhmp" podUID="e6504036-2514-4ecc-b78c-c47061f60c9f" containerName="rollout-operator" probeResult="failure" output="HTTP probe failed with statuscode:500"`, + `I0507 11:59:37.915108 4726 prober.go:107] "Probe failed" probeType="Readiness" pod="agent-management-dev-002/agent-management-api-7ff7b9b9-k9nft" podUID="9893f9ac-f3e4-41fb-8da7-592061d2386c" containerName="agent-management-api" probeResult="failure" output="HTTP probe failed with statuscode:400"`, `I0507 11:59:38.116658 2791 azure_credentials.go:220] image(us.gcr.io/hosted-grafana/hg-plugins) is not from ACR, return empty authentication`, `I0507 11:59:39.168633 2776 kubelet.go:2493] "SyncLoop (probe)" probe="readiness" status="" pod="hosted-grafana/dafdeveuwest2-grafana-7845d969b5-f8h5q"`, `I0507 11:59:39.560605 4739 kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." pod="logs-endpoint-dev-005/kafka-exporter-766c6757b5-bggf6" secret="" err="secret \"not-needed\" not found"`, - `I0507 <_> 2791 azure_credentials.go:220] image(us.gcr.io/hosted-grafana/hgrun) is not from ACR, return empty authentication`, - `I0507 <_> 3224 reconciler_common.go:300] "Volume detached for volume <_> (UniqueName: <_> on node \"ip-10-60-2-58.us-east-2.compute.internal\" DevicePath \"\""`, - `I0507 <_> 6247 prober.go:107] "Probe failed" probeType="Readiness" pod="grafana-agent/grafana-agent-helm-4" podUID="c36c5200-1cd6-4093-893c-c022f91af996" containerName="grafana-agent" probeResult="failure" output="Get \"http://10.0.99.125:3090/-/ready\": dial tcp 10.0.99.125:3090: connect: connection refused"`, - `I0507 <_> <_> <_> "Cleaned up orphaned pod volumes dir" <_> <_>`, - `I0507 <_> <_> <_> "SyncLoop (PLEG): event for pod" <_> <_>`, - `I0507 <_> <_> <_> "SyncLoop DELETE" source="api" <_>`, - `I0507 <_> <_> <_> "SyncLoop REMOVE" source="api" <_>`, - `I0507 <_> <_> generic.go:334] "Generic (PLEG): container finished" <_> <_> exitCode=1`, - `I0507 <_> <_> kubelet.go:2498] "SyncLoop (probe)" probe="liveness" status="unhealthy" <_>`, - `I0507 <_> <_> kubelet.go:2498] "SyncLoop (probe)" probe="readiness" status="ready" <_>`, - `I0507 <_> <_> kubelet_getters.go:187] "Pod status updated" <_> status="Running"`, - `I0507 <_> <_> kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." <_> secret="" err="secret \"dockerhub\" not found"`, - `I0507 <_> <_> kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." <_> secret="" err="secret \"gcr\" not found"`, - `I0507 <_> <_> pod_container_deletor.go:53] "DeleteContainer returned error" <_> err="failed to get container status <_> rpc error: code = NotFound desc = an error occurred when try to find container <_> not found"`, - `I0507 <_> <_> prober.go:107] "Probe failed" probeType="Readiness" <_> <_> containerName="grafana" probeResult="failure" output=<`, - `I0507 <_> <_> scope.go:117] "RemoveContainer" <_>`, - `I0507 <_> <_> cache.go:40] re-using cached key and certificate`, + `I0507 11:59:<_> 2791 azure_credentials.go:220] image(us.gcr.io/hosted-grafana/hgrun) is not from ACR, return empty authentication`, + `I0507 11:59:<_> 3224 reconciler_common.go:300] "Volume detached for volume <_> (UniqueName: <_> on node \"ip-10-60-2-58.us-east-2.compute.internal\" DevicePath \"\""`, + `I0507 11:59:<_> 6247 prober.go:107] "Probe failed" probeType="Readiness" pod="grafana-agent/grafana-agent-helm-4" podUID="c36c5200-1cd6-4093-893c-c022f91af996" containerName="grafana-agent" probeResult="failure" output="Get \"http://10.0.99.125:3090/-/ready\": dial tcp 10.0.99.125:3090: connect: connection refused"`, + `I0507 11:59:<_> <_> generic.go:334] "Generic (PLEG): container finished" <_> <_> exitCode=1`, + `I0507 11:59:<_> <_> kubelet.go:<_> "SyncLoop (PLEG): event for pod" <_> event={"ID":<_> "ContainerDied","Data":<_>`, + `I0507 11:59:<_> <_> kubelet.go:<_> "SyncLoop (PLEG): event for pod" <_> event={"ID":<_> "ContainerStarted","Data":<_>`, + `I0507 11:59:<_> <_> kubelet.go:<_> "SyncLoop DELETE" source="api" <_>`, + `I0507 11:59:<_> <_> kubelet.go:<_> "SyncLoop REMOVE" source="api" <_>`, + `I0507 11:59:<_> <_> kubelet_getters.go:187] "Pod status updated" <_> status="Running"`, + `I0507 11:59:<_> <_> kubelet_volumes.go:<_> "Cleaned up orphaned pod volumes dir" <_> <_>`, + `I0507 11:59:<_> <_> pod_container_deletor.go:53] "DeleteContainer returned error" containerID={"Type":"containerd","ID":<_> err="failed to get container status <_> rpc error: code =NotFound desc =an error occurred when try to find container <_> not found"`, + `I0507 11:59:<_> <_> scope.go:117] "RemoveContainer" <_>`, + `I0507 11:59:<_> <_> cache.go:40] re-using cached key and certificate`, + `I0507 <_> <_> kubelet.go:2498] "SyncLoop (probe)" probe="liveness" status="unhealthy" pod=<_>`, + `I0507 <_> <_> kubelet.go:2498] "SyncLoop (probe)" probe="readiness" status="ready" pod=<_>`, + `I0507 <_> <_> kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." pod=<_> secret="" err="secret \"dockerhub\" not found"`, + `I0507 <_> <_> kubelet_pods.go:906] "Unable to retrieve pull secret, the image pull may not succeed." pod=<_> secret="" err="secret \"gcr\" not found"`, + `I0507 <_> <_> prober.go:107] "Probe failed" probeType="Readiness" pod=<_> podUID=<_> containerName="grafana" probeResult="failure" output=<`, `IPv4: martian source <_> from <_> on dev eth0`, `PRC: Renewing lease on eth0.`, `RCV: Reply message on eth0 from fe80::e9:7eff:fedf:3d37.`, `Removed slice libcontainer container kubepods-burstable-pod25cb986c_3d6c_4ed0_abf3_ee59ed6175f9.slice.`, - `Started cri-containerd-95bf586cd79d43120ff44582d4dbd2476de61744411f8515b9b2c527a41fd5d9.scope.`, `Started libcontainer container <_>`, `XMT: Renew on eth0, interval 9700ms.`, `XMT: Solicit on eth0, interval <_>`, - `audit: type=1400 <_> apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" <_> comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined"`, + `audit:type=1400 <_> apparmor="DENIED" operation="ptrace" profile="cri-containerd.apparmor.d" pid=<_> comm="pidof" requested_mask="read" denied_mask="read" peer="unconfined"`, `kauditd_printk_skb: <_> callbacks suppressed`, `ll header: 00000000: 42 01 0a 80 00 <_> 42 01 0a 80 00 01 08 00`, `net_ratelimit: 2 callbacks suppressed`, + `time="2024-05-07T11:59:32.755926053Z" level=info msg="CreateContainer within sandbox \"81e019a0248a0300a328fd59f9939c3eaa1b98aa7f325a7f6e00592633275ef6\" for container &ContainerMetadata{Name:checkoutservice,Attempt:3417,}"`, `time="2024-05-07T11:59:34.519591759Z" level=info msg="StopContainer for \"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\" with timeout 30 (s)"`, `time="2024-05-07T11:59:34.520032214Z" level=info msg="Stop container \"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\" with signal terminated"`, `time="2024-05-07T11:59:34.591282703Z" level=info msg="StopContainer for \"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\" returns successfully"`, @@ -247,39 +189,77 @@ func TestDrain_TrainExtractsPatterns(t *testing.T) { `time="2024-05-07T11:59:34.592084495Z" level=info msg="Container to stop \"c91436db00920ec961b9d5d6b4859d80a912e862e34fb5c45d8a85684fe6a97e\" must be in running or unknown state, current state \"CONTAINER_EXITED\""`, `time="2024-05-07T11:59:34.706960850Z" level=info msg="TearDown network for sandbox \"c605ad2cdc74c6b5288f2532ad71cce81a28ef6965f97a89ff6609deb825553a\" successfully"`, `time="2024-05-07T11:59:34.707025668Z" level=info msg="StopPodSandbox for \"c605ad2cdc74c6b5288f2532ad71cce81a28ef6965f97a89ff6609deb825553a\" returns successfully"`, - `time="2024-05-07T11:59:38.484586527Z" level=error msg="Failed to delete exec process \"d9e0a1867ce73695ad859f2b0a76fe8f5053db8a5e49142d747e53a445729bd4\" for container \"6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408\"" error="ttrpc: closed: unknown"`, + `time="2024-05-07T11:59:36.177858616Z" level=info msg="CreateContainer within sandbox \"81e019a0248a0300a328fd59f9939c3eaa1b98aa7f325a7f6e00592633275ef6\" for &ContainerMetadata{Name:checkoutservice,Attempt:3417,} returns container id \"95bf586cd79d43120ff44582d4dbd2476de61744411f8515b9b2c527a41fd5d9\""`, + `time="2024-05-07T11:59:38.484586527Z" level=error msg="Failed to delete exec process \"d9e0a1867ce73695ad859f2b0a76fe8f5053db8a5e49142d747e53a445729bd4\" for container \"6ad3e55547f2192f865518e75009243418b177091c1c781236e2ac8f0324b408\"" error="ttrpc:closed:unknown"`, + `time="2024-05-07T11:59:43.941729092Z" level=info msg="CreateContainer within sandbox \"ee9dc07bca79ef7dffe2a6eb326e27236e9e97c35913c7aae16ee0a62632fc25\" for container &ContainerMetadata{Name:cortex-gw,Attempt:1660,}"`, + `time="2024-05-07T11:59:43.954289531Z" level=info msg="CreateContainer within sandbox \"ee9dc07bca79ef7dffe2a6eb326e27236e9e97c35913c7aae16ee0a62632fc25\" for &ContainerMetadata{Name:cortex-gw,Attempt:1660,} returns container id \"93fa5decd62691912f90c9b27526f5e00183239bfa4d3f4ea8578a7873b9c2b4\""`, + `time="2024-05-07T11:59:<_> level=error msg="ExecSync for <_> failed" error="rpc error: code =NotFound desc =failed to exec in container: failed to load task: no running task found: task <_> not found: not found"`, + `time="2024-05-07T11:59:<_> level=error msg="PullImage \"us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> failed" error="failed to pull and unpack image \"us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> failed to resolve reference \"us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> unexpected status from HEAD request to https:<_> 403 Forbidden"`, + `time="2024-05-07T11:59:<_> level=error msg="PullImage \"us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> failed" error="rpc error: code =NotFound desc =failed to pull and unpack image \"us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> failed to resolve reference \"us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> us.gcr.io/hosted-grafana/hosted-grafana-pro:<_> not found"`, + `time="2024-05-07T11:59:<_> level=info msg="CreateContainer within sandbox <_> for &ContainerMetadata{Name:grafana,Attempt:<_> returns container id <_>`, + `time="2024-05-07T11:59:<_> level=info msg="CreateContainer within sandbox <_> for &ContainerMetadata{Name:hgrun,Attempt:0,} returns container id <_>`, + `time="2024-05-07T11:59:<_> level=info msg="CreateContainer within sandbox <_> for container &ContainerMetadata{Name:grafana,Attempt:<_>`, + `time="2024-05-07T11:59:<_> level=info msg="CreateContainer within sandbox <_> for container &ContainerMetadata{Name:hgrun,Attempt:0,}"`, + `time="2024-05-07T11:59:<_> level=info msg="ImageCreate event name:<_> <_> labels:{key:\"io.cri-containerd.image\" value:\"managed\"}"`, + `time="2024-05-07T11:59:<_> level=info msg="ImageUpdate event name:<_> <_> labels:{key:\"io.cri-containerd.image\" value:\"managed\"}"`, + `time="2024-05-07T11:59:<_> level=info msg="PullImage \"us.gcr.io/hosted-grafana/hgrun:0.1.452\" returns image reference \"sha256:9fb1bce3e4a228f50768d21842cd7d7fafc1d586eaa0326c9d3c86d79a36868a\""`, + `time="2024-05-07T11:59:<_> level=info msg="PullImage \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-70397\" returns image reference \"sha256:0036b00b52fc547c944c1c820817d91fba6e20775cbf4e6c3e09ad2e682dbd73\""`, + `time="2024-05-07T11:59:<_> level=info msg="Pulled image \"us.gcr.io/hosted-grafana/hgrun:0.1.452\" with image id \"sha256:9fb1bce3e4a228f50768d21842cd7d7fafc1d586eaa0326c9d3c86d79a36868a\", repo tag \"us.gcr.io/hosted-grafana/hgrun:0.1.452\", repo digest \"us.gcr.io/hosted-grafana/hgrun@sha256:b492dbbbee9faf9dba63c9fd89e6f9e148239765454c6a54c4284a2828dec153\", size \"19109699\" in <_>`, + `time="2024-05-07T11:59:<_> level=info msg="Pulled image \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-70397\" with image id \"sha256:0036b00b52fc547c944c1c820817d91fba6e20775cbf4e6c3e09ad2e682dbd73\", repo tag \"us.gcr.io/hosted-grafana/hosted-grafana-pro:11.1.0-70397\", repo digest \"us.gcr.io/hosted-grafana/hosted-grafana-pro@sha256:0853965a142fb95648de3281a7c71de0d05fb51616bc32b523dc2f1da6ca06dc\", size \"173405048\" in <_>`, + `time=<_> level=error msg="ContainerStatus for <_> failed" error="rpc error:code = NotFound desc = an error occurred when try to find container <_> not found"`, + `time=<_> level=info msg="PullImage <_>`, + `time=<_> level=info msg="RemoveContainer for <_>`, + `time=<_> level=info msg="RemoveContainer for <_> returns successfully"`, + `time=<_> level=info msg="StartContainer for <_>`, + `time=<_> level=info msg="StartContainer for <_> returns successfully"`, + `time=<_> level=info msg="cleaning up dead shim" namespace=k8s.io`, + `time=<_> level=info msg="shim disconnected" id=<_> namespace=k8s.io`, + `time=<_> level=info msg="stop pulling image <_> active requests=0, bytes read=<_>`, + `time=<_> level=info msg="trying next host - response was http.StatusNotFound" host=us.gcr.io`, + `time=<_> level=warning msg="cleaning up after shim disconnected" id=<_> namespace=k8s.io`, }, }, { - name: "Patterns for kafka logs", drain: New(DefaultConfig(), nil), inputFile: "testdata/kafka.txt", patterns: []string{ - `[2024-05-07 10:55:40,559] INFO [LocalLog partition=ingest-7, dir=/bitnami/kafka/data] Deleting segment files LogSegment(baseOffset=179133378, size=16987985, lastModifiedTime=1715075760072, largestRecordTimestamp=Some(1715075760047)),LogSegment(baseOffset=179135832, size=16999459, lastModifiedTime=1715075765431, largestRecordTimestamp=Some(1715075765398)),LogSegment(baseOffset=179138321, size=16994485, lastModifiedTime=1715075770425, largestRecordTimestamp=Some(1715075770404)),LogSegment(baseOffset=179140761, size=16996810, lastModifiedTime=1715075775622, largestRecordTimestamp=Some(1715075775619)),LogSegment(baseOffset=179143198, size=16998520, lastModifiedTime=1715075780912, largestRecordTimestamp=Some(1715075780889)),LogSegment(baseOffset=179145674, size=16988474, lastModifiedTime=1715075786051, largestRecordTimestamp=Some(1715075786030)),LogSegment(baseOffset=179148084, size=16956099, lastModifiedTime=1715075791514, largestRecordTimestamp=Some(1715075791486)),LogSegment(baseOffset=179150568, size=16995476, lastModifiedTime=1715075796360, largestRecordTimestamp=Some(1715075796329)),LogSegment(baseOffset=179152727, size=16993313, lastModifiedTime=1715075800440, largestRecordTimestamp=Some(1715075800430)),LogSegment(baseOffset=179154861, size=16992142, lastModifiedTime=1715075805147, largestRecordTimestamp=Some(1715075805135)),LogSegment(baseOffset=179157056, size=16999919, lastModifiedTime=1715075810155, largestRecordTimestamp=Some(1715075810153)),LogSegment(baseOffset=179159230, size=16995021, lastModifiedTime=1715075815018, largestRecordTimestamp=Some(1715075815016)),LogSegment(baseOffset=179161550, size=16966526, lastModifiedTime=1715075819528, largestRecordTimestamp=Some(1715075819521)),LogSegment(baseOffset=179163962, size=16990848, lastModifiedTime=1715075825066, largestRecordTimestamp=Some(1715075825042)),LogSegment(baseOffset=179166414, size=16997833, lastModifiedTime=1715075830662, largestRecordTimestamp=Some(1715075830656)),LogSegment(baseOffset=179168915, size=16992619, lastModifiedTime=1715075835771, largestRecordTimestamp=Some(1715075835741)),LogSegment(baseOffset=179171302, size=16999091, lastModifiedTime=1715075841031, largestRecordTimestamp=Some(1715075841022)),LogSegment(baseOffset=179173853, size=16993953, lastModifiedTime=1715075846197, largestRecordTimestamp=Some(1715075846181)),LogSegment(baseOffset=179176191, size=16997479, lastModifiedTime=1715075853192, largestRecordTimestamp=Some(1715075853172)),LogSegment(baseOffset=179179037, size=16997174, lastModifiedTime=1715075858693, largestRecordTimestamp=Some(1715075858682)),LogSegment(baseOffset=179181478, size=16986004, lastModifiedTime=1715075863400, largestRecordTimestamp=Some(1715075863396)),LogSegment(baseOffset=179183786, size=16995316, lastModifiedTime=1715075866123, largestRecordTimestamp=Some(1715075866112)),LogSegment(baseOffset=179185434, size=16990492, lastModifiedTime=1715075870154, largestRecordTimestamp=Some(1715075870146)),LogSegment(baseOffset=179187398, size=16999541, lastModifiedTime=1715075874980, largestRecordTimestamp=Some(1715075874961)),LogSegment(baseOffset=179189664, size=16987383, lastModifiedTime=1715075879670, largestRecordTimestamp=Some(1715075879639)),LogSegment(baseOffset=179192076, size=16991701, lastModifiedTime=1715075885010, largestRecordTimestamp=Some(1715075884995)),LogSegment(baseOffset=179194546, size=16989109, lastModifiedTime=1715075890220, largestRecordTimestamp=Some(1715075890208)),LogSegment(baseOffset=179197009, size=16962782, lastModifiedTime=1715075895466, largestRecordTimestamp=Some(1715075895456)),LogSegment(baseOffset=179199373, size=16974715, lastModifiedTime=1715075900757, largestRecordTimestamp=Some(1715075900746)),LogSegment(baseOffset=179201897, size=16993973, lastModifiedTime=1715075905639, largestRecordTimestamp=Some(1715075905638)),LogSegment(baseOffset=179204346, size=16979828, lastModifiedTime=1715075910798, largestRecordTimestamp=Some(1715075910782)),LogSegment(baseOffset=179206836, size=16992092, lastModifiedTime=1715075915638, largestRecordTimestamp=Some(1715075915632)),LogSegment(baseOffset=179208986, size=16988849, lastModifiedTime=1715075920193, largestRecordTimestamp=Some(1715075920176)),LogSegment(baseOffset=179211133, size=16989206, lastModifiedTime=1715075924352, largestRecordTimestamp=Some(1715075924338)),LogSegment(baseOffset=179213268, size=16989737, lastModifiedTime=1715075929343, largestRecordTimestamp=Some(1715075929332)),LogSegment(baseOffset=179215514, size=16997903, lastModifiedTime=1715075934074, largestRecordTimestamp=Some(1715075934056)),LogSegment(baseOffset=179217793, size=16995100, lastModifiedTime=1715075938937, largestRecordTimestamp=Some(1715075938925)),LogSegment(baseOffset=179220122, size=16981574, lastModifiedTime=1715075944296, largestRecordTimestamp=Some(1715075944288)),LogSegment(baseOffset=179222600, size=16999794, lastModifiedTime=1715075949454, largestRecordTimestamp=Some(1715075949432)),LogSegment(baseOffset=179224988, size=16998870, lastModifiedTime=1715075954567, largestRecordTimestamp=Some(1715075954544)),LogSegment(baseOffset=179227402, size=16986053, lastModifiedTime=1715075959815, largestRecordTimestamp=Some(1715075959813)),LogSegment(baseOffset=179229948, size=16999937, lastModifiedTime=1715075964787, largestRecordTimestamp=Some(1715075964779)),LogSegment(baseOffset=179232368, size=16992995, lastModifiedTime=1715075970109, largestRecordTimestamp=Some(1715075970096)),LogSegment(baseOffset=179234885, size=16995271, lastModifiedTime=1715075975078, largestRecordTimestamp=Some(1715075975066)),LogSegment(baseOffset=179237038, size=16987833, lastModifiedTime=1715075979534, largestRecordTimestamp=Some(1715075979499)),LogSegment(baseOffset=179239147, size=16844618, lastModifiedTime=1715075984150, largestRecordTimestamp=Some(1715075984139)),LogSegment(baseOffset=179241334, size=16968482, lastModifiedTime=1715075988727, largestRecordTimestamp=Some(1715075988700)),LogSegment(baseOffset=179243472, size=16991395, lastModifiedTime=1715075993359, largestRecordTimestamp=Some(1715075993333)),LogSegment(baseOffset=179245756, size=16985926, lastModifiedTime=1715075998010, largestRecordTimestamp=Some(1715075998005)),LogSegment(baseOffset=179248096, size=16948574, lastModifiedTime=1715076003328, largestRecordTimestamp=Some(1715076003298)),LogSegment(baseOffset=179250530, size=16986047, lastModifiedTime=1715076008650, largestRecordTimestamp=Some(1715076008628)),LogSegment(baseOffset=179252915, size=16998875, lastModifiedTime=1715076013551, largestRecordTimestamp=Some(1715076013516)),LogSegment(baseOffset=179255312, size=16997990, lastModifiedTime=1715076018832, largestRecordTimestamp=Some(1715076018797)),LogSegment(baseOffset=179257861, size=16999525, lastModifiedTime=1715076023621, largestRecordTimestamp=Some(1715076023601)),LogSegment(baseOffset=179260226, size=16997755, lastModifiedTime=1715076028814, largestRecordTimestamp=Some(1715076028800)),LogSegment(baseOffset=179262715, size=16981492, lastModifiedTime=1715076034150, largestRecordTimestamp=Some(1715076034140)),LogSegment(baseOffset=179265040, size=16998332, lastModifiedTime=1715076038676, largestRecordTimestamp=Some(1715076038657)) (kafka.log.LocalLog$)`, `[2024-05-07 10:55:40,626] INFO [LocalLog partition=ingest-6, dir=/bitnami/kafka/data] Deleting segment files LogSegment(baseOffset=180391157, size=16991045, lastModifiedTime=1715075754780, largestRecordTimestamp=Some(1715075754774)),LogSegment(baseOffset=180393429, size=16997692, lastModifiedTime=1715075760206, largestRecordTimestamp=Some(1715075760186)),LogSegment(baseOffset=180395889, size=16998200, lastModifiedTime=1715075765542, largestRecordTimestamp=Some(1715075765526)),LogSegment(baseOffset=180398373, size=16977347, lastModifiedTime=1715075770515, largestRecordTimestamp=Some(1715075770504)) (kafka.log.LocalLog$)`, `[2024-05-07 10:55:40,638] INFO [LocalLog partition=ingest-6, dir=/bitnami/kafka/data] Deleting segment files LogSegment(baseOffset=180400817, size=16997594, lastModifiedTime=1715075775780, largestRecordTimestamp=Some(1715075775771)),LogSegment(baseOffset=180403261, size=16992344, lastModifiedTime=1715075781053, largestRecordTimestamp=Some(1715075781021)),LogSegment(baseOffset=180405723, size=16989895, lastModifiedTime=1715075786205, largestRecordTimestamp=Some(1715075786174)),LogSegment(baseOffset=180408118, size=16998698, lastModifiedTime=1715075791681, largestRecordTimestamp=Some(1715075791673)),LogSegment(baseOffset=180410608, size=16995676, lastModifiedTime=1715075796438, largestRecordTimestamp=Some(1715075796430)),LogSegment(baseOffset=180412733, size=16963278, lastModifiedTime=1715075800534, largestRecordTimestamp=Some(1715075800511)),LogSegment(baseOffset=180414883, size=16984328, lastModifiedTime=1715075805272, largestRecordTimestamp=Some(1715075805230)),LogSegment(baseOffset=180417063, size=16989109, lastModifiedTime=1715075810381, largestRecordTimestamp=Some(1715075810372)),LogSegment(baseOffset=180419267, size=16996871, lastModifiedTime=1715075815153, largestRecordTimestamp=Some(1715075815125)),LogSegment(baseOffset=180421560, size=16988558, lastModifiedTime=1715075819785, largestRecordTimestamp=Some(1715075819763)),LogSegment(baseOffset=180424008, size=16999292, lastModifiedTime=1715075825336, largestRecordTimestamp=Some(1715075825303)),LogSegment(baseOffset=180426459, size=16990595, lastModifiedTime=1715075830839, largestRecordTimestamp=Some(1715075830827)),LogSegment(baseOffset=180428944, size=16995859, lastModifiedTime=1715075835942, largestRecordTimestamp=Some(1715075835904)),LogSegment(baseOffset=180431327, size=16992294, lastModifiedTime=1715075841219, largestRecordTimestamp=Some(1715075841214)),LogSegment(baseOffset=180433867, size=16966736, lastModifiedTime=1715075846443, largestRecordTimestamp=Some(1715075846401)),LogSegment(baseOffset=180436204, size=16894731, lastModifiedTime=1715075853273, largestRecordTimestamp=Some(1715075853244)),LogSegment(baseOffset=180438984, size=16983529, lastModifiedTime=1715075858911, largestRecordTimestamp=Some(1715075858891)),LogSegment(baseOffset=180441466, size=16996933, lastModifiedTime=1715075863566, largestRecordTimestamp=Some(1715075863554)),LogSegment(baseOffset=180443778, size=16999841, lastModifiedTime=1715075866199, largestRecordTimestamp=Some(1715075866185)),LogSegment(baseOffset=180445367, size=16992471, lastModifiedTime=1715075870385, largestRecordTimestamp=Some(1715075870347)),LogSegment(baseOffset=180447366, size=16999996, lastModifiedTime=1715075875102, largestRecordTimestamp=Some(1715075875091)),LogSegment(baseOffset=180449601, size=16994426, lastModifiedTime=1715075879927, largestRecordTimestamp=Some(1715075879926)),LogSegment(baseOffset=180452079, size=16998020, lastModifiedTime=1715075885293, largestRecordTimestamp=Some(1715075885263)),LogSegment(baseOffset=180454546, size=16992231, lastModifiedTime=1715075890424, largestRecordTimestamp=Some(1715075890409)),LogSegment(baseOffset=180456986, size=16970315, lastModifiedTime=1715075895719, largestRecordTimestamp=Some(1715075895690)),LogSegment(baseOffset=180459366, size=16990785, lastModifiedTime=1715075900996, largestRecordTimestamp=Some(1715075900985)),LogSegment(baseOffset=180461885, size=16996655, lastModifiedTime=1715075905847, largestRecordTimestamp=Some(1715075905841)),LogSegment(baseOffset=180464299, size=16982181, lastModifiedTime=1715075911052, largestRecordTimestamp=Some(1715075911028)),LogSegment(baseOffset=180466821, size=16997630, lastModifiedTime=1715075915962, largestRecordTimestamp=Some(1715075915953)),LogSegment(baseOffset=180468968, size=16995723, lastModifiedTime=1715075920325, largestRecordTimestamp=Some(1715075920308)),LogSegment(baseOffset=180471046, size=16979316, lastModifiedTime=1715075924724, largestRecordTimestamp=Some(1715075924697)),LogSegment(baseOffset=180473259, size=16995238, lastModifiedTime=1715075929645, largestRecordTimestamp=Some(1715075929624)),LogSegment(baseOffset=180475486, size=16988461, lastModifiedTime=1715075934288, largestRecordTimestamp=Some(1715075934283)),LogSegment(baseOffset=180477735, size=16993767, lastModifiedTime=1715075939277, largestRecordTimestamp=Some(1715075939270)),LogSegment(baseOffset=180480095, size=16995409, lastModifiedTime=1715075944639, largestRecordTimestamp=Some(1715075944635)),LogSegment(baseOffset=180482560, size=16992784, lastModifiedTime=1715075949760, largestRecordTimestamp=Some(1715075949760)),LogSegment(baseOffset=180484967, size=16990838, lastModifiedTime=1715075954937, largestRecordTimestamp=Some(1715075954929)),LogSegment(baseOffset=180487377, size=16976794, lastModifiedTime=1715075960151, largestRecordTimestamp=Some(1715075960119)),LogSegment(baseOffset=180489919, size=16997379, lastModifiedTime=1715075965116, largestRecordTimestamp=Some(1715075965085)),LogSegment(baseOffset=180492304, size=16956613, lastModifiedTime=1715075970448, largestRecordTimestamp=Some(1715075970424)),LogSegment(baseOffset=180494832, size=16895640, lastModifiedTime=1715075975354, largestRecordTimestamp=Some(1715075975341)),LogSegment(baseOffset=180496930, size=16998328, lastModifiedTime=1715075979813, largestRecordTimestamp=Some(1715075979796)),LogSegment(baseOffset=180499079, size=16995699, lastModifiedTime=1715075984309, largestRecordTimestamp=Some(1715075984285)),LogSegment(baseOffset=180501183, size=16993785, lastModifiedTime=1715075989086, largestRecordTimestamp=Some(1715075989064)),LogSegment(baseOffset=180503431, size=16989600, lastModifiedTime=1715075993713, largestRecordTimestamp=Some(1715075993683)),LogSegment(baseOffset=180505674, size=16984790, lastModifiedTime=1715075998337, largestRecordTimestamp=Some(1715075998318)),LogSegment(baseOffset=180508022, size=16982630, lastModifiedTime=1715076003671, largestRecordTimestamp=Some(1715076003660)),LogSegment(baseOffset=180510439, size=16999488, lastModifiedTime=1715076009000, largestRecordTimestamp=Some(1715076008996)),LogSegment(baseOffset=180512848, size=16997845, lastModifiedTime=1715076014033, largestRecordTimestamp=Some(1715076014032)),LogSegment(baseOffset=180515281, size=16990661, lastModifiedTime=1715076019245, largestRecordTimestamp=Some(1715076019216)),LogSegment(baseOffset=180517815, size=16996244, lastModifiedTime=1715076023989, largestRecordTimestamp=Some(1715076023963)),LogSegment(baseOffset=180520112, size=16992012, lastModifiedTime=1715076029243, largestRecordTimestamp=Some(1715076029231)) (kafka.log.LocalLog$)`, - `[2024-05-07 10:55:40,713] INFO [LocalLog partition=ingest-3, dir=/bitnami/kafka/data] Deleting segment files LogSegment(baseOffset=182526165, size=16998661, lastModifiedTime=1715075758062, largestRecordTimestamp=Some(1715075758061)),LogSegment(baseOffset=182528560, size=16999718, lastModifiedTime=1715075763583, largestRecordTimestamp=Some(1715075763577)),LogSegment(baseOffset=182531056, size=16994792, lastModifiedTime=1715075768711, largestRecordTimestamp=Some(1715075768697)),LogSegment(baseOffset=182533514, size=16987578, lastModifiedTime=1715075773552, largestRecordTimestamp=Some(1715075773536)),LogSegment(baseOffset=182535953, size=16987705, lastModifiedTime=1715075779055, largestRecordTimestamp=Some(1715075779046)),LogSegment(baseOffset=182538482, size=16997466, lastModifiedTime=1715075784005, largestRecordTimestamp=Some(1715075784004)),LogSegment(baseOffset=182540856, size=16981250, lastModifiedTime=1715075789523, largestRecordTimestamp=Some(1715075789487)),LogSegment(baseOffset=182543386, size=16980484, lastModifiedTime=1715075794637, largestRecordTimestamp=Some(1715075794632)),LogSegment(baseOffset=182545622, size=16999738, lastModifiedTime=1715075799008, largestRecordTimestamp=Some(1715075799000)),LogSegment(baseOffset=182547827, size=16872695, lastModifiedTime=1715075803273, largestRecordTimestamp=Some(1715075803251)),LogSegment(baseOffset=182550001, size=16999890, lastModifiedTime=1715075808368, largestRecordTimestamp=Some(1715075808355)),LogSegment(baseOffset=182552113, size=16959982, lastModifiedTime=1715075813294, largestRecordTimestamp=Some(1715075813293)),LogSegment(baseOffset=182554415, size=16988073, lastModifiedTime=1715075817816, largestRecordTimestamp=Some(1715075817783)),LogSegment(baseOffset=182556814, size=16974731, lastModifiedTime=1715075823018, largestRecordTimestamp=Some(1715075823016)),LogSegment(baseOffset=182559282, size=16996090, lastModifiedTime=1715075828672, largestRecordTimestamp=Some(1715075828632)),LogSegment(baseOffset=182561708, size=16999327, lastModifiedTime=1715075833742, largestRecordTimestamp=Some(1715075833709)),LogSegment(baseOffset=182564173, size=16992947, lastModifiedTime=1715075839121, largestRecordTimestamp=Some(1715075839114)),LogSegment(baseOffset=182566740, size=16982572, lastModifiedTime=1715075844268, largestRecordTimestamp=Some(1715075844254)),LogSegment(baseOffset=182569086, size=16994786, lastModifiedTime=1715075850659, largestRecordTimestamp=Some(1715075850642)),LogSegment(baseOffset=182571815, size=16998391, lastModifiedTime=1715075856704, largestRecordTimestamp=Some(1715075856684)),LogSegment(baseOffset=182574372, size=16994403, lastModifiedTime=1715075861956, largestRecordTimestamp=Some(1715075861922)),LogSegment(baseOffset=182576828, size=16984546, lastModifiedTime=1715075865194, largestRecordTimestamp=Some(1715075865180)),LogSegment(baseOffset=182578716, size=16987846, lastModifiedTime=1715075868470, largestRecordTimestamp=Some(1715075868460)),LogSegment(baseOffset=182580437, size=16958237, lastModifiedTime=1715075873168, largestRecordTimestamp=Some(1715075873151)),LogSegment(baseOffset=182582637, size=16999432, lastModifiedTime=1715075877858, largestRecordTimestamp=Some(1715075877850)),LogSegment(baseOffset=182585006, size=16938567, lastModifiedTime=1715075882952, largestRecordTimestamp=Some(1715075882938)),LogSegment(baseOffset=182587493, size=16998214, lastModifiedTime=1715075888306, largestRecordTimestamp=Some(1715075888285)),LogSegment(baseOffset=182589965, size=16996264, lastModifiedTime=1715075893370, largestRecordTimestamp=Some(1715075893365)),LogSegment(baseOffset=182592327, size=16991650, lastModifiedTime=1715075898806, largestRecordTimestamp=Some(1715075898802)),LogSegment(baseOffset=182594863, size=16998234, lastModifiedTime=1715075903737, largestRecordTimestamp=Some(1715075903733)),LogSegment(baseOffset=182597289, size=16996241, lastModifiedTime=1715075908805, largestRecordTimestamp=Some(1715075908797)),LogSegment(baseOffset=182599811, size=16993657, lastModifiedTime=1715075913918, largestRecordTimestamp=Some(1715075913915)),LogSegment(baseOffset=182602171, size=16993112, lastModifiedTime=1715075918570, largestRecordTimestamp=Some(1715075918570)),LogSegment(baseOffset=182604245, size=16959963, lastModifiedTime=1715075922720, largestRecordTimestamp=Some(1715075922714)),LogSegment(baseOffset=182606451, size=16998518, lastModifiedTime=1715075927490, largestRecordTimestamp=Some(1715075927484)),LogSegment(baseOffset=182608616, size=16999103, lastModifiedTime=1715075932207, largestRecordTimestamp=Some(1715075932188)),LogSegment(baseOffset=182610888, size=16999389, lastModifiedTime=1715075937118, largestRecordTimestamp=Some(1715075937103)),LogSegment(baseOffset=182613221, size=16982597, lastModifiedTime=1715075942170, largestRecordTimestamp=Some(1715075942153)),LogSegment(baseOffset=182615634, size=16986904, lastModifiedTime=1715075947544, largestRecordTimestamp=Some(1715075947541)),LogSegment(baseOffset=182618074, size=16998820, lastModifiedTime=1715075952370, largestRecordTimestamp=Some(1715075952351)),LogSegment(baseOffset=182620446, size=16985066, lastModifiedTime=1715075957884, largestRecordTimestamp=Some(1715075957865)),LogSegment(baseOffset=182623007, size=16998235, lastModifiedTime=1715075963030, largestRecordTimestamp=Some(1715075963008)),LogSegment(baseOffset=182625520, size=16987568, lastModifiedTime=1715075967944, largestRecordTimestamp=Some(1715075967934)),LogSegment(baseOffset=182627921, size=16997118, lastModifiedTime=1715075973216, largestRecordTimestamp=Some(1715075973204)),LogSegment(baseOffset=182630290, size=16978465, lastModifiedTime=1715075978064, largestRecordTimestamp=Some(1715075978053)),LogSegment(baseOffset=182632463, size=16901644, lastModifiedTime=1715075982228, largestRecordTimestamp=Some(1715075982211)),LogSegment(baseOffset=182634546, size=16992477, lastModifiedTime=1715075986935, largestRecordTimestamp=Some(1715075986914)),LogSegment(baseOffset=182636738, size=16951087, lastModifiedTime=1715075991658, largestRecordTimestamp=Some(1715075991636)),LogSegment(baseOffset=182639001, size=16994471, lastModifiedTime=1715075996281, largestRecordTimestamp=Some(1715075996266)),LogSegment(baseOffset=182641298, size=16995754, lastModifiedTime=1715076001319, largestRecordTimestamp=Some(1715076001269)),LogSegment(baseOffset=182643712, size=16992752, lastModifiedTime=1715076006604, largestRecordTimestamp=Some(1715076006583)),LogSegment(baseOffset=182646095, size=16992944, lastModifiedTime=1715076011511, largestRecordTimestamp=Some(1715076011470)),LogSegment(baseOffset=182648504, size=16998993, lastModifiedTime=1715076016908, largestRecordTimestamp=Some(1715076016908)),LogSegment(baseOffset=182651018, size=16996765, lastModifiedTime=1715076021971, largestRecordTimestamp=Some(1715076021968)),LogSegment(baseOffset=182653526, size=16995808, lastModifiedTime=1715076026767, largestRecordTimestamp=Some(1715076026752)),LogSegment(baseOffset=182655860, size=16993535, lastModifiedTime=1715076032181, largestRecordTimestamp=Some(1715076032131)),LogSegment(baseOffset=182658341, size=16971926, lastModifiedTime=1715076037067, largestRecordTimestamp=Some(1715076037053)) (kafka.log.LocalLog$)`, `[2024-05-07 10:55:53,038] INFO [LocalLog partition=mimir-dev-09-aggregations-offsets-1, dir=/bitnami/kafka/data] Deleting segment files LogSegment(baseOffset=447957, size=948, lastModifiedTime=1715059232052, largestRecordTimestamp=Some(1715059232002)),LogSegment(baseOffset=447969, size=948, lastModifiedTime=1715059424352, largestRecordTimestamp=Some(1715059424301)) (kafka.log.LocalLog$)`, - `[2024-05-07 <_> INFO Deleted log <_> (kafka.log.LogSegment)`, - `[2024-05-07 <_> INFO Deleted offset index <_> (kafka.log.LogSegment)`, - `[2024-05-07 <_> INFO Deleted producer state snapshot <_> (kafka.log.SnapshotFile)`, - `[2024-05-07 <_> INFO Deleted time index <_> (kafka.log.LogSegment)`, - `[2024-05-07 <_> INFO [LocalLog <_> dir=/bitnami/kafka/data] Rolled new log segment at offset <_> in <_> ms. (kafka.log.LocalLog)`, - `[2024-05-07 <_> INFO [LocalLog partition=mimir-dev-09-aggregations-offsets-0, dir=/bitnami/kafka/data] Deleting segment files <_> size=948, <_> <_> (kafka.log.LocalLog$)`, - `[2024-05-07 <_> INFO [ProducerStateManager <_> Wrote producer snapshot at offset <_> with 0 producer ids in <_> ms. (kafka.log.ProducerStateManager)`, - `[2024-05-07 <_> INFO [UnifiedLog <_> dir=/bitnami/kafka/data] Deleting segment <_> <_> <_> <_> due to retention size <_> breach. Log size after deletion will be <_> (kafka.log.UnifiedLog)`, - `[2024-05-07 <_> INFO [UnifiedLog <_> dir=/bitnami/kafka/data] Deleting segments due to log start offset <_> breach: <_> <_> <_> <_> (kafka.log.UnifiedLog)`, - `[2024-05-07 <_> INFO [UnifiedLog <_> dir=/bitnami/kafka/data] Deleting segments due to log start offset <_> breach: <_> size=948, <_> <_> size=948, <_> <_> (kafka.log.UnifiedLog)`, - `[2024-05-07 <_> INFO [UnifiedLog <_> dir=/bitnami/kafka/data] Incremented log start offset to <_> due to leader offset increment (kafka.log.UnifiedLog)`, - `[2024-05-07 <_> INFO [UnifiedLog <_> dir=/bitnami/kafka/data] Incremented log start offset to <_> due to segment deletion (kafka.log.UnifiedLog)`, + `[2024-05-07 10:55:<_> INFO Deleted log <_> (kafka.log.LogSegment)`, + `[2024-05-07 10:55:<_> INFO Deleted offset index <_> (kafka.log.LogSegment)`, + `[2024-05-07 10:55:<_> INFO Deleted producer state snapshot <_> (kafka.log.SnapshotFile)`, + `[2024-05-07 10:55:<_> INFO Deleted time index <_> (kafka.log.LogSegment)`, + `[2024-05-07 10:55:<_> INFO [ProducerStateManager <_> Wrote producer snapshot at offset <_> with 0 producer ids in <_> ms. (kafka.log.ProducerStateManager)`, + `[2024-05-07 <_> INFO [LocalLog partition=<_> dir=/bitnami/kafka/data] Deleting segment files LogSegment(baseOffset=<_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> (kafka.log.LocalLog$)`, + `[2024-05-07 <_> INFO [LocalLog partition=<_> dir=/bitnami/kafka/data] Rolled new log segment at offset <_> in <_> ms. (kafka.log.LocalLog)`, + `[2024-05-07 <_> INFO [LocalLog partition=mimir-dev-09-aggregations-offsets-0, dir=/bitnami/kafka/data] Deleting segment files LogSegment(baseOffset=<_> size=948, lastModifiedTime=<_> largestRecordTimestamp=<_> (kafka.log.LocalLog$)`, + `[2024-05-07 <_> INFO [UnifiedLog partition=<_> dir=/bitnami/kafka/data] Deleting segment LogSegment(baseOffset=<_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> due to retention size <_> breach. Log size after deletion will be <_> (kafka.log.UnifiedLog)`, + `[2024-05-07 <_> INFO [UnifiedLog partition=<_> dir=/bitnami/kafka/data] Deleting segments due to log start offset <_> breach:LogSegment(baseOffset=<_> size=948, lastModifiedTime=<_> largestRecordTimestamp=<_> <_> size=948, lastModifiedTime=<_> largestRecordTimestamp=<_> (kafka.log.UnifiedLog)`, + `[2024-05-07 <_> INFO [UnifiedLog partition=<_> dir=/bitnami/kafka/data] Deleting segments due to log start offset <_> breach:LogSegment(baseOffset=<_> size=<_> lastModifiedTime=<_> largestRecordTimestamp=<_> (kafka.log.UnifiedLog)`, + `[2024-05-07 <_> INFO [UnifiedLog partition=<_> dir=/bitnami/kafka/data] Incremented log start offset to <_> due to leader offset increment (kafka.log.UnifiedLog)`, + `[2024-05-07 <_> INFO [UnifiedLog partition=<_> dir=/bitnami/kafka/data] Incremented log start offset to <_> due to segment deletion (kafka.log.UnifiedLog)`, }, }, { - name: "Patterns for kubernetes logs", drain: New(DefaultConfig(), nil), inputFile: "testdata/kubernetes.txt", patterns: []string{ `I0507 12:02:27.947830 1 nodeutilization.go:274] "Evicting pods based on priority, if they have same priority, they'll be evicted based on QoS tiers"`, + `I0507 12:02:<_> 1 defaultevictor.go:163] "pod does not fit on any other node because of nodeSelector(s), Taint(s), or nodes marked as unschedulable" <_>`, + `I0507 12:02:<_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod is a DaemonSet pod, pod has local storage and descheduler is not configured with evictLocalStoragePods]"`, + `I0507 12:02:<_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="pod has local storage and descheduler is not configured with evictLocalStoragePods"`, + `I0507 12:02:<_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="pod is a DaemonSet pod"`, + `I0507 12:02:<_> 1 node.go:157] "Pod does not fit on any other node" pod:<_> node:<_> error:="[pod node selector does not match the node label, insufficient <_>`, + `I0507 12:02:<_> 1 node.go:157] "Pod does not fit on any other node" pod:<_> node:<_> error:="[pod node selector does not match the node label, insufficient <_> insufficient <_>`, + `I0507 12:02:<_> 1 node.go:157] "Pod does not fit on any other node" pod:<_> node:<_> error:="[pod node selector does not match the node label, insufficient <_> insufficient <_> insufficient pods]"`, + `I0507 12:02:<_> 1 node.go:157] "Pod does not fit on any other node" pod:<_> node:<_> error:="[pod node selector does not match the node label, pod does not tolerate taints on the node, insufficient <_>`, + `I0507 12:02:<_> 1 node.go:157] "Pod does not fit on any other node" pod:<_> node:<_> error:="[pod node selector does not match the node label, pod does not tolerate taints on the node, insufficient <_> insufficient <_>`, + `I0507 12:02:<_> 1 node.go:157] "Pod does not fit on any other node" pod:<_> node:<_> error:="[pod node selector does not match the node label, pod does not tolerate taints on the node]"`, + `I0507 12:02:<_> 1 node.go:157] "Pod does not fit on any other node" pod:<_> node:<_> error:="insufficient cpu"`, + `I0507 12:02:<_> 1 node.go:157] "Pod does not fit on any other node" pod:="loki-dev-005/querier-burst-6b5f6db455-5zvkm" node:<_> error:="[insufficient <_> insufficient <_>`, + `I0507 12:02:<_> 1 node.go:157] "Pod does not fit on any other node" pod:="loki-dev-005/querier-burst-6b5f6db455-5zvkm" node:<_> error:="pod node selector does not match the node label"`, + `I0507 12:02:<_> 1 node.go:339] "no Pod antiaffinity rule found" <_>`, `I0507 12:04:17.595169 1 descheduler.go:155] Building a pod evictor`, `I0507 12:04:17.596431 1 nodeutilization.go:204] "Node is underutilized" node="gke-dev-eu-west-3-main-n2s8-1-1dd39c-d1c92061-4z2l" usage={"cpu":"984m","memory":"611Mi","pods":"16"} usagePercentage={"cpu":12.44,"memory":2.15,"pods":25}`, `I0507 12:04:17.596484 1 highnodeutilization.go:107] "Criteria for a node below target utilization" CPU=50 Mem=50 Pods=100`, @@ -287,136 +267,119 @@ func TestDrain_TrainExtractsPatterns(t *testing.T) { `I0507 12:04:17.596528 1 nodeutilization.go:260] "Total capacity to be moved" CPU=5060 Mem=112216292800 Pods=163`, `I0507 12:04:17.596651 1 defaultevictor.go:202] "Pod fails the following checks" pod="kube-system/metrics-server-v0.6.3-68f5b7c4d5-t5mz8" checks="[pod has system critical priority, pod has higher priority than specified priority class threshold, pod has local storage and descheduler is not configured with evictLocalStoragePods]"`, `I0507 12:04:17.596803 1 defaultevictor.go:202] "Pod fails the following checks" pod="gadget/gadget-zjjts" checks="[pod is a DaemonSet pod, pod has local storage and descheduler is not configured with evictLocalStoragePods]"`, - `I0507 <_> 1 <_> "Evicting pods from node" <_> <_>`, - `I0507 <_> 1 <_> "No removable pods on node, try next node" <_>`, - `I0507 <_> 1 <_> "Number of evicted pods" <_>`, - `I0507 <_> 1 <_> "Pods on node" <_> <_> <_> <_>`, - `I0507 <_> 1 <_> "Total number of pods evicted" extension point="Balance" <_>`, - `I0507 <_> 1 <_> <_> Watch close - <_> total <_> items received`, - `I0507 <_> 1 defaultevictor.go:163] "pod does not fit on any other node because of nodeSelector(s), Taint(s), or nodes marked as unschedulable" <_>`, - `I0507 <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod has system critical priority, pod has higher priority than specified priority class threshold]"`, - `I0507 <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod is a DaemonSet pod, pod has higher priority than specified priority class threshold, pod has local storage and descheduler is not configured with evictLocalStoragePods]"`, - `I0507 <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod is a DaemonSet pod, pod has higher priority than specified priority class threshold]"`, - `I0507 <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod is a DaemonSet pod, pod has local storage and descheduler is not configured with evictLocalStoragePods]"`, - `I0507 <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod is a DaemonSet pod, pod has system critical priority, pod has higher priority than specified priority class threshold, pod has local storage and descheduler is not configured with evictLocalStoragePods]"`, - `I0507 <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod is a DaemonSet pod, pod has system critical priority, pod has higher priority than specified priority class threshold]"`, - `I0507 <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod is a mirror pod, pod is a static pod, pod has system critical priority, pod has higher priority than specified priority class threshold, pod has local storage and descheduler is not configured with evictLocalStoragePods]"`, - `I0507 <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="pod has local storage and descheduler is not configured with evictLocalStoragePods"`, - `I0507 <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="pod is a DaemonSet pod"`, - `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" <_> <_> error:="[pod node selector does not match the node label, insufficient <_>`, - `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" <_> <_> error:="[pod node selector does not match the node label, insufficient <_> insufficient <_>`, - `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" <_> <_> error:="[pod node selector does not match the node label, insufficient <_> insufficient <_> insufficient pods]"`, - `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" <_> <_> error:="[pod node selector does not match the node label, pod does not tolerate taints on the node, insufficient <_>`, - `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" <_> <_> error:="[pod node selector does not match the node label, pod does not tolerate taints on the node, insufficient <_> insufficient <_>`, - `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" <_> <_> error:="[pod node selector does not match the node label, pod does not tolerate taints on the node]"`, - `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" <_> <_> error:="insufficient cpu"`, - `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" pod:="loki-dev-005/querier-burst-6b5f6db455-5zvkm" <_> error:="[insufficient cpu, insufficient memory]"`, - `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" pod:="loki-dev-005/querier-burst-6b5f6db455-5zvkm" <_> error:="[insufficient memory, insufficient cpu]"`, - `I0507 <_> 1 node.go:157] "Pod does not fit on any other node" pod:="loki-dev-005/querier-burst-6b5f6db455-5zvkm" <_> error:="pod node selector does not match the node label"`, - `I0507 <_> 1 node.go:339] "no Pod antiaffinity rule found" <_>`, - `I0507 <_> 1 nodeutilization.go:207] "Node is overutilized" <_> <_> <_>`, + `I0507 12:04:<_> 1 nodeutilization.go:207] "Node is overutilized" <_> usage={"cpu":<_> <_> <_> usagePercentage={"cpu":<_> <_> <_>`, + `I0507 12:<_> <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod has system critical priority, pod has higher priority than specified priority class threshold]"`, + `I0507 12:<_> <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod is a DaemonSet pod, pod has higher priority than specified priority class threshold, pod has local storage and descheduler is not configured with evictLocalStoragePods]"`, + `I0507 12:<_> <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod is a DaemonSet pod, pod has higher priority than specified priority class threshold]"`, + `I0507 12:<_> <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod is a DaemonSet pod, pod has system critical priority, pod has higher priority than specified priority class threshold, pod has local storage and descheduler is not configured with evictLocalStoragePods]"`, + `I0507 12:<_> <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod is a DaemonSet pod, pod has system critical priority, pod has higher priority than specified priority class threshold]"`, + `I0507 12:<_> <_> 1 defaultevictor.go:202] "Pod fails the following checks" <_> checks="[pod is a mirror pod, pod is a static pod, pod has system critical priority, pod has higher priority than specified priority class threshold, pod has local storage and descheduler is not configured with evictLocalStoragePods]"`, + `I0507 12:<_> <_> 1 descheduler.go:<_> "Number of evicted pods" <_>`, + `I0507 12:<_> <_> 1 nodeutilization.go:<_> "Evicting pods from node" <_> usage={"cpu":<_> <_> <_>`, + `I0507 12:<_> <_> 1 nodeutilization.go:<_> "No removable pods on node, try next node" <_>`, + `I0507 12:<_> <_> 1 profile.go:<_> "Total number of pods evicted" extension point="Balance" <_>`, + `I0507 12:<_> <_> 1 reflector.go:<_> k8s.io/client-go/informers/factory.go:<_> Watch close - <_> total <_> items received`, + `I0507 <_> 1 <_> "Pods on node" node=<_> allPods=<_> nonRemovablePods=<_> removablePods=<_>`, }, }, { - name: "Patterns for vault logs", drain: New(DefaultConfig(), nil), inputFile: "testdata/vault.txt", patterns: []string{ - "<_> [INFO] expiration: revoked lease: <_>", + `2024-05-07T10:<_> <_> [INFO] expiration: revoked lease: <_>`, }, }, { - name: "Patterns for calico logs", drain: New(DefaultConfig(), nil), inputFile: "testdata/calico.txt", patterns: []string{ `2024-05-08 15:23:56.403 [DEBUG][615489] felix/table.go 699: Finished loading iptables state ipVersion=0x4 table="filter"`, `2024-05-08 15:23:56.614 [DEBUG][76] felix/int_dataplane.go 1777: Refreshing routes`, `2024-05-08 15:23:56.615 [DEBUG][76] felix/route_rule.go 179: Queueing a resync of routing rules. ipVersion=4`, - `2024-05-08 15:23:56.615 [DEBUG][76] felix/route_table.go 480: Queueing a resync of routing table. ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, - `2024-05-08 15:23:56.615 [DEBUG][76] felix/route_table.go 480: Queueing a resync of routing table. ifaceRegex="^wireguard.cali$" ipVersion=0x4 tableIndex=1`, + `2024-05-08 15:23:56.615 [DEBUG][76] felix/route_table.go 480:Queueing a resync of routing table. ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, + `2024-05-08 15:23:56.615 [DEBUG][76] felix/route_table.go 480:Queueing a resync of routing table. ifaceRegex="^wireguard.cali$" ipVersion=0x4 tableIndex=1`, `2024-05-08 15:23:56.615 [DEBUG][76] felix/route_table.go 533: Check interfaces matching regex`, `2024-05-08 15:23:56.615 [DEBUG][76] felix/wireguard.go 605: Queueing a resync of wireguard configuration ipVersion=0x4`, `2024-05-08 15:23:56.615 [DEBUG][76] felix/wireguard.go 654: Wireguard is not in-sync - verifying wireguard configuration is removed ipVersion=0x4`, `2024-05-08 15:23:56.617 [DEBUG][76] felix/wireguard.go 1503: Wireguard is disabled and does not exist ifaceName="wireguard.cali" ipVersion=0x4`, `2024-05-08 15:23:56.619 [DEBUG][76] felix/route_table.go 584: Flag no OIF for full re-sync`, - `2024-05-08 15:23:56.619 [DEBUG][76] felix/route_table.go 614: Synchronised routes on interface ifaceName="*NoOIF*" ifaceRegex="^wireguard.cali$" ipVersion=0x4 tableIndex=1`, - `2024-05-08 15:23:56.619 [DEBUG][76] felix/route_table.go 661: Syncing interface routes ifaceName="*NoOIF*" ifaceRegex="^wireguard.cali$" ipVersion=0x4 tableIndex=1`, - `2024-05-08 15:23:56.619 [DEBUG][76] felix/route_table.go 686: Reconcile against kernel programming ifaceName="*NoOIF*" ifaceRegex="^wireguard.cali$" ipVersion=0x4 tableIndex=1`, - `2024-05-08 15:23:57.942 [WARNING][56] felix/table.go 654: Detected out-of-sync inserts, marking for resync actualRuleIDs=[]string{"", "", "", "", "", "", "", "", "", "", "", "", "tVnHkvAo15HuiPy0", "", ""} chainName="OUTPUT" expectedRuleIDs=[]string{"tVnHkvAo15HuiPy0", "", "", "", "", "", "", "", "", "", "", "", "", "", ""} ipVersion=0x4 table="raw"`, - `2024-05-08 15:23:57.942 [WARNING][56] felix/table.go 654: Detected out-of-sync inserts, marking for resync actualRuleIDs=[]string{"", "", "", "", "6gwbT8clXdHdC1b1"} chainName="PREROUTING" expectedRuleIDs=[]string{"6gwbT8clXdHdC1b1", "", "", "", ""} ipVersion=0x4 table="raw"`, - `2024-05-08 15:23:57.969 [WARNING][56] felix/table.go 654: Detected out-of-sync inserts, marking for resync actualRuleIDs=[]string{"", "", "", "", "Cz_u1IQiXIMmKD4c", "", "", "", "", "", "", "", "", "", "", "", ""} chainName="INPUT" expectedRuleIDs=[]string{"Cz_u1IQiXIMmKD4c", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""} ipVersion=0x4 table="filter"`, - `2024-05-08 15:23:57.969 [WARNING][56] felix/table.go 654: Detected out-of-sync inserts, marking for resync actualRuleIDs=[]string{"", "", "", "", "tVnHkvAo15HuiPy0", "", "", "", "", ""} chainName="OUTPUT" expectedRuleIDs=[]string{"tVnHkvAo15HuiPy0", "", "", "", "", "", "", "", "", ""} ipVersion=0x4 table="filter"`, + `2024-05-08 15:23:56.619 [DEBUG][76] felix/route_table.go 614:Synchronised routes on interface ifaceName="*NoOIF*" ifaceRegex="^wireguard.cali$" ipVersion=0x4 tableIndex=1`, + `2024-05-08 15:23:56.619 [DEBUG][76] felix/route_table.go 661:Syncing interface routes ifaceName="*NoOIF*" ifaceRegex="^wireguard.cali$" ipVersion=0x4 tableIndex=1`, + `2024-05-08 15:23:56.619 [DEBUG][76] felix/route_table.go 686:Reconcile against kernel programming ifaceName="*NoOIF*" ifaceRegex="^wireguard.cali$" ipVersion=0x4 tableIndex=1`, + `2024-05-08 15:23:57.942 [WARNING][56] felix/table.go 654:Detected out-of-sync inserts, marking for resync actualRuleIDs=[]string{"", "", "", "", "", "", "", "", "", "", "", "", "tVnHkvAo15HuiPy0", "", ""} chainName="OUTPUT" expectedRuleIDs=[]string{"tVnHkvAo15HuiPy0", "", "", "", "", "", "", "", "", "", "", "", "", "", ""} ipVersion=0x4 table="raw"`, + `2024-05-08 15:23:57.942 [WARNING][56] felix/table.go 654:Detected out-of-sync inserts, marking for resync actualRuleIDs=[]string{"", "", "", "", "6gwbT8clXdHdC1b1"} chainName="PREROUTING" expectedRuleIDs=[]string{"6gwbT8clXdHdC1b1", "", "", "", ""} ipVersion=0x4 table="raw"`, + `2024-05-08 15:23:57.969 [WARNING][56] felix/table.go 654:Detected out-of-sync inserts, marking for resync actualRuleIDs=[]string{"", "", "", "", "Cz_u1IQiXIMmKD4c", "", "", "", "", "", "", "", "", "", "", "", ""} chainName="INPUT" expectedRuleIDs=[]string{"Cz_u1IQiXIMmKD4c", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""} ipVersion=0x4 table="filter"`, + `2024-05-08 15:23:57.969 [WARNING][56] felix/table.go 654:Detected out-of-sync inserts, marking for resync actualRuleIDs=[]string{"", "", "", "", "tVnHkvAo15HuiPy0", "", "", "", "", ""} chainName="OUTPUT" expectedRuleIDs=[]string{"tVnHkvAo15HuiPy0", "", "", "", "", "", "", "", "", ""} ipVersion=0x4 table="filter"`, `2024-05-08 15:23:58.566 [DEBUG][3576126] felix/int_dataplane.go 957: Examining link for MTU calculation mtu=1500 name="eth0"`, `2024-05-08 15:23:58.680 [DEBUG][216945] felix/int_dataplane.go 1785: Reschedule kick received`, `2024-05-08 15:23:58.681 [DEBUG][216945] felix/feature_detect.go 112: Refreshing detected iptables features`, - `2024-05-08 15:23:58.681 [DEBUG][216945] felix/table.go 944: Invalidating dataplane cache ipVersion=0x4 reason="refresh timer" table="nat"`, + `2024-05-08 15:23:58.681 [DEBUG][216945] felix/table.go 944:Invalidating dataplane cache ipVersion=0x4 reason="refresh timer" table="nat"`, `2024-05-08 15:23:58.684 [DEBUG][216945] felix/feature_detect.go 242: Ran iptables --version rawVersion="iptables v1.8.4 (legacy)\n"`, `2024-05-08 15:23:58.684 [DEBUG][216945] felix/feature_detect.go 255: Parsed iptables version version=1.8.4`, `2024-05-08 15:23:58.684 [DEBUG][216945] felix/table.go 604: Loading current iptables state and checking it is correct. ipVersion=0x4 table="nat"`, `2024-05-08 15:23:58.684 [DEBUG][216945] felix/versionparse.go 110: Raw kernel version rawVersion="Linux version 5.15.0-1057-azure (buildd@lcy02-amd64-033) (gcc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0, GNU ld (GNU Binutils for Ubuntu) 2.38) #65-Ubuntu SMP Fri Feb 9 18:39:24 UTC 2024\n"`, `2024-05-08 15:23:58.684 [DEBUG][216945] felix/versionparse.go 118: Parsed kernel version version=5.15.0-1057`, `2024-05-08 15:23:58.715 [DEBUG][216945] felix/table.go 851: Parsing line ipVersion=0x4 line="# Generated by iptables-nft-save v1.8.4 on Wed May 8 15:23:58 2024" table="nat"`, - `2024-05-08 15:23:58.716 [DEBUG][216945] felix/table.go 851: Parsing line ipVersion=0x4 line="*nat" table="nat"`, + `2024-05-08 15:23:58.716 [DEBUG][216945] felix/table.go 851:Parsing line ipVersion=0x4 line="*nat" table="nat"`, `2024-05-08 15:23:58.716 [DEBUG][216945] felix/table.go 881: Not an append, skipping ipVersion=0x4 line="# Generated by iptables-nft-save v1.8.4 on Wed May 8 15:23:58 2024" table="nat"`, - `2024-05-08 15:23:58.716 [DEBUG][216945] felix/table.go 881: Not an append, skipping ipVersion=0x4 line="*nat" table="nat"`, + `2024-05-08 15:23:58.716 [DEBUG][216945] felix/table.go 881:Not an append, skipping ipVersion=0x4 line="*nat" table="nat"`, `2024-05-08 15:23:58.717 [DEBUG][216945] felix/table.go 851: Parsing line ipVersion=0x4 line=":POSTROUTING ACCEPT [0:0]" table="nat"`, `2024-05-08 15:23:58.717 [DEBUG][216945] felix/table.go 870: Found forward-reference chainName="POSTROUTING" ipVersion=0x4 line=":POSTROUTING ACCEPT [0:0]" table="nat"`, `2024-05-08 15:23:58.718 [DEBUG][216945] felix/table.go 851: Parsing line ipVersion=0x4 line=":OUTPUT ACCEPT [0:0]" table="nat"`, `2024-05-08 15:23:58.718 [DEBUG][216945] felix/table.go 851: Parsing line ipVersion=0x4 line=":PREROUTING ACCEPT [0:0]" table="nat"`, `2024-05-08 15:23:58.718 [DEBUG][216945] felix/table.go 870: Found forward-reference chainName="OUTPUT" ipVersion=0x4 line=":OUTPUT ACCEPT [0:0]" table="nat"`, `2024-05-08 15:23:58.718 [DEBUG][216945] felix/table.go 870: Found forward-reference chainName="PREROUTING" ipVersion=0x4 line=":PREROUTING ACCEPT [0:0]" table="nat"`, - `2024-05-08 <_> <_> felix/endpoint_mgr.go 443: Reporting endpoint status. dirtyEndpoints=set.Set{}`, - `2024-05-08 <_> <_> felix/health.go 167: Health: <_>`, - `2024-05-08 <_> <_> felix/health.go 196: Checking state of reporter reporter=&health.reporterState{name:"async_calc_graph", reports:health.HealthReport{Live:true, Ready:true, Detail:""}, timeout:20000000000, latest:health.HealthReport{Live:true, Ready:true, Detail:""}, <_> <_> loc:(*time.Location)(0x4ce3aa0)}}`, - `2024-05-08 <_> <_> felix/health.go 196: Checking state of reporter reporter=&health.reporterState{name:"felix-startup", reports:health.HealthReport{Live:true, Ready:true, Detail:""}, timeout:0, latest:health.HealthReport{Live:true, Ready:true, Detail:""}, <_> <_> loc:(*time.Location)(0x4ce3aa0)}}`, - `2024-05-08 <_> <_> felix/health.go 196: Checking state of reporter reporter=&health.reporterState{name:"int_dataplane", reports:health.HealthReport{Live:true, Ready:true, Detail:""}, timeout:90000000000, latest:health.HealthReport{Live:true, Ready:true, Detail:""}, <_> <_> loc:(*time.Location)(0x4ce3aa0)}}`, - `2024-05-08 <_> <_> felix/health.go 245: Calculated health summary healthResult=&health.HealthReport{Live:true, Ready:true, Detail:"+------------------+---------+----------------+-----------------+--------+\n| COMPONENT | TIMEOUT | LIVENESS | READINESS | DETAIL |\n+------------------+---------+----------------+-----------------+--------+\n| async_calc_graph | 20s | reporting live | reporting ready | |\n| felix-startup | 0s | reporting live | reporting ready | |\n| int_dataplane | 1m30s | reporting live | reporting ready | |\n+------------------+---------+----------------+-----------------+--------+"}`, - `2024-05-08 <_> <_> felix/health.go <_> GET <_>`, - `2024-05-08 <_> <_> felix/int_dataplane.go 1773: Refreshing IP sets state`, - `2024-05-08 <_> <_> felix/int_dataplane.go 1807: Applying dataplane updates`, - `2024-05-08 <_> <_> felix/int_dataplane.go 2080: Asked to reschedule. <_>`, - `2024-05-08 <_> <_> felix/ipsets.go 234: Asked to resync with the dataplane on next update. family="inet"`, - `2024-05-08 <_> <_> felix/ipsets.go 314: Resyncing ipsets with dataplane. family="inet"`, - `2024-05-08 <_> <_> felix/ipsets.go 366: Finished IPSets resync family="inet" numInconsistenciesFound=0 <_>`, - `2024-05-08 <_> <_> felix/ipsets.go 426: Parsing IP set. family="inet" <_>`, - `2024-05-08 <_> <_> felix/ipsets.go 467: Found member in dataplane <_> family="inet" <_> setID="this-host"`, - `2024-05-08 <_> <_> felix/ipsets.go 589: Whitelisting IP sets. ID="all-ipam-pools" family="inet" mainName="cali40all-ipam-pools"`, - `2024-05-08 <_> <_> felix/ipsets.go 589: Whitelisting IP sets. ID="masq-ipam-pools" family="inet" mainName="cali40masq-ipam-pools"`, - `2024-05-08 <_> <_> felix/ipsets.go 589: Whitelisting IP sets. ID="this-host" family="inet" mainName="cali40this-host"`, - `2024-05-08 <_> <_> felix/ipsets.go 607: Skipping expected Calico IP set. family="inet" <_>`, - `2024-05-08 <_> <_> felix/ipsets.go 643: No dirty IP sets. family="inet"`, - `2024-05-08 <_> <_> felix/summary.go 100: Summarising <_> dataplane reconciliation loops over <_> <_> <_> <_>`, - `2024-05-08 <_> <_> felix/sync_client.go 347: Ping received from Typha connID=0x0 connection=&discovery.Typha{Addr:"", IP:"", NodeName:(*string)(nil)} type=""`, - `2024-05-08 <_> <_> felix/sync_client.go 356: Pong sent to Typha connID=0x0 connection=&discovery.Typha{Addr:"", IP:"", NodeName:(*string)(nil)} type=""`, - `2024-05-08 <_> <_> felix/sync_client.go 434: New message from Typha. connID=0x0 connection=&discovery.Typha{Addr:"", IP:"", NodeName:(*string)(nil)} envelope=syncproto.Envelope{Message:syncproto.MsgPing{Timestamp:time.Date(2024, time.May, 8, 15, 23, <_> <_> time.Local)}} type=""`, - `2024-05-08 <_> <_> felix/table.go 1233: In nftables mode, restarting transaction between updates and deletions. ipVersion=0x4 <_>`, - `2024-05-08 <_> <_> felix/table.go 1263: Update ended up being no-op, skipping call to ip(6)tables-restore. ipVersion=0x4 <_>`, - `2024-05-08 <_> <_> felix/wireguard.go 652: Wireguard is not enabled, skipping sync ipVersion=0x4`, - `2024-05-08 <_> <_> felix/xdp_state.go 1004: Updating ipsetIDsToMembers cache. family=4`, - `2024-05-08 <_> <_> felix/xdp_state.go 1043: Processing pending diff state. cs=&intdataplane.xdpSystemState{IfaceNameToData:map[string]intdataplane.xdpIfaceData{}, XDPEligiblePolicies:map[proto.PolicyID]intdataplane.xdpRules{}} family=4`, - `2024-05-08 <_> <_> felix/xdp_state.go 1270: Finished processing pending diff state. bpfActions=intdataplane.xdpBPFActions{CreateMap:set.Typed[string]{}, RemoveMap:set.Typed[string]{}, AddToMap:map[string]map[string]uint32{}, RemoveFromMap:map[string]map[string]uint32{}, InstallXDP:set.Typed[string]{}, UninstallXDP:set.Typed[string]{}, MembersToDrop:map[string]map[string]uint32{}, MembersToAdd:map[string]map[string]uint32{}} family=4 newCS=&intdataplane.xdpSystemState{IfaceNameToData:map[string]intdataplane.xdpIfaceData{}, XDPEligiblePolicies:map[proto.PolicyID]intdataplane.xdpRules{}}`, - `2024-05-08 <_> <_> felix/xdp_state.go 1605: Getting member changes. family=4 oldMembers=map[string]set.Set[string]{}`, - `2024-05-08 <_> <_> felix/xdp_state.go 1798: Processing BPF actions. family="ipv4"`, - `2024-05-08 <_> <_> felix/xdp_state.go 1932: Finished processing BPF actions. family="ipv4"`, - `2024-05-08 <_> <_> felix/xdp_state.go 968: Processing member updates. family=4`, - `2024-05-08 <_> [DEBUG][216945] felix/table.go 851: Parsing line ipVersion=0x4 <_> - [0:0]" table="nat"`, - `2024-05-08 <_> [DEBUG][216945] felix/table.go 870: Found forward-reference <_> ipVersion=0x4 <_> - [0:0]" table="nat"`, - `2024-05-08 <_> [DEBUG][3576126] felix/int_dataplane.go 954: Skipping interface for MTU detection <_> <_>`, - `2024-05-08 <_> [DEBUG][615489] felix/table.go 677: Skipping expected chain <_> ipVersion=0x4 table="filter"`, - `2024-05-08 <_> [DEBUG][76] felix/route_table.go 557: Resync: found calico-owned interface <_> ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, - `2024-05-08 <_> [DEBUG][76] felix/route_table.go 614: Synchronised routes on interface <_> ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, - `2024-05-08 <_> [DEBUG][76] felix/route_table.go 661: Syncing interface routes <_> ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, - `2024-05-08 <_> [DEBUG][76] felix/route_table.go 686: Reconcile against kernel programming <_> ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, - `2024-05-08 <_> [DEBUG][76] felix/route_table.go 880: Processing route: 254 <_> <_> <_> ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, - `2024-05-08 <_> [DEBUG][76] felix/route_table.go 915: Route is correct <_> <_> ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, + `2024-05-08 15:23:<_> <_> felix/endpoint_mgr.go 443: Reporting endpoint status. dirtyEndpoints=set.Set{}`, + `2024-05-08 15:23:<_> <_> felix/health.go 167: Health: <_>`, + `2024-05-08 15:23:<_> <_> felix/health.go 196: Checking state of reporter reporter=&health.reporterState{name:"async_calc_graph", reports:health.HealthReport{Live:true, Ready:true, Detail:""}, timeout:20000000000, latest:health.HealthReport{Live:true, Ready:true, Detail:""}, timestamp:time.Time{wall:<_> ext:<_> loc:(*time.Location)(0x4ce3aa0)}}`, + `2024-05-08 15:23:<_> <_> felix/health.go 196: Checking state of reporter reporter=&health.reporterState{name:"felix-startup", reports:health.HealthReport{Live:true, Ready:true, Detail:""}, timeout:0, latest:health.HealthReport{Live:true, Ready:true, Detail:""}, timestamp:time.Time{wall:<_> ext:<_> loc:(*time.Location)(0x4ce3aa0)}}`, + `2024-05-08 15:23:<_> <_> felix/health.go 196: Checking state of reporter reporter=&health.reporterState{name:"int_dataplane", reports:health.HealthReport{Live:true, Ready:true, Detail:""}, timeout:90000000000, latest:health.HealthReport{Live:true, Ready:true, Detail:""}, timestamp:time.Time{wall:<_> ext:<_> loc:(*time.Location)(0x4ce3aa0)}}`, + `2024-05-08 15:23:<_> <_> felix/health.go 245: Calculated health summary healthResult=&health.HealthReport{Live:true, Ready:true, Detail:"+------------------+---------+----------------+-----------------+--------+\n| COMPONENT | TIMEOUT | LIVENESS | READINESS | DETAIL |\n+------------------+---------+----------------+-----------------+--------+\n| async_calc_graph | 20s | reporting live | reporting ready | |\n| felix-startup | 0s | reporting live | reporting ready | |\n| int_dataplane | 1m30s | reporting live | reporting ready | |\n+------------------+---------+----------------+-----------------+--------+"}`, + `2024-05-08 15:23:<_> <_> felix/health.go <_> GET <_>`, + `2024-05-08 15:23:<_> <_> felix/int_dataplane.go 1773: Refreshing IP sets state`, + `2024-05-08 15:23:<_> <_> felix/int_dataplane.go 1807: Applying dataplane updates`, + `2024-05-08 15:23:<_> <_> felix/int_dataplane.go 2080: Asked to reschedule. <_>`, + `2024-05-08 15:23:<_> <_> felix/ipsets.go 234: Asked to resync with the dataplane on next update. family="inet"`, + `2024-05-08 15:23:<_> <_> felix/ipsets.go 314: Resyncing ipsets with dataplane. family="inet"`, + `2024-05-08 15:23:<_> <_> felix/ipsets.go 426: Parsing IP set. family="inet" <_>`, + `2024-05-08 15:23:<_> <_> felix/ipsets.go 607: Skipping expected Calico IP set. family="inet" <_>`, + `2024-05-08 15:23:<_> <_> felix/ipsets.go 643: No dirty IP sets. family="inet"`, + `2024-05-08 15:23:<_> <_> felix/summary.go 100: Summarising <_> dataplane reconciliation loops over <_> <_> <_> <_>`, + `2024-05-08 15:23:<_> <_> felix/sync_client.go 347: Ping received from Typha connID=0x0 connection=&discovery.Typha{Addr:"", IP:"", NodeName:(*string)(nil)} type=""`, + `2024-05-08 15:23:<_> <_> felix/sync_client.go 356: Pong sent to Typha connID=0x0 connection=&discovery.Typha{Addr:"", IP:"", NodeName:(*string)(nil)} type=""`, + `2024-05-08 15:23:<_> <_> felix/sync_client.go 434: New message from Typha. connID=0x0 connection=&discovery.Typha{Addr:"", IP:"", NodeName:(*string)(nil)} envelope=syncproto.Envelope{Message:syncproto.MsgPing{Timestamp:time.Date(2024, time.May, 8, 15, 23, <_> <_> time.Local)}} type=""`, + `2024-05-08 15:23:<_> <_> felix/table.go 1233: In nftables mode, restarting transaction between updates and deletions. ipVersion=0x4 <_>`, + `2024-05-08 15:23:<_> <_> felix/table.go 1263: Update ended up being no-op, skipping call to ip(6)tables-restore. ipVersion=0x4 <_>`, + `2024-05-08 15:23:<_> <_> felix/wireguard.go 652: Wireguard is not enabled, skipping sync ipVersion=0x4`, + `2024-05-08 15:23:<_> <_> felix/xdp_state.go 1004: Updating ipsetIDsToMembers cache. family=4`, + `2024-05-08 15:23:<_> <_> felix/xdp_state.go 1043: Processing pending diff state. cs=&intdataplane.xdpSystemState{IfaceNameToData:map[string]intdataplane.xdpIfaceData{}, XDPEligiblePolicies:map[proto.PolicyID]intdataplane.xdpRules{}} family=4`, + `2024-05-08 15:23:<_> <_> felix/xdp_state.go 1270: Finished processing pending diff state. bpfActions=intdataplane.xdpBPFActions{CreateMap:set.Typed[string]{}, RemoveMap:set.Typed[string]{}, AddToMap:map[string]map[string]uint32{}, RemoveFromMap:map[string]map[string]uint32{}, InstallXDP:set.Typed[string]{}, UninstallXDP:set.Typed[string]{}, MembersToDrop:map[string]map[string]uint32{}, MembersToAdd:map[string]map[string]uint32{}} family=4 newCS=&intdataplane.xdpSystemState{IfaceNameToData:map[string]intdataplane.xdpIfaceData{}, XDPEligiblePolicies:map[proto.PolicyID]intdataplane.xdpRules{}}`, + `2024-05-08 15:23:<_> <_> felix/xdp_state.go 1605: Getting member changes. family=4 oldMembers=map[string]set.Set[string]{}`, + `2024-05-08 15:23:<_> <_> felix/xdp_state.go 1798: Processing BPF actions. family="ipv4"`, + `2024-05-08 15:23:<_> <_> felix/xdp_state.go 1932: Finished processing BPF actions. family="ipv4"`, + `2024-05-08 15:23:<_> <_> felix/xdp_state.go 968: Processing member updates. family=4`, + `2024-05-08 15:23:<_> [DEBUG][216945] felix/table.go 851: Parsing line ipVersion=0x4 line=":<_> - [0:0]" table="nat"`, + `2024-05-08 15:23:<_> [DEBUG][216945] felix/table.go 870: Found forward-reference <_> ipVersion=0x4 line=":<_> - [0:0]" table="nat"`, + `2024-05-08 15:23:<_> [DEBUG][3576126] felix/int_dataplane.go 954: Skipping interface for MTU detection <_> <_>`, + `2024-05-08 <_> <_> felix/ipsets.go 366:Finished IPSets resync family="inet" numInconsistenciesFound=0 resyncDuration=<_>`, + `2024-05-08 <_> <_> felix/ipsets.go 467:Found member in dataplane canon=<_> family="inet" member=<_> setID="this-host"`, + `2024-05-08 <_> <_> felix/ipsets.go 589:Whitelisting IP sets. ID="all-ipam-pools" family="inet" mainName="cali40all-ipam-pools"`, + `2024-05-08 <_> <_> felix/ipsets.go 589:Whitelisting IP sets. ID="masq-ipam-pools" family="inet" mainName="cali40masq-ipam-pools"`, + `2024-05-08 <_> <_> felix/ipsets.go 589:Whitelisting IP sets. ID="this-host" family="inet" mainName="cali40this-host"`, + `2024-05-08 <_> [DEBUG][615489] felix/table.go 677:Skipping expected chain chainName=<_> ipVersion=0x4 table="filter"`, + `2024-05-08 <_> [DEBUG][76] felix/route_table.go 557:Resync:found calico-owned interface ifaceName=<_> ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, + `2024-05-08 <_> [DEBUG][76] felix/route_table.go 614:Synchronised routes on interface ifaceName=<_> ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, + `2024-05-08 <_> [DEBUG][76] felix/route_table.go 661:Syncing interface routes ifaceName=<_> ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, + `2024-05-08 <_> [DEBUG][76] felix/route_table.go 686:Reconcile against kernel programming ifaceName=<_> ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, + `2024-05-08 <_> [DEBUG][76] felix/route_table.go 880:Processing route:254 <_> <_> ifaceName=<_> ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, + `2024-05-08 <_> [DEBUG][76] felix/route_table.go 915:Route is correct dest=<_> ifaceName=<_> ifaceRegex="^azv.*" ipVersion=0x4 tableIndex=0`, `bird: Netlink: No route to host`, }, }, } for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { + t.Run(tt.inputFile, func(t *testing.T) { file, err := os.Open(tt.inputFile) require.NoError(t, err) defer file.Close() @@ -441,6 +404,7 @@ func TestDrain_TrainExtractsPatterns(t *testing.T) { } require.Equal(t, tt.patterns, output) + require.Falsef(t, outputPatternsForTestUpdate, `outputPatternsForTestUpdate should only be used locally to update test patterns.`) }) } } @@ -456,23 +420,23 @@ func TestDrain_TrainGeneratesMatchablePatterns(t *testing.T) { name: "should match each line against a pattern", drain: New(DefaultConfig(), nil), inputLines: []string{ - "test test test", - "test test test", - "test test test", - "test test test", + "test test test test", + "test test test test", + "test test test test", + "test test test test", }, }, { name: "should also match newlines", drain: New(DefaultConfig(), nil), inputLines: []string{ - `test test test + `test test test test `, - `test test test + `test test test test `, - `test test test + `test test test test `, - `test test test + `test test test test `, }, }, @@ -504,23 +468,23 @@ func TestDrain_TrainGeneratesPatternsMatchableByLokiPatternFilter(t *testing.T) name: "should extract patterns that all lines match", drain: New(DefaultConfig(), nil), inputLines: []string{ - "test 1 test", - "test 2 test", - "test 3 test", - "test 4 test", + "test 1 test test", + "test 2 test test", + "test 3 test test", + "test 4 test test", }, }, { name: "should extract patterns that match if line ends with newlines", drain: New(DefaultConfig(), nil), inputLines: []string{ - `test 1 test + `test 1 test test `, - `test 2 test + `test 2 test test `, - `test 3 test + `test 3 test test `, - `test 4 test + `test 4 test test `, }, }, @@ -528,20 +492,20 @@ func TestDrain_TrainGeneratesPatternsMatchableByLokiPatternFilter(t *testing.T) name: "should extract patterns that match if line ends with empty space", drain: New(DefaultConfig(), nil), inputLines: []string{ - `test 1 test `, - `test 2 test `, - `test 3 test `, - `test 4 test `, + `test 1 test test `, + `test 2 test test `, + `test 3 test test `, + `test 4 test test `, }, }, { name: "should extract patterns that match if line starts with empty space", drain: New(DefaultConfig(), nil), inputLines: []string{ - ` test 1 test`, - ` test 2 test`, - ` test 3 test`, - ` test 4 test`, + ` test 1 test test`, + ` test 2 test test`, + ` test 3 test test`, + ` test 4 test test`, }, }, } diff --git a/pkg/pattern/drain/line_tokenizer.go b/pkg/pattern/drain/line_tokenizer.go new file mode 100644 index 000000000000..1317fbe3fca8 --- /dev/null +++ b/pkg/pattern/drain/line_tokenizer.go @@ -0,0 +1,55 @@ +package drain + +import "strings" + +type LineTokenizer interface { + Tokenize(line string) []string + Join(tokens []string) string +} + +type spacesTokenizer struct{} + +func (spacesTokenizer) Tokenize(line string) []string { + return strings.Split(line, " ") +} + +func (spacesTokenizer) Join(tokens []string) string { + return strings.Join(tokens, " ") +} + +type splittingTokenizer struct{} + +func (splittingTokenizer) Tokenize(line string) []string { + numEquals := strings.Count(line, "=") + numColons := strings.Count(line, ":") + numSpaces := strings.Count(line, " ") + + expectedTokens := numSpaces + numEquals + keyvalSeparator := "=" + if numColons > numEquals { + keyvalSeparator = ":" + expectedTokens = numSpaces + numColons + } + + tokens := make([]string, 0, expectedTokens) + for _, token := range strings.SplitAfter(line, keyvalSeparator) { + tokens = append(tokens, strings.Split(token, " ")...) + } + return tokens +} + +func (splittingTokenizer) Join(tokens []string) string { + var builder strings.Builder + for _, token := range tokens { + if strings.HasSuffix(token, "=") || strings.HasSuffix(token, ":") { + builder.WriteString(token) + } else { + builder.WriteString(token + " ") + } + } + output := builder.String() + if output[len(output)-1] == ' ' { + return output[:len(output)-1] + } + return output +} diff --git a/pkg/pattern/drain/line_tokenizer_test.go b/pkg/pattern/drain/line_tokenizer_test.go new file mode 100644 index 000000000000..8cb541a61b62 --- /dev/null +++ b/pkg/pattern/drain/line_tokenizer_test.go @@ -0,0 +1,60 @@ +package drain + +import ( + "reflect" + "testing" +) + +func TestSplittingTokenizer_Tokenize(t *testing.T) { + tokenizer := splittingTokenizer{} + + tests := []struct { + name string + line string + want []string + }{ + { + name: "Test with equals sign", + line: "key1=value1 key2=value2", + want: []string{"key1=", "value1", "key2=", "value2"}, + }, + { + name: "Test with colon", + line: "key1:value1 key2:value2", + want: []string{"key1:", "value1", "key2:", "value2"}, + }, + { + name: "Test with mixed delimiters, more = than :", + line: "key1=value1 key2:value2 key3=value3", + want: []string{"key1=", "value1", "key2:value2", "key3=", "value3"}, + }, + { + name: "Test with mixed delimiters, more : than =", + line: "key1:value1 key2:value2 key3=value3", + want: []string{"key1:", "value1", "key2:", "value2", "key3=value3"}, + }, + { + name: "Dense json", + line: `{"key1":"value1","key2":"value2","key3":"value3"}`, + want: []string{`{"key1":`, `"value1","key2":`, `"value2","key3":`, `"value3"}`}, + }, + { + name: "json with spaces", + line: `{"key1":"value1", "key2":"value2", "key3":"value3"}`, + want: []string{`{"key1":`, `"value1",`, `"key2":`, `"value2",`, `"key3":`, `"value3"}`}, + }, + { + name: "logfmt multiword values", + line: `key1=value1 key2=value2 msg="this is a message"`, + want: []string{"key1=", "value1", "key2=", "value2", "msg=", `"this`, "is", "a", `message"`}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tokenizer.Tokenize(tt.line); !reflect.DeepEqual(got, tt.want) { + t.Errorf("splittingTokenizer.Tokenize() = %v, want %v", got, tt.want) + } + }) + } +} From 2923a7d95818055a6ae9557d4b2f733b1af826f3 Mon Sep 17 00:00:00 2001 From: benclive Date: Thu, 30 May 2024 13:53:06 +0100 Subject: [PATCH 37/41] fix: Update expected patterns when pruning (#13079) --- pkg/pattern/ingester_querier_test.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/pattern/ingester_querier_test.go b/pkg/pattern/ingester_querier_test.go index d1016b326df7..9d23662d419e 100644 --- a/pkg/pattern/ingester_querier_test.go +++ b/pkg/pattern/ingester_querier_test.go @@ -6,12 +6,13 @@ import ( "testing" "github.com/stretchr/testify/require" + "golang.org/x/exp/slices" "github.com/grafana/loki/v3/pkg/logproto" ) func Test_prunePatterns(t *testing.T) { - file, err := os.Open("testdata/patterns.txt") + file, err := os.Open(`testdata/patterns.txt`) require.NoError(t, err) defer file.Close() @@ -26,18 +27,20 @@ func Test_prunePatterns(t *testing.T) { prunePatterns(resp, 0) expectedPatterns := []string{ + `<_> caller=aggregator.go:139 level=info msg="received kafka message" topic=cortex-dev-01-aggregations partition=<_>`, + `<_> caller=batcher.go:155 level=info msg="batcher:processing aggregation result" <_> partitionID=<_> +0000 UTC, <_>`, + `<_> caller=batcher.go:155 level=info msg="batcher:processing aggregation result" result="user=9960, partitionID=<_> +0000 UTC, <_>`, + `<_> caller=batcher.go:155 level=info msg="batcher:processing aggregation result" result="user=9960, partitionID=<_> sampleTimestamp=2024-04-03 <_> +0000 UTC, <_>`, + `<_> caller=offset_committer.go:174 level=info msg="partition offset committer committed offset" topic=cortex-dev-01-aggregations partition=<_> +0000 UTC" <_> +0000 UTC" <_> currentBuckets="unsupported value type"`, + `<_> caller=offset_committer.go:174 level=info msg="partition offset committer committed offset" topic=cortex-dev-01-aggregations partition=<_> handledMessageTime="2024-04-03 <_> +0000 UTC" <_> +0000 UTC" <_> currentBuckets="unsupported value type"`, `<_> caller=wrapper.go:48 level=info component=distributor msg="sample remote write" eventType=bi <_>`, - `<_> caller=offset_committer.go:174 level=info msg="partition offset committer committed offset" topic=cortex-dev-01-aggregations <_> +0000 UTC" <_> +0000 UTC" <_> currentBuckets="unsupported value type"`, - `<_> caller=aggregator.go:139 level=info msg="received kafka message" topic=cortex-dev-01-aggregations <_>`, - `<_> caller=batcher.go:155 level=info msg="batcher: processing aggregation result" result="user=9960, <_> sampleTimestamp=2024-04-03 <_> +0000 UTC, <_>`, - `<_> caller=offset_committer.go:174 level=info msg="partition offset committer committed offset" topic=cortex-dev-01-aggregations <_> handledMessageTime="2024-04-03 <_> +0000 UTC" <_> +0000 UTC" <_> currentBuckets="unsupported value type"`, - `<_> caller=batcher.go:155 level=info msg="batcher: processing aggregation result" <_> +0000 UTC, <_>`, } patterns := make([]string, 0, len(resp.Series)) for _, p := range resp.Series { patterns = append(patterns, p.Pattern) } + slices.Sort(patterns) require.Equal(t, expectedPatterns, patterns) } From 21dd4afdc76d7790e177d2dd364ecf5b629c8112 Mon Sep 17 00:00:00 2001 From: Jack Baldry Date: Thu, 30 May 2024 14:34:05 +0100 Subject: [PATCH 38/41] docs: Republish the sizing calculator but don't list it in the table of contents and don't index it (#13070) Co-authored-by: J Stickler --- docs/sources/setup/size/_index.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/sources/setup/size/_index.md b/docs/sources/setup/size/_index.md index 74dcb8e50496..162748eb9e3b 100644 --- a/docs/sources/setup/size/_index.md +++ b/docs/sources/setup/size/_index.md @@ -1,4 +1,7 @@ --- +_build: + list: false +noindex: true title: Size the cluster menuTitle: Size the cluster description: Provides a tool that generates a Helm Chart values.yaml file based on expected ingestion, retention rate, and node type, to help size your Grafana deployment. @@ -6,7 +9,6 @@ aliases: - ../installation/sizing/ - ../installation/helm/generate weight: 100 -draft: true --- From f6f8babf83f3d90f4e6f3f9b732fe22382861f47 Mon Sep 17 00:00:00 2001 From: benclive Date: Fri, 31 May 2024 10:21:50 +0100 Subject: [PATCH 39/41] feat: Add utf8 support to Pattern Lexer to support utf8 chars (#13085) --- pkg/logql/log/pattern/lexer.go | 5 +- pkg/logql/log/pattern/lexer.rl | 16 +- pkg/logql/log/pattern/lexer.rl.go | 362 ++++++++++++++------------ pkg/logql/log/pattern/lexer_test.go | 1 + pkg/logql/log/pattern/parser_test.go | 5 + pkg/logql/log/pattern/pattern_test.go | 21 ++ pkg/pattern/drain/drain_test.go | 16 +- 7 files changed, 252 insertions(+), 174 deletions(-) diff --git a/pkg/logql/log/pattern/lexer.go b/pkg/logql/log/pattern/lexer.go index 2f867f93dc30..febe39a1331e 100644 --- a/pkg/logql/log/pattern/lexer.go +++ b/pkg/logql/log/pattern/lexer.go @@ -1,5 +1,7 @@ package pattern +import "unicode/utf8" + type lexer struct { data []byte p, pe, cs int @@ -57,6 +59,7 @@ func (lex *lexer) identifier(out *exprSymType) (int, error) { // nolint func (lex *lexer) literal(out *exprSymType) (int, error) { - out.literal = rune(lex.data[lex.ts]) + decoded, _ := utf8.DecodeRune(lex.data[lex.ts:lex.te]) + out.literal = decoded return LITERAL, nil } diff --git a/pkg/logql/log/pattern/lexer.rl b/pkg/logql/log/pattern/lexer.rl index 7b1d25467c45..b28579afb424 100644 --- a/pkg/logql/log/pattern/lexer.rl +++ b/pkg/logql/log/pattern/lexer.rl @@ -13,11 +13,25 @@ package pattern } }%% +%%{ +utf8 = ( + 0x00..0x7F | + 0xC2..0xDF 0x80..0xBF | + 0xE0 0xA0..0xBF 0x80..0xBF | + 0xE1..0xEC 0x80..0xBF 0x80..0xBF | + 0xED 0x80..0x9F 0x80..0xBF | + 0xEE..0xEF 0x80..0xBF 0x80..0xBF | + 0xF0 0x90..0xBF 0x80..0xBF 0x80..0xBF | + 0xF1..0xF3 0x80..0xBF 0x80..0xBF 0x80..0xBF | + 0xF4 0x80..0x8F 0x80..0xBF 0x80..0xBF +); +}%% + const LEXER_ERROR = 0 %%{ identifier = '<' (alpha| '_') (alnum | '_' )* '>'; - literal = any; + literal = utf8; }%% func (lex *lexer) Lex(out *exprSymType) int { diff --git a/pkg/logql/log/pattern/lexer.rl.go b/pkg/logql/log/pattern/lexer.rl.go index 4e3b3e188fca..67adde2c3101 100644 --- a/pkg/logql/log/pattern/lexer.rl.go +++ b/pkg/logql/log/pattern/lexer.rl.go @@ -1,251 +1,271 @@ + //line pkg/logql/log/pattern/lexer.rl:1 package pattern + //line pkg/logql/log/pattern/lexer.rl.go:7 var _pattern_actions []byte = []byte{ - 0, 1, 0, 1, 1, 1, 2, 1, 3, - 1, 4, 1, 5, 1, 6, + 0, 1, 0, 1, 1, 1, 2, 1, 3, + 1, 4, 1, 5, 1, 6, } var _pattern_key_offsets []byte = []byte{ - 0, 8, 9, + 0, 0, 8, 10, 12, 14, 16, 18, + 20, 22, 37, } var _pattern_trans_keys []byte = []byte{ - 62, 95, 48, 57, 65, 90, 97, 122, - 60, 95, 65, 90, 97, 122, + 62, 95, 48, 57, 65, 90, 97, 122, + 128, 191, 160, 191, 128, 191, 128, 159, + 144, 191, 128, 191, 128, 143, 60, 224, + 237, 240, 244, 128, 193, 194, 223, 225, + 239, 241, 243, 245, 255, 95, 65, 90, + 97, 122, } var _pattern_single_lengths []byte = []byte{ - 2, 1, 1, + 0, 2, 0, 0, 0, 0, 0, 0, + 0, 5, 1, } var _pattern_range_lengths []byte = []byte{ - 3, 0, 2, + 0, 3, 1, 1, 1, 1, 1, 1, + 1, 5, 2, } var _pattern_index_offsets []byte = []byte{ - 0, 6, 8, + 0, 0, 6, 8, 10, 12, 14, 16, + 18, 20, 31, +} + +var _pattern_indicies []byte = []byte{ + 2, 1, 1, 1, 1, 0, 3, 4, + 5, 4, 5, 4, 5, 4, 6, 4, + 6, 4, 6, 4, 7, 8, 9, 10, + 12, 4, 5, 6, 11, 4, 3, 1, + 1, 1, 13, } var _pattern_trans_targs []byte = []byte{ - 1, 0, 0, 0, 0, 1, 2, 1, - 0, 0, 0, 1, 1, 1, + 9, 1, 9, 9, 0, 2, 4, 10, + 3, 5, 6, 7, 8, 9, } var _pattern_trans_actions []byte = []byte{ - 7, 0, 0, 0, 0, 13, 5, 9, - 0, 0, 0, 11, 13, 11, + 13, 0, 7, 9, 0, 0, 0, 5, + 0, 0, 0, 0, 0, 11, } var _pattern_to_state_actions []byte = []byte{ - 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, } var _pattern_from_state_actions []byte = []byte{ - 0, 3, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 3, 0, } var _pattern_eof_trans []byte = []byte{ - 13, 0, 14, + 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 14, } -const pattern_start int = 1 -const pattern_first_final int = 1 -const pattern_error int = -1 +const pattern_start int = 9 +const pattern_first_final int = 9 +const pattern_error int = 0 + +const pattern_en_main int = 9 -const pattern_en_main int = 1 //line pkg/logql/log/pattern/lexer.rl:14 + + +//line pkg/logql/log/pattern/lexer.rl:28 + + const LEXER_ERROR = 0 -//line pkg/logql/log/pattern/lexer.rl:21 + +//line pkg/logql/log/pattern/lexer.rl:35 + func (lex *lexer) Lex(out *exprSymType) int { - eof := lex.pe - tok := 0 + eof := lex.pe + tok := 0 -//line pkg/logql/log/pattern/lexer.rl.go:77 + +//line pkg/logql/log/pattern/lexer.rl.go:100 { - var _klen int - var _trans int - var _acts int - var _nacts uint - var _keys int - if (lex.p) == (lex.pe) { - goto _test_eof - } - _resume: - _acts = int(_pattern_from_state_actions[lex.cs]) - _nacts = uint(_pattern_actions[_acts]) - _acts++ - for ; _nacts > 0; _nacts-- { - _acts++ - switch _pattern_actions[_acts-1] { - case 1: + var _klen int + var _trans int + var _acts int + var _nacts uint + var _keys int + if ( lex.p) == ( lex.pe) { + goto _test_eof + } + if lex.cs == 0 { + goto _out + } +_resume: + _acts = int(_pattern_from_state_actions[ lex.cs]) + _nacts = uint(_pattern_actions[_acts]); _acts++ + for ; _nacts > 0; _nacts-- { + _acts++ + switch _pattern_actions[_acts - 1] { + case 1: //line NONE:1 - lex.ts = (lex.p) + lex.ts = ( lex.p) -//line pkg/logql/log/pattern/lexer.rl.go:97 - } +//line pkg/logql/log/pattern/lexer.rl.go:123 } + } - _keys = int(_pattern_key_offsets[lex.cs]) - _trans = int(_pattern_index_offsets[lex.cs]) - - _klen = int(_pattern_single_lengths[lex.cs]) - if _klen > 0 { - _lower := int(_keys) - var _mid int - _upper := int(_keys + _klen - 1) - for { - if _upper < _lower { - break - } - - _mid = _lower + ((_upper - _lower) >> 1) - switch { - case lex.data[(lex.p)] < _pattern_trans_keys[_mid]: - _upper = _mid - 1 - case lex.data[(lex.p)] > _pattern_trans_keys[_mid]: - _lower = _mid + 1 - default: - _trans += int(_mid - int(_keys)) - goto _match - } + _keys = int(_pattern_key_offsets[ lex.cs]) + _trans = int(_pattern_index_offsets[ lex.cs]) + + _klen = int(_pattern_single_lengths[ lex.cs]) + if _klen > 0 { + _lower := int(_keys) + var _mid int + _upper := int(_keys + _klen - 1) + for { + if _upper < _lower { + break } - _keys += _klen - _trans += _klen - } - _klen = int(_pattern_range_lengths[lex.cs]) - if _klen > 0 { - _lower := int(_keys) - var _mid int - _upper := int(_keys + (_klen << 1) - 2) - for { - if _upper < _lower { - break - } - - _mid = _lower + (((_upper - _lower) >> 1) & ^1) - switch { - case lex.data[(lex.p)] < _pattern_trans_keys[_mid]: - _upper = _mid - 2 - case lex.data[(lex.p)] > _pattern_trans_keys[_mid+1]: - _lower = _mid + 2 - default: - _trans += int((_mid - int(_keys)) >> 1) - goto _match - } + _mid = _lower + ((_upper - _lower) >> 1) + switch { + case lex.data[( lex.p)] < _pattern_trans_keys[_mid]: + _upper = _mid - 1 + case lex.data[( lex.p)] > _pattern_trans_keys[_mid]: + _lower = _mid + 1 + default: + _trans += int(_mid - int(_keys)) + goto _match } - _trans += _klen } + _keys += _klen + _trans += _klen + } - _match: - _eof_trans: - lex.cs = int(_pattern_trans_targs[_trans]) + _klen = int(_pattern_range_lengths[ lex.cs]) + if _klen > 0 { + _lower := int(_keys) + var _mid int + _upper := int(_keys + (_klen << 1) - 2) + for { + if _upper < _lower { + break + } - if _pattern_trans_actions[_trans] == 0 { - goto _again + _mid = _lower + (((_upper - _lower) >> 1) & ^1) + switch { + case lex.data[( lex.p)] < _pattern_trans_keys[_mid]: + _upper = _mid - 2 + case lex.data[( lex.p)] > _pattern_trans_keys[_mid + 1]: + _lower = _mid + 2 + default: + _trans += int((_mid - int(_keys)) >> 1) + goto _match + } } + _trans += _klen + } + +_match: + _trans = int(_pattern_indicies[_trans]) +_eof_trans: + lex.cs = int(_pattern_trans_targs[_trans]) + + if _pattern_trans_actions[_trans] == 0 { + goto _again + } - _acts = int(_pattern_trans_actions[_trans]) - _nacts = uint(_pattern_actions[_acts]) + _acts = int(_pattern_trans_actions[_trans]) + _nacts = uint(_pattern_actions[_acts]); _acts++ + for ; _nacts > 0; _nacts-- { _acts++ - for ; _nacts > 0; _nacts-- { - _acts++ - switch _pattern_actions[_acts-1] { - case 2: + switch _pattern_actions[_acts-1] { + case 2: //line NONE:1 - lex.te = (lex.p) + 1 - - case 3: -//line pkg/logql/log/pattern/lexer.rl:30 - lex.te = (lex.p) + 1 - { - tok = lex.handle(lex.identifier(out)) - (lex.p)++ - goto _out - } - case 4: -//line pkg/logql/log/pattern/lexer.rl:31 - lex.te = (lex.p) + 1 - { - tok = lex.handle(lex.literal(out)) - (lex.p)++ - goto _out - } - case 5: -//line pkg/logql/log/pattern/lexer.rl:31 - lex.te = (lex.p) - (lex.p)-- - { - tok = lex.handle(lex.literal(out)) - (lex.p)++ - goto _out - } - case 6: -//line pkg/logql/log/pattern/lexer.rl:31 - (lex.p) = (lex.te) - 1 - { - tok = lex.handle(lex.literal(out)) - (lex.p)++ - goto _out - } -//line pkg/logql/log/pattern/lexer.rl.go:191 - } + lex.te = ( lex.p)+1 + + case 3: +//line pkg/logql/log/pattern/lexer.rl:44 + lex.te = ( lex.p)+1 +{ tok = lex.handle(lex.identifier(out)); ( lex.p)++; goto _out + } + case 4: +//line pkg/logql/log/pattern/lexer.rl:45 + lex.te = ( lex.p)+1 +{ tok = lex.handle(lex.literal(out)); ( lex.p)++; goto _out + } + case 5: +//line pkg/logql/log/pattern/lexer.rl:45 + lex.te = ( lex.p) +( lex.p)-- +{ tok = lex.handle(lex.literal(out)); ( lex.p)++; goto _out + } + case 6: +//line pkg/logql/log/pattern/lexer.rl:45 +( lex.p) = ( lex.te) - 1 +{ tok = lex.handle(lex.literal(out)); ( lex.p)++; goto _out + } +//line pkg/logql/log/pattern/lexer.rl.go:218 } + } - _again: - _acts = int(_pattern_to_state_actions[lex.cs]) - _nacts = uint(_pattern_actions[_acts]) +_again: + _acts = int(_pattern_to_state_actions[ lex.cs]) + _nacts = uint(_pattern_actions[_acts]); _acts++ + for ; _nacts > 0; _nacts-- { _acts++ - for ; _nacts > 0; _nacts-- { - _acts++ - switch _pattern_actions[_acts-1] { - case 0: + switch _pattern_actions[_acts-1] { + case 0: //line NONE:1 - lex.ts = 0 + lex.ts = 0 -//line pkg/logql/log/pattern/lexer.rl.go:205 - } +//line pkg/logql/log/pattern/lexer.rl.go:232 } + } - (lex.p)++ - if (lex.p) != (lex.pe) { - goto _resume - } - _test_eof: - { - } - if (lex.p) == eof { - if _pattern_eof_trans[lex.cs] > 0 { - _trans = int(_pattern_eof_trans[lex.cs] - 1) - goto _eof_trans - } + if lex.cs == 0 { + goto _out + } + ( lex.p)++ + if ( lex.p) != ( lex.pe) { + goto _resume + } + _test_eof: {} + if ( lex.p) == eof { + if _pattern_eof_trans[ lex.cs] > 0 { + _trans = int(_pattern_eof_trans[ lex.cs] - 1) + goto _eof_trans } + } - _out: - { - } + _out: {} } -//line pkg/logql/log/pattern/lexer.rl:35 +//line pkg/logql/log/pattern/lexer.rl:49 + - return tok + return tok; } -func (lex *lexer) init() { -//line pkg/logql/log/pattern/lexer.rl.go:233 +func (lex *lexer) init() { + +//line pkg/logql/log/pattern/lexer.rl.go:263 { - lex.cs = pattern_start - lex.ts = 0 - lex.te = 0 - lex.act = 0 + lex.cs = pattern_start + lex.ts = 0 + lex.te = 0 + lex.act = 0 } -//line pkg/logql/log/pattern/lexer.rl:43 +//line pkg/logql/log/pattern/lexer.rl:57 } diff --git a/pkg/logql/log/pattern/lexer_test.go b/pkg/logql/log/pattern/lexer_test.go index ff4d61591fb3..3e6bcf1b12e9 100644 --- a/pkg/logql/log/pattern/lexer_test.go +++ b/pkg/logql/log/pattern/lexer_test.go @@ -18,6 +18,7 @@ func Test_Lex(t *testing.T) { {`<_1foo>`, []int{IDENTIFIER}}, {`<_1foo> bar `, []int{IDENTIFIER, LITERAL, LITERAL, LITERAL, LITERAL, LITERAL, IDENTIFIER}}, {`<1foo>`, []int{LITERAL, LITERAL, LITERAL, LITERAL, LITERAL, LITERAL}}, + {`▶`, []int{LITERAL}}, } { tc := tc t.Run(tc.input, func(t *testing.T) { diff --git a/pkg/logql/log/pattern/parser_test.go b/pkg/logql/log/pattern/parser_test.go index 8a40d983c0a3..dbcb418fd382 100644 --- a/pkg/logql/log/pattern/parser_test.go +++ b/pkg/logql/log/pattern/parser_test.go @@ -47,6 +47,11 @@ func Test_Parse(t *testing.T) { expr{capture("ip"), literals(" - "), capture("user"), literals(" ["), capture("_"), literals(`] "`), capture("method"), literals(" "), capture("path"), literals(" "), capture('_'), literals(`" `), capture("status"), literals(" "), capture("size"), literals(" "), capture("url"), literals(" "), capture("user_agent")}, nil, }, + { + "▶", + expr{literals("▶")}, + nil, + }, } { tc := tc actual, err := parseExpr(tc.input) diff --git a/pkg/logql/log/pattern/pattern_test.go b/pkg/logql/log/pattern/pattern_test.go index 0d1c47f0bea2..ca4f3ea47d96 100644 --- a/pkg/logql/log/pattern/pattern_test.go +++ b/pkg/logql/log/pattern/pattern_test.go @@ -1,6 +1,7 @@ package pattern import ( + "bytes" "fmt" "testing" @@ -151,6 +152,26 @@ var fixtures = []struct { []string{"POST", "/api/v1/locations", "204", "154", "0", "226", "100", "10.0.35.28", "nsq2http", "tcp://10.0.2.1:80"}, true, }, + { + // UTF-8: Matches a unicode character + `unicode character`, + `unicode 🤷 character`, + []string{`🤷`}, + true, + }, + { + // UTF-8: Parses unicode character as literal + "unicode ▶ ", + "unicode ▶ character", + []string{"character"}, + true, + }, +} + +func Test_BytesIndexUnicode(t *testing.T) { + data := []byte("Hello ▶ World") + index := bytes.Index(data, []byte("▶")) + require.Equal(t, 6, index) } func Test_matcher_Matches(t *testing.T) { diff --git a/pkg/pattern/drain/drain_test.go b/pkg/pattern/drain/drain_test.go index 7c502a895e7e..cc16f0b7fd64 100644 --- a/pkg/pattern/drain/drain_test.go +++ b/pkg/pattern/drain/drain_test.go @@ -508,6 +508,19 @@ func TestDrain_TrainGeneratesPatternsMatchableByLokiPatternFilter(t *testing.T) ` test 4 test test`, }, }, + { + name: "Unicode characters are matchable", + drain: New(DefaultConfig(), nil), + inputLines: []string{ + `13:25:18.033470 ▶ INFO route ops sending to dest https://graphite-cortex-ops-blocks-us-east4.grafana.net/graphite/metrics: service_is_carbon-relay-ng.instance_is_carbon-relay-ng-c665b7b-j2trk.mtype_is_gauge.dest_is_https_graphite-cortex-ops-blocks-us-east4_grafana_netgraphitemetrics.unit_is_B.what_is_FlushSize.type_is_manual.stat_is_max_999 0.00 1717075518`, + `13:25:18.033422 ▶ INFO route ops sending to dest https://graphite-cortex-ops-blocks-us-east4.grafana.net/graphite/metrics: service_is_carbon-relay-ng.instance_is_carbon-relay-ng-c665b7b-j2trk.mtype_is_gauge.dest_is_https_graphite-cortex-ops-blocks-us-east4_grafana_netgraphitemetrics.unit_is_B.what_is_FlushSize.type_is_manual.stat_is_max_99 0.00 1717075518`, + `13:25:18.033394 ▶ INFO route ops sending to dest https://graphite-cortex-ops-blocks-us-east4.grafana.net/graphite/metrics: service_is_carbon-relay-ng.instance_is_carbon-relay-ng-c665b7b-j2trk.mtype_is_gauge.dest_is_https_graphite-cortex-ops-blocks-us-east4_grafana_netgraphitemetrics.unit_is_B.what_is_FlushSize.type_is_manual.stat_is_max_95 0.00 1717075518`, + `13:25:18.033364 ▶ INFO route ops sending to dest https://graphite-cortex-ops-blocks-us-east4.grafana.net/graphite/metrics: service_is_carbon-relay-ng.instance_is_carbon-relay-ng-c665b7b-j2trk.mtype_is_gauge.dest_is_https_graphite-cortex-ops-blocks-us-east4_grafana_netgraphitemetrics.unit_is_B.what_is_FlushSize.type_is_manual.stat_is_max_75 0.00 1717075518`, + `13:25:18.033335 ▶ INFO route ops sending to dest https://graphite-cortex-ops-blocks-us-east4.grafana.net/graphite/metrics: service_is_carbon-relay-ng.instance_is_carbon-relay-ng-c665b7b-j2trk.mtype_is_gauge.dest_is_https_graphite-cortex-ops-blocks-us-east4_grafana_netgraphitemetrics.unit_is_B.what_is_FlushSize.type_is_manual.stat_is_max_50 0.00 1717075518`, + `13:25:18.033304 ▶ INFO route ops sending to dest https://graphite-cortex-ops-blocks-us-east4.grafana.net/graphite/metrics: service_is_carbon-relay-ng.instance_is_carbon-relay-ng-c665b7b-j2trk.mtype_is_gauge.dest_is_https_graphite-cortex-ops-blocks-us-east4_grafana_netgraphitemetrics.unit_is_B.what_is_FlushSize.type_is_manual.stat_is_std 0.00 1717075518`, + `13:25:18.033281 ▶ INFO route ops sending to dest https://graphite-cortex-ops-blocks-us-east4.grafana.net/graphite/metrics: service_is_carbon-relay-ng.instance_is_carbon-relay-ng-c665b7b-j2trk.mtype_is_gauge.dest_is_https_graphite-cortex-ops-blocks-us-east4_grafana_netgraphitemetrics.unit_is_B.what_is_FlushSize.type_is_manual.stat_is_mean 0.00 1717075518`, + }, + }, } for _, tt := range tests { tt := tt @@ -523,7 +536,8 @@ func TestDrain_TrainGeneratesPatternsMatchableByLokiPatternFilter(t *testing.T) for _, line := range tt.inputLines { passes := matcher.Test([]byte(line)) - require.Truef(t, passes, `Line %q should match extracted pattern`, line) + require.Truef(t, passes, "Line should match extracted pattern: \nPatt[%q] \nLine[%q]", cluster.String(), line) + } }) } From 00d3c7a52d9f2b48fccb0cd5b105a2577b3d0305 Mon Sep 17 00:00:00 2001 From: Ed Welch Date: Fri, 31 May 2024 08:26:37 -0400 Subject: [PATCH 40/41] feat: add profile tagging to ingester (#13068) Signed-off-by: Edward Welch Co-authored-by: poyzannur Co-authored-by: Poyzan <31743851+poyzannur@users.noreply.github.com> --- pkg/ingester/ingester.go | 408 ++++++++++++++++++++++----------------- 1 file changed, 229 insertions(+), 179 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 6d27d349c93f..f23e1de227f6 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -9,6 +9,7 @@ import ( "os" "path" "path/filepath" + "runtime/pprof" "sync" "time" @@ -837,7 +838,12 @@ func (i *Ingester) Push(ctx context.Context, req *logproto.PushRequest) (*logpro if err != nil { return &logproto.PushResponse{}, err } - return &logproto.PushResponse{}, instance.Push(ctx, req) + + pprof.Do(ctx, pprof.Labels("path", "write", "tenant", instanceID), func(c context.Context) { + err = instance.Push(ctx, req) + }) + + return &logproto.PushResponse{}, err } // GetStreamRates returns a response containing all streams and their current rate @@ -848,7 +854,11 @@ func (i *Ingester) GetStreamRates(ctx context.Context, _ *logproto.StreamRatesRe defer sp.LogKV("event", "ingester finished handling GetStreamRates") } - allRates := i.streamRateCalculator.Rates() + var allRates []logproto.StreamRate + pprof.Do(ctx, pprof.Labels("path", "write"), func(c context.Context) { + allRates = i.streamRateCalculator.Rates() + }) + rates := make([]*logproto.StreamRate, len(allRates)) for idx := range allRates { rates[idx] = &allRates[idx] @@ -902,39 +912,45 @@ func (i *Ingester) Query(req *logproto.QueryRequest, queryServer logproto.Querie if err != nil { return err } - it, err := instance.Query(ctx, logql.SelectLogParams{QueryRequest: req}) - if err != nil { - return err - } - if start, end, ok := buildStoreRequest(i.cfg, req.Start, req.End, time.Now()); ok { - storeReq := logql.SelectLogParams{QueryRequest: &logproto.QueryRequest{ - Selector: req.Selector, - Direction: req.Direction, - Start: start, - End: end, - Limit: req.Limit, - Shards: req.Shards, - Deletes: req.Deletes, - Plan: req.Plan, - }} - storeItr, err := i.store.SelectLogs(ctx, storeReq) + pprof.Do(ctx, pprof.Labels("path", "read", "type", "log", "tenant", instanceID), func(c context.Context) { + var it iter.EntryIterator + it, err = instance.Query(ctx, logql.SelectLogParams{QueryRequest: req}) if err != nil { - util.LogErrorWithContext(ctx, "closing iterator", it.Close) - return err + return } - it = iter.NewMergeEntryIterator(ctx, []iter.EntryIterator{it, storeItr}, req.Direction) - } - defer util.LogErrorWithContext(ctx, "closing iterator", it.Close) + if start, end, ok := buildStoreRequest(i.cfg, req.Start, req.End, time.Now()); ok { + storeReq := logql.SelectLogParams{QueryRequest: &logproto.QueryRequest{ + Selector: req.Selector, + Direction: req.Direction, + Start: start, + End: end, + Limit: req.Limit, + Shards: req.Shards, + Deletes: req.Deletes, + Plan: req.Plan, + }} + var storeItr iter.EntryIterator + storeItr, err = i.store.SelectLogs(ctx, storeReq) + if err != nil { + util.LogErrorWithContext(ctx, "closing iterator", it.Close) + return + } + it = iter.NewMergeEntryIterator(ctx, []iter.EntryIterator{it, storeItr}, req.Direction) + } - // sendBatches uses -1 to specify no limit. - batchLimit := int32(req.Limit) - if batchLimit == 0 { - batchLimit = -1 - } + defer util.LogErrorWithContext(ctx, "closing iterator", it.Close) - return sendBatches(ctx, it, queryServer, batchLimit) + // sendBatches uses -1 to specify no limit. + batchLimit := int32(req.Limit) + if batchLimit == 0 { + batchLimit = -1 + } + err = sendBatches(ctx, it, queryServer, batchLimit) + }) + + return err } // QuerySample the ingesters for series from logs matching a set of matchers. @@ -965,35 +981,41 @@ func (i *Ingester) QuerySample(req *logproto.SampleQueryRequest, queryServer log return err } - it, err := instance.QuerySample(ctx, logql.SelectSampleParams{SampleQueryRequest: req}) - if err != nil { - return err - } - if sp != nil { - sp.LogKV("event", "finished instance query sample", "selector", req.Selector, "start", req.Start, "end", req.End) - } - - if start, end, ok := buildStoreRequest(i.cfg, req.Start, req.End, time.Now()); ok { - storeReq := logql.SelectSampleParams{SampleQueryRequest: &logproto.SampleQueryRequest{ - Start: start, - End: end, - Selector: req.Selector, - Shards: req.Shards, - Deletes: req.Deletes, - Plan: req.Plan, - }} - storeItr, err := i.store.SelectSamples(ctx, storeReq) + pprof.Do(ctx, pprof.Labels("path", "read", "type", "metric", "tenant", instanceID), func(c context.Context) { + var it iter.SampleIterator + it, err = instance.QuerySample(ctx, logql.SelectSampleParams{SampleQueryRequest: req}) if err != nil { - util.LogErrorWithContext(ctx, "closing iterator", it.Close) - return err + return + } + if sp != nil { + sp.LogKV("event", "finished instance query sample", "selector", req.Selector, "start", req.Start, "end", req.End) } - it = iter.NewMergeSampleIterator(ctx, []iter.SampleIterator{it, storeItr}) - } + if start, end, ok := buildStoreRequest(i.cfg, req.Start, req.End, time.Now()); ok { + storeReq := logql.SelectSampleParams{SampleQueryRequest: &logproto.SampleQueryRequest{ + Start: start, + End: end, + Selector: req.Selector, + Shards: req.Shards, + Deletes: req.Deletes, + Plan: req.Plan, + }} + var storeItr iter.SampleIterator + storeItr, err = i.store.SelectSamples(ctx, storeReq) + if err != nil { + util.LogErrorWithContext(ctx, "closing iterator", it.Close) + return + } + + it = iter.NewMergeSampleIterator(ctx, []iter.SampleIterator{it, storeItr}) + } + + defer util.LogErrorWithContext(ctx, "closing iterator", it.Close) - defer util.LogErrorWithContext(ctx, "closing iterator", it.Close) + err = sendSampleBatches(ctx, it, queryServer) + }) - return sendSampleBatches(ctx, it, queryServer) + return err } // asyncStoreMaxLookBack returns a max look back period only if active index type is one of async index stores like `boltdb-shipper` and `tsdb`. @@ -1039,24 +1061,27 @@ func (i *Ingester) GetChunkIDs(ctx context.Context, req *logproto.GetChunkIDsReq return nil, err } - // get chunk references - chunksGroups, _, err := i.store.GetChunks(ctx, orgID, start, end, chunk.NewPredicate(matchers, nil), nil) - if err != nil { - return nil, err - } + var resp logproto.GetChunkIDsResponse + pprof.Do(ctx, pprof.Labels("path", "read", "type", "chunkIDs", "tenant", orgID), func(c context.Context) { + // get chunk references + chunksGroups, _, err := i.store.GetChunks(ctx, orgID, start, end, chunk.NewPredicate(matchers, nil), nil) + if err != nil { + return + } - // todo (Callum) ingester should maybe store the whole schema config? - s := config.SchemaConfig{ - Configs: i.periodicConfigs, - } + // todo (Callum) ingester should maybe store the whole schema config? + s := config.SchemaConfig{ + Configs: i.periodicConfigs, + } - // build the response - resp := logproto.GetChunkIDsResponse{ChunkIDs: []string{}} - for _, chunks := range chunksGroups { - for _, chk := range chunks { - resp.ChunkIDs = append(resp.ChunkIDs, s.ExternalKey(chk.ChunkRef)) + // build the response + resp = logproto.GetChunkIDsResponse{ChunkIDs: []string{}} + for _, chunks := range chunksGroups { + for _, chk := range chunks { + resp.ChunkIDs = append(resp.ChunkIDs, s.ExternalKey(chk.ChunkRef)) + } } - } + }) return &resp, nil } @@ -1081,49 +1106,59 @@ func (i *Ingester) Label(ctx context.Context, req *logproto.LabelRequest) (*logp } } - resp, err := instance.Label(ctx, req, matchers...) - if err != nil { - return nil, err - } - - if req.Start == nil { - return resp, nil - } + var resp *logproto.LabelResponse + var storeValues []string + pprof.Do(ctx, pprof.Labels("path", "read", "type", "labels", "tenant", userID), func(c context.Context) { + resp, err = instance.Label(ctx, req, matchers...) + if err != nil { + return + } + if req.Start == nil { + return + } - // Only continue if the active index type is one of async index store types or QueryStore flag is true. - asyncStoreMaxLookBack := i.asyncStoreMaxLookBack() - if asyncStoreMaxLookBack == 0 && !i.cfg.QueryStore { - return resp, nil - } + // Only continue if the active index type is one of async index store types or QueryStore flag is true. + asyncStoreMaxLookBack := i.asyncStoreMaxLookBack() + if asyncStoreMaxLookBack == 0 && !i.cfg.QueryStore { + return + } - var cs storage.Store - var ok bool - if cs, ok = i.store.(storage.Store); !ok { - return resp, nil - } + var cs storage.Store + var ok bool + if cs, ok = i.store.(storage.Store); !ok { + return + } - maxLookBackPeriod := i.cfg.QueryStoreMaxLookBackPeriod - if asyncStoreMaxLookBack != 0 { - maxLookBackPeriod = asyncStoreMaxLookBack - } - // Adjust the start time based on QueryStoreMaxLookBackPeriod. - start := adjustQueryStartTime(maxLookBackPeriod, *req.Start, time.Now()) - if start.After(*req.End) { - // The request is older than we are allowed to query the store, just return what we have. - return resp, nil - } - from, through := model.TimeFromUnixNano(start.UnixNano()), model.TimeFromUnixNano(req.End.UnixNano()) - var storeValues []string - if req.Values { - storeValues, err = cs.LabelValuesForMetricName(ctx, userID, from, through, "logs", req.Name, matchers...) - if err != nil { - return nil, err + maxLookBackPeriod := i.cfg.QueryStoreMaxLookBackPeriod + if asyncStoreMaxLookBack != 0 { + maxLookBackPeriod = asyncStoreMaxLookBack } - } else { - storeValues, err = cs.LabelNamesForMetricName(ctx, userID, from, through, "logs") - if err != nil { - return nil, err + // Adjust the start time based on QueryStoreMaxLookBackPeriod. + start := adjustQueryStartTime(maxLookBackPeriod, *req.Start, time.Now()) + if start.After(*req.End) { + // The request is older than we are allowed to query the store, just return what we have. + return } + from, through := model.TimeFromUnixNano(start.UnixNano()), model.TimeFromUnixNano(req.End.UnixNano()) + + if req.Values { + storeValues, err = cs.LabelValuesForMetricName(ctx, userID, from, through, "logs", req.Name, matchers...) + if err != nil { + return + } + } else { + storeValues, err = cs.LabelNamesForMetricName(ctx, userID, from, through, "logs") + if err != nil { + return + } + } + }) + + // When wrapping the work above in the pprof.Do function we created a possible scenario where resp could + // be populated with values but an error occurred later on, prior to this profiling wrapper we would have + // always exited with a nil response and the error message, this is here to keep that behavior. + if err != nil { + return nil, err } return &logproto.LabelResponse{ @@ -1142,7 +1177,13 @@ func (i *Ingester) Series(ctx context.Context, req *logproto.SeriesRequest) (*lo if err != nil { return nil, err } - return instance.Series(ctx, req) + + var series *logproto.SeriesResponse + pprof.Do(ctx, pprof.Labels("path", "read", "type", "series", "tenant", instanceID), func(c context.Context) { + series, err = instance.Series(ctx, req) + }) + + return series, err } func (i *Ingester) GetStats(ctx context.Context, req *logproto.IndexStatsRequest) (*logproto.IndexStatsResponse, error) { @@ -1163,43 +1204,47 @@ func (i *Ingester) GetStats(ctx context.Context, req *logproto.IndexStatsRequest return nil, err } - type f func() (*logproto.IndexStatsResponse, error) - jobs := []f{ - f(func() (*logproto.IndexStatsResponse, error) { - return instance.GetStats(ctx, req) - }), - f(func() (*logproto.IndexStatsResponse, error) { - return i.store.Stats(ctx, user, req.From, req.Through, matchers...) - }), - } - resps := make([]*logproto.IndexStatsResponse, len(jobs)) - - if err := concurrency.ForEachJob( - ctx, - len(jobs), - 2, - func(_ context.Context, idx int) error { - res, err := jobs[idx]() - resps[idx] = res - return err - }, - ); err != nil { - return nil, err - } + var merged logproto.IndexStatsResponse + pprof.Do(ctx, pprof.Labels("path", "read", "type", "stats", "tenant", user), func(c context.Context) { - merged := index_stats.MergeStats(resps...) - if sp != nil { - sp.LogKV( - "user", user, - "from", req.From.Time(), - "through", req.Through.Time(), - "matchers", syntax.MatchersString(matchers), - "streams", merged.Streams, - "chunks", merged.Chunks, - "bytes", merged.Bytes, - "entries", merged.Entries, - ) - } + type f func() (*logproto.IndexStatsResponse, error) + jobs := []f{ + f(func() (*logproto.IndexStatsResponse, error) { + return instance.GetStats(ctx, req) + }), + f(func() (*logproto.IndexStatsResponse, error) { + return i.store.Stats(ctx, user, req.From, req.Through, matchers...) + }), + } + resps := make([]*logproto.IndexStatsResponse, len(jobs)) + + if err := concurrency.ForEachJob( + ctx, + len(jobs), + 2, + func(_ context.Context, idx int) error { + res, err := jobs[idx]() + resps[idx] = res + return err + }, + ); err != nil { + return + } + + merged = index_stats.MergeStats(resps...) + if sp != nil { + sp.LogKV( + "user", user, + "from", req.From.Time(), + "through", req.Through.Time(), + "matchers", syntax.MatchersString(matchers), + "streams", merged.Streams, + "chunks", merged.Chunks, + "bytes", merged.Bytes, + "entries", merged.Entries, + ) + } + }) return &merged, nil } @@ -1220,31 +1265,33 @@ func (i *Ingester) GetVolume(ctx context.Context, req *logproto.VolumeRequest) ( return nil, err } - type f func() (*logproto.VolumeResponse, error) - jobs := []f{ - f(func() (*logproto.VolumeResponse, error) { - return instance.GetVolume(ctx, req) - }), - f(func() (*logproto.VolumeResponse, error) { - return i.store.Volume(ctx, user, req.From, req.Through, req.Limit, req.TargetLabels, req.AggregateBy, matchers...) - }), - } - resps := make([]*logproto.VolumeResponse, len(jobs)) - - if err := concurrency.ForEachJob( - ctx, - len(jobs), - 2, - func(_ context.Context, idx int) error { - res, err := jobs[idx]() - resps[idx] = res - return err - }, - ); err != nil { - return nil, err - } - - merged := seriesvolume.Merge(resps, req.Limit) + var merged *logproto.VolumeResponse + pprof.Do(ctx, pprof.Labels("path", "read", "type", "volume", "tenant", user), func(c context.Context) { + type f func() (*logproto.VolumeResponse, error) + jobs := []f{ + f(func() (*logproto.VolumeResponse, error) { + return instance.GetVolume(ctx, req) + }), + f(func() (*logproto.VolumeResponse, error) { + return i.store.Volume(ctx, user, req.From, req.Through, req.Limit, req.TargetLabels, req.AggregateBy, matchers...) + }), + } + resps := make([]*logproto.VolumeResponse, len(jobs)) + + if err := concurrency.ForEachJob( + ctx, + len(jobs), + 2, + func(_ context.Context, idx int) error { + res, err := jobs[idx]() + resps[idx] = res + return err + }, + ); err != nil { + return + } + merged = seriesvolume.Merge(resps, req.Limit) + }) return merged, nil } @@ -1401,19 +1448,22 @@ func (i *Ingester) GetDetectedLabels(ctx context.Context, req *logproto.Detected } } - labelMap, err := instance.LabelsWithValues(ctx, req.Start, matchers...) + var result map[string]*logproto.UniqueLabelValues + pprof.Do(ctx, pprof.Labels("path", "read", "type", "detectedLabels", "tenant", userID), func(c context.Context) { + labelMap, err := instance.LabelsWithValues(ctx, req.Start, matchers...) + if err != nil { + return + } + result = make(map[string]*logproto.UniqueLabelValues) + for label, values := range labelMap { + var uniqueValues []string + for v := range values { + uniqueValues = append(uniqueValues, v) + } - if err != nil { - return nil, err - } - result := make(map[string]*logproto.UniqueLabelValues) - for label, values := range labelMap { - var uniqueValues []string - for v := range values { - uniqueValues = append(uniqueValues, v) + result[label] = &logproto.UniqueLabelValues{Values: uniqueValues} } + }) - result[label] = &logproto.UniqueLabelValues{Values: uniqueValues} - } return &logproto.LabelToValuesResponse{Labels: result}, nil } From 4f3ed77cb92c2ffd605743237e609c28f7841728 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Mon, 3 Jun 2024 11:02:15 +0200 Subject: [PATCH 41/41] fix(operator): Use a minimum value for replay memory ceiling (#13066) --- .../manifests/internal/config/build_test.go | 68 +++++++++---------- .../manifests/internal/config/options.go | 10 +++ 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go index fa20c45a0226..1d9a0287b398 100644 --- a/operator/internal/manifests/internal/config/build_test.go +++ b/operator/internal/manifests/internal/config/build_test.go @@ -69,7 +69,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 536870912 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -240,7 +240,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 0, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -327,7 +327,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -588,7 +588,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -673,7 +673,7 @@ func TestBuild_ConfigAndRuntimeConfig_CreateLokiConfigFailed(t *testing.T) { }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -754,7 +754,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -1026,7 +1026,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -1113,7 +1113,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -1386,7 +1386,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -1473,7 +1473,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -1776,7 +1776,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -1866,7 +1866,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -2112,7 +2112,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -2202,7 +2202,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -2535,7 +2535,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -2635,7 +2635,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -2881,7 +2881,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -2967,7 +2967,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -3377,7 +3377,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -3465,7 +3465,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -3638,7 +3638,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -3727,7 +3727,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -3901,7 +3901,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -3990,7 +3990,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -4162,7 +4162,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -4254,7 +4254,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -4460,7 +4460,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -4554,7 +4554,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -4760,7 +4760,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -4852,7 +4852,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -5019,7 +5019,7 @@ overrides: }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -5109,7 +5109,7 @@ func defaultOptions() Options { }, WriteAheadLog: WriteAheadLog{ Directory: "/tmp/wal", - IngesterMemoryRequest: 5000, + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, }, ObjectStorage: storage.Options{ SharedStore: lokiv1.ObjectStorageSecretS3, @@ -5343,7 +5343,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 @@ -5518,7 +5518,7 @@ ingester: wal: enabled: true dir: /tmp/wal - replay_memory_ceiling: 2500 + replay_memory_ceiling: 2147483648 ingester_client: grpc_client_config: max_recv_msg_size: 67108864 diff --git a/operator/internal/manifests/internal/config/options.go b/operator/internal/manifests/internal/config/options.go index 110bce9ee8df..e19d4c34f326 100644 --- a/operator/internal/manifests/internal/config/options.go +++ b/operator/internal/manifests/internal/config/options.go @@ -223,10 +223,20 @@ type WriteAheadLog struct { IngesterMemoryRequest int64 } +const ( + // minimumReplayCeiling contains the minimum value that will be used for the replay_memory_ceiling. + // It is set, so that even when the ingester has a low memory request, the replay will not flush each block + // on its own. + minimumReplayCeiling = 512 * 1024 * 1024 +) + // ReplayMemoryCeiling calculates 50% of the ingester memory // for the ingester to use for the write-ahead-log capbability. func (w WriteAheadLog) ReplayMemoryCeiling() string { value := int64(math.Ceil(float64(w.IngesterMemoryRequest) * float64(0.5))) + if value < minimumReplayCeiling { + value = minimumReplayCeiling + } return fmt.Sprintf("%d", value) }