From 5a643c711547e0d5bfc5b109e8b5650d195b152a Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Fri, 3 May 2024 02:06:51 -0400 Subject: [PATCH] chore: reduces span footprint + double recording (#12864) Trying to remove some cruft from traces. --- pkg/bloomgateway/querier.go | 11 ++++++----- pkg/logql/engine.go | 1 - pkg/querier/handler.go | 4 ---- pkg/querier/queryrange/downstreamer.go | 1 - pkg/querier/queryrange/limits.go | 2 -- pkg/querier/queryrange/shard_resolver.go | 8 +++----- pkg/querier/worker/scheduler_processor.go | 10 ---------- 7 files changed, 9 insertions(+), 28 deletions(-) diff --git a/pkg/bloomgateway/querier.go b/pkg/bloomgateway/querier.go index 58415a1fd667..0e523817dc70 100644 --- a/pkg/bloomgateway/querier.go +++ b/pkg/bloomgateway/querier.go @@ -5,7 +5,6 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" - "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -16,6 +15,7 @@ import ( v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper" "github.com/grafana/loki/v3/pkg/util/constants" + "github.com/grafana/loki/v3/pkg/util/spanlogger" ) type querierMetrics struct { @@ -90,8 +90,9 @@ func (bq *BloomQuerier) FilterChunkRefs(ctx context.Context, tenant string, from if !bq.limits.BloomGatewayEnabled(tenant) || len(chunkRefs) == 0 || len(v1.ExtractTestableLineFilters(queryPlan.AST)) == 0 { return chunkRefs, nil } - sp, ctx := opentracing.StartSpanFromContext(ctx, "bloomquerier.FilterChunkRefs") - defer sp.Finish() + + logger, ctx := spanlogger.NewWithLogger(ctx, bq.logger, "bloomquerier.FilterChunkRefs") + defer logger.Finish() grouped := groupedChunksRefPool.Get(len(chunkRefs)) defer groupedChunksRefPool.Put(grouped) @@ -141,8 +142,7 @@ func (bq *BloomQuerier) FilterChunkRefs(ctx context.Context, tenant string, from postFilterChunks := len(result) postFilterSeries := len(deduped) - level.Debug(bq.logger).Log( - "operation", "bloomquerier.FilterChunkRefs", + level.Debug(logger).Log( "tenant", tenant, "from", from.Time(), "through", through.Time(), @@ -153,6 +153,7 @@ func (bq *BloomQuerier) FilterChunkRefs(ctx context.Context, tenant string, from "preFilterSeries", preFilterSeries, "postFilterSeries", postFilterSeries, "filteredSeries", preFilterSeries-postFilterSeries, + "operation", "bloomquerier.FilterChunkRefs", ) bq.metrics.chunksTotal.Add(float64(preFilterChunks)) diff --git a/pkg/logql/engine.go b/pkg/logql/engine.go index eabbf1c43323..a25a726bb302 100644 --- a/pkg/logql/engine.go +++ b/pkg/logql/engine.go @@ -232,7 +232,6 @@ func (q *query) Exec(ctx context.Context) (logqlmodel.Result, error) { sp, ctx := opentracing.StartSpanFromContext(ctx, "query.Exec") defer sp.Finish() spLogger := spanlogger.FromContext(ctx) - defer spLogger.Finish() sp.LogKV( "type", GetRangeType(q.params), diff --git a/pkg/querier/handler.go b/pkg/querier/handler.go index ee5d0648ac27..bb01a0e3754b 100644 --- a/pkg/querier/handler.go +++ b/pkg/querier/handler.go @@ -5,8 +5,6 @@ import ( "fmt" "net/http" - "github.com/opentracing/opentracing-go" - "github.com/grafana/loki/v3/pkg/loghttp" "github.com/grafana/loki/v3/pkg/logproto" "github.com/grafana/loki/v3/pkg/querier/queryrange" @@ -24,8 +22,6 @@ func NewQuerierHandler(api *QuerierAPI) *Handler { } func (h *Handler) Do(ctx context.Context, req queryrangebase.Request) (queryrangebase.Response, error) { - span, ctx := opentracing.StartSpanFromContext(ctx, "queryHandler") - defer span.Finish() switch concrete := req.(type) { case *queryrange.LokiRequest: diff --git a/pkg/querier/queryrange/downstreamer.go b/pkg/querier/queryrange/downstreamer.go index 7103a38ab376..cf1cfc36dc4a 100644 --- a/pkg/querier/queryrange/downstreamer.go +++ b/pkg/querier/queryrange/downstreamer.go @@ -143,7 +143,6 @@ func (in instance) Downstream(ctx context.Context, queries []logql.DownstreamQue sp, ctx := opentracing.StartSpanFromContext(ctx, "DownstreamHandler.instance") defer sp.Finish() logger := spanlogger.FromContext(ctx) - defer logger.Finish() level.Debug(logger).Log("shards", fmt.Sprintf("%+v", qry.Params.Shards()), "query", req.GetQuery(), "step", req.GetStep(), "handler", reflect.TypeOf(in.handler), "engine", "downstream") res, err := in.handler.Do(ctx, req) diff --git a/pkg/querier/queryrange/limits.go b/pkg/querier/queryrange/limits.go index b557663cf246..5d63402fc9f3 100644 --- a/pkg/querier/queryrange/limits.go +++ b/pkg/querier/queryrange/limits.go @@ -340,8 +340,6 @@ func (q *querySizeLimiter) guessLimitName() string { } func (q *querySizeLimiter) Do(ctx context.Context, r queryrangebase.Request) (queryrangebase.Response, error) { - span, ctx := opentracing.StartSpanFromContext(ctx, "query_size_limits") - defer span.Finish() log := spanlogger.FromContext(ctx) defer log.Finish() diff --git a/pkg/querier/queryrange/shard_resolver.go b/pkg/querier/queryrange/shard_resolver.go index 9950190ae0eb..c8a66de69091 100644 --- a/pkg/querier/queryrange/shard_resolver.go +++ b/pkg/querier/queryrange/shard_resolver.go @@ -221,9 +221,7 @@ func (r *dynamicShardResolver) ShardingRanges(expr syntax.Expr, targetBytesPerSh []logproto.ChunkRefGroup, error, ) { - sp, ctx := opentracing.StartSpanFromContext(r.ctx, "dynamicShardResolver.ShardingRanges") - defer sp.Finish() - log := spanlogger.FromContext(ctx) + log := spanlogger.FromContext(r.ctx) defer log.Finish() adjustedFrom := r.from @@ -254,7 +252,7 @@ func (r *dynamicShardResolver) ShardingRanges(expr syntax.Expr, targetBytesPerSh exprStr := expr.String() // try to get shards for the given expression // if it fails, fallback to linearshards based on stats - resp, err := r.next.Do(ctx, &logproto.ShardsRequest{ + resp, err := r.next.Do(r.ctx, &logproto.ShardsRequest{ From: adjustedFrom, Through: r.through, Query: expr.String(), @@ -271,7 +269,7 @@ func (r *dynamicShardResolver) ShardingRanges(expr syntax.Expr, targetBytesPerSh } // accumulate stats - logqlstats.JoinResults(ctx, casted.Response.Statistics) + logqlstats.JoinResults(r.ctx, casted.Response.Statistics) var refs int for _, x := range casted.Response.ChunkGroups { diff --git a/pkg/querier/worker/scheduler_processor.go b/pkg/querier/worker/scheduler_processor.go index 00b08219e5db..a19975486ef3 100644 --- a/pkg/querier/worker/scheduler_processor.go +++ b/pkg/querier/worker/scheduler_processor.go @@ -29,7 +29,6 @@ import ( "github.com/grafana/loki/v3/pkg/querier/queryrange" querier_stats "github.com/grafana/loki/v3/pkg/querier/stats" "github.com/grafana/loki/v3/pkg/scheduler/schedulerpb" - httpgrpcutil "github.com/grafana/loki/v3/pkg/util/httpgrpc" util_log "github.com/grafana/loki/v3/pkg/util/log" ) @@ -147,15 +146,6 @@ func (sp *schedulerProcessor) querierLoop(c schedulerpb.SchedulerForQuerier_Quer ctx := user.InjectOrgID(ctx, request.UserID) sp.metrics.inflightRequests.Inc() - tracer := opentracing.GlobalTracer() - // Ignore errors here. If we cannot get parent span, we just don't create new one. - parentSpanContext, _ := httpgrpcutil.GetParentSpanForRequest(tracer, request) - if parentSpanContext != nil { - queueSpan, spanCtx := opentracing.StartSpanFromContextWithTracer(ctx, tracer, "querier_processor_runRequest", opentracing.ChildOf(parentSpanContext)) - defer queueSpan.Finish() - - ctx = spanCtx - } logger := util_log.WithContext(ctx, sp.log) switch r := request.Request.(type) {