Add the enable tracing opt-in flag (#4685)

## Which problem is this PR solving?
Resolves #4680

## Description of the changes
- Add an opt-in option `--query.enable-tracing` to enable tracing for
the jaeger-query component.
- The jaeger all-in-one component does not expose this flag since traces
are emitted to port 4317 by default, which all-in-one listens on.

## How was this change tested?

```
# Run jaeger-query component with tracing enabled and verify that the connection errors are appearing in stdout.
$ SPAN_STORAGE_TYPE=memory go run -tags ui ./cmd/query/main.go --query.enable-tracing
...
{"level":"info","ts":1692363754.9049716,"caller":"grpc/clientconn.go:1301","msg":"[core][Channel #1 SubChannel #2] Subchannel Connectivity change to CONNECTING","system":"grpc","grpc_log":true}
{"level":"info","ts":1692363754.9050152,"caller":"grpc/clientconn.go:1414","msg":"[core][Channel #1 SubChannel #2] Subchannel picks a new address \"localhost:4317\" to connect","system":"grpc","grpc_log":true}
{"level":"warn","ts":1692363754.9058733,"caller":"grpc/clientconn.go:1476","msg":"[core][Channel #1 SubChannel #2] grpc: addrConn.createTransport failed to connect to {Addr: \"localhost:4317\", ServerName: \"localhost:4317\", }. Err: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:4317: connect: connection refused\"","system":"grpc","grpc_log":true}
{"level":"info","ts":1692363754.9067123,"caller":"grpc/clientconn.go:1303","msg":"[core][Channel #1 SubChannel #2] Subchannel Connectivity change to TRANSIENT_FAILURE, last error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:4317: connect: connection refused\"","system":"grpc","grpc_log":true}
...

# Run jaeger-query component with tracing disabled and verify that the connection errors no longer appear.
# Of course, we can't see traces in Jaeger UI because there's nothing to receive the traces.
$ SPAN_STORAGE_TYPE=memory go run -tags ui ./cmd/query/main.go

# Start an all-in-one instance just as a quick and dirty way to bring up an in-memory jaeger stack to
# receive traces from jaeger-query
$ make run-all-in-one

# Run jaeger-query as a separate component, listening on different ports to all-in-one to avoid port binding collisions.
$ SPAN_STORAGE_TYPE=memory go run -tags ui ./cmd/query/main.go --query.enable-tracing --query.grpc-server.host-port :17685 --query.http-server.host-port :17686 --admin.http.host-port :17687

# Open localhost:17686 in a browser and refresh a few times to emit traces to jaeger all-in-one.
```

Confirmed that `jaeger-query` is visible and contains traces:

<img width="1572" alt="Screenshot 2023-08-18 at 11 45 26 pm"
src="https://github.com/jaegertracing/jaeger/assets/26584478/a348e804-6d37-49f9-9d9f-f73854e9b6bc">


## Checklist
- [x] I have read
https://github.com/jaegertracing/jaeger/blob/master/CONTRIBUTING_GUIDELINES.md
- [x] I have signed all commits
~- [] I have added unit tests for the new functionality~
- [x] I have run lint and test steps successfully
  - for `jaeger`: `make lint test`
  - for `jaeger-ui`: `yarn lint` and `yarn test`

---------

Signed-off-by: albertteoh <[email protected]>

cmd/all-in-one/main.go

@@ -24,7 +24,6 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"go.opentelemetry.io/otel"
 	_ "go.uber.org/automaxprocs"
 	"go.uber.org/zap"
 
@@ -105,7 +104,6 @@ by default uses only in-memory database.`,
 			if err != nil {
 				logger.Fatal("Failed to initialize tracer", zap.Error(err))
 			}
-			otel.SetTracerProvider(tracer.OTEL)
 
 			storageFactory.InitFromViper(v, logger)
 			if err := storageFactory.Initialize(metricsFactory, logger); err != nil {

cmd/query/app/flags.go

@@ -47,6 +47,7 @@ const (
 	queryTokenPropagation      = "query.bearer-token-propagation"
 	queryAdditionalHeaders     = "query.additional-headers"
 	queryMaxClockSkewAdjust    = "query.max-clock-skew-adjustment"
+	queryEnableTracing         = "query.enable-tracing"
 )
 
 var tlsGRPCFlagsConfig = tlscfg.ServerFlagsConfig{
@@ -85,6 +86,8 @@ type QueryOptions struct {
 	MaxClockSkewAdjust time.Duration
 	// Tenancy configures tenancy for query
 	Tenancy tenancy.Options
+	// EnableTracing determines whether traces will be emitted by jaeger-query.
+	EnableTracing bool
 }
 
 // AddFlags adds flags for QueryOptions
@@ -98,6 +101,7 @@ func AddFlags(flagSet *flag.FlagSet) {
 	flagSet.String(queryUIConfig, "", "The path to the UI configuration file in JSON format")
 	flagSet.Bool(queryTokenPropagation, false, "Allow propagation of bearer token to be used by storage plugins")
 	flagSet.Duration(queryMaxClockSkewAdjust, 0, "The maximum delta by which span timestamps may be adjusted in the UI due to clock skew; set to 0s to disable clock skew adjustments")
+	flagSet.Bool(queryEnableTracing, false, "Enables emitting jaeger-query traces")
 	tlsGRPCFlagsConfig.AddFlags(flagSet)
 	tlsHTTPFlagsConfig.AddFlags(flagSet)
 }
@@ -131,6 +135,7 @@ func (qOpts *QueryOptions) InitFromViper(v *viper.Viper, logger *zap.Logger) (*Q
 		qOpts.AdditionalHeaders = headers
 	}
 	qOpts.Tenancy = tenancy.InitFromViper(v)
+	qOpts.EnableTracing = v.GetBool(queryEnableTracing)
 	return qOpts, nil
 }
 

cmd/query/main.go

@@ -23,7 +23,6 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"go.opentelemetry.io/otel"
 	_ "go.uber.org/automaxprocs"
 	"go.uber.org/zap"
 
@@ -73,15 +72,20 @@ func main() {
 			baseFactory := svc.MetricsFactory.Namespace(metrics.NSOptions{Name: "jaeger"})
 			metricsFactory := baseFactory.Namespace(metrics.NSOptions{Name: "query"})
 			version.NewInfoMetrics(metricsFactory)
-			jtracer, err := jtracer.New("jaeger-query")
-			if err != nil {
-				logger.Fatal("Failed to create tracer:", zap.Error(err))
-			}
-			otel.SetTracerProvider(jtracer.OTEL)
+
 			queryOpts, err := new(app.QueryOptions).InitFromViper(v, logger)
 			if err != nil {
 				logger.Fatal("Failed to configure query service", zap.Error(err))
 			}
+
+			jt := jtracer.NoOp()
+			if queryOpts.EnableTracing {
+				jt, err = jtracer.New("jaeger-query")
+				if err != nil {
+					logger.Fatal("Failed to create tracer", zap.Error(err))
+				}
+			}
+
 			// TODO: Need to figure out set enable/disable propagation on storage plugins.
 			v.Set(bearertoken.StoragePropagationKey, queryOpts.BearerTokenPropagation)
 			storageFactory.InitFromViper(v, logger)
@@ -108,7 +112,7 @@ func main() {
 				dependencyReader,
 				*queryServiceOptions)
 			tm := tenancy.NewManager(&queryOpts.Tenancy)
-			server, err := app.NewServer(svc.Logger, queryService, metricsQueryService, queryOpts, tm, jtracer)
+			server, err := app.NewServer(svc.Logger, queryService, metricsQueryService, queryOpts, tm, jt)
 			if err != nil {
 				logger.Fatal("Failed to create server", zap.Error(err))
 			}
@@ -128,7 +132,7 @@ func main() {
 				if err := storageFactory.Close(); err != nil {
 					logger.Error("Failed to close storage factory", zap.Error(err))
 				}
-				if err = jtracer.Close(context.Background()); err != nil {
+				if err = jt.Close(context.Background()); err != nil {
 					logger.Fatal("Error shutting down tracer provider", zap.Error(err))
 				}
 			})

examples/memstore-plugin/main.go

@@ -23,7 +23,6 @@ import (
 	"github.com/hashicorp/go-plugin"
 	"github.com/spf13/viper"
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
-	"go.opentelemetry.io/otel"
 	googleGRPC "google.golang.org/grpc"
 
 	"github.com/jaegertracing/jaeger/pkg/jtracer"
@@ -62,7 +61,6 @@ func main() {
 		panic(fmt.Errorf("failed to initialize tracer: %w", err))
 	}
 	defer tracer.Close(context.Background())
-	otel.SetTracerProvider(tracer.OTEL)
 
 	memStorePlugin := grpcMemory.NewStoragePlugin(memory.NewStore(), memory.NewStore())
 	service := &shared.PluginServices{

pkg/jtracer/jtracer.go

@@ -88,6 +88,8 @@ func initOTEL(ctx context.Context, svc string) (*sdktrace.TracerProvider, error)
 			))
 	})
 
+	otel.SetTracerProvider(tracerProvider)
+
 	return tracerProvider, nil
 }