feat: add jaeger rest endpoints for grafana tracing support (#4197)

* feat: add jaeger rest endpoints for grafana tracing support * chore: minor enhancements, add a couple of tests * chore: use require helper method for optional jaeger parameter * chore: path renaming as per CR * chore: apply CR suggestions, small improvements * feat: add jaeger structs and helper methods * chore: add process, process_id handling logic * chore: add request param duration parsing logic * chore: minor style adjustments * chore: extract common logic into a method for traces * Review. * Make tests work. * chore: add tests * Refactor. * chore: add swagger docs * fix: remove go_parse_duration dependency, util.rs * chore: fix linting * Refactor parse duration for jaeger. * Clean. * chore: apply clippy suggestion * fix: add tags support * chore: apply clippy suggestions * chore: adjust the tag logic, add assertions for tags * Last commit before merging. * Remove collect_vec * Use bytes in jaeger models. --------- Co-authored-by: fmassot <[email protected]>
quickwit-oss · Dec 21, 2023 · 2074c2a · 2074c2a
1 parent fbe8dab
commit 2074c2a
Show file tree

Hide file tree

Showing 13 changed files with 1,366 additions and 13 deletions.
diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock
diff --git a/quickwit/quickwit-jaeger/src/lib.rs b/quickwit/quickwit-jaeger/src/lib.rs
@@ -72,6 +72,7 @@ type JaegerResult<T> = Result<T, Status>;
 
 type SpanStream = ReceiverStream<Result<SpansResponseChunk, Status>>;
 
+#[derive(Clone)]
 pub struct JaegerService {
     search_service: Arc<dyn SearchService>,
     lookback_period_secs: i64,

diff --git a/quickwit/quickwit-opentelemetry/src/otlp/traces.rs b/quickwit/quickwit-opentelemetry/src/otlp/traces.rs
@@ -724,7 +724,7 @@ impl OtlpGrpcTracesService {
             .inc_by(num_bytes);
 
         let response = ExportTraceServiceResponse {
-            // `rejected_spans=0` and `error_message=""` is consided a "full" success.
+            // `rejected_spans=0` and `error_message=""` is considered a "full" success.
             partial_success: Some(ExportTracePartialSuccess {
                 rejected_spans: num_parse_errors as i64,
                 error_message,

diff --git a/quickwit/quickwit-serve/Cargo.toml b/quickwit/quickwit-serve/Cargo.toml
@@ -12,11 +12,13 @@ documentation = "https://quickwit.io/docs/"
 [dependencies]
 anyhow = { workspace = true }
 async-trait = { workspace = true }
+base64 = { workspace = true }
 bytes = { workspace = true }
 bytesize = { workspace = true }
 elasticsearch-dsl = "0.4.15"
 futures = { workspace = true }
 futures-util = { workspace = true }
+hex = { workspace = true }
 humantime = { workspace = true }
 http-serde = { workspace = true }
 hyper = { workspace = true }
@@ -25,6 +27,7 @@ mime_guess = { workspace = true }
 num_cpus = { workspace = true }
 once_cell = { workspace = true }
 percent-encoding = { workspace = true }
+prost-types = { workspace = true }
 regex = { workspace = true }
 rust-embed = { workspace = true }
 serde = { workspace = true }

diff --git a/quickwit/quickwit-serve/resources/tests/jaeger_ui_trace.json b/quickwit/quickwit-serve/resources/tests/jaeger_ui_trace.json
@@ -0,0 +1,168 @@
+{
+    "traceID": "0000000000000001",
+    "spans": [
+      {
+        "traceID": "0000000000000001",
+        "spanID": "0000000000000001",
+        "operationName": "test-general-conversion",
+        "references": [],
+        "startTime": 1485467191639875,
+        "duration": 5,
+        "flags": 0,
+        "tags": [],
+        "logs": [
+          {
+            "timestamp": 1485467191639875,
+            "fields": [
+              {
+                "key": "event",
+                "type": "string",
+                "value": "some-event"
+              }
+            ]
+          },
+          {
+            "timestamp": 1485467191639875,
+            "fields": [
+              {
+                "key": "x",
+                "type": "string",
+                "value": "y"
+              }
+            ]
+          }
+        ],
+        "processID": "p1",
+        "warnings": []
+      },
+      {
+        "traceID": "0000000000000001",
+        "spanID": "0000000000000002",
+        "operationName": "some-operation",
+        "references": [],
+        "flags": 0,
+        "startTime": 1485467191639875,
+        "duration": 5,
+        "tags": [
+          {
+            "key": "peer.service",
+            "type": "string",
+            "value": "service-y"
+          },
+          {
+            "key": "peer.ipv4",
+            "type": "int64",
+            "value": 23456
+          },
+          {
+            "key": "error",
+            "type": "bool",
+            "value": true
+          },
+          {
+            "key": "temperature",
+            "type": "float64",
+            "value": 72.5
+          },
+          {
+            "key": "javascript_limit",
+            "type": "int64",
+            "value": "9223372036854775222"
+          },
+          {
+            "key": "blob",
+            "type": "binary",
+            "value": "AAAwOQ=="
+          }
+        ],
+        "logs": [],
+        "processID": "p1",
+        "warnings": []
+      },
+      {
+        "traceID": "0000000000000001",
+        "spanID": "0000000000000003",
+        "operationName": "some-operation",
+        "flags": 0,
+        "references": [
+          {
+            "refType": "CHILD_OF",
+            "traceID": "0000000000000001",
+            "spanID": "0000000000000002"
+          }
+        ],
+        "startTime": 1485467191639875,
+        "duration": 5,
+        "tags": [],
+        "logs": [],
+        "processID": "p2",
+        "warnings": []
+      },
+      {
+        "traceID": "0000000000000001",
+        "spanID": "0000000000000004",
+        "operationName": "reference-test",
+        "flags": 0,
+        "references": [
+          {
+            "refType": "CHILD_OF",
+            "traceID": "00000000000000ff",
+            "spanID": "00000000000000ff"
+          },
+          {
+            "refType": "CHILD_OF",
+            "traceID": "0000000000000001",
+            "spanID": "0000000000000002"
+          },
+          {
+            "refType": "FOLLOWS_FROM",
+            "traceID": "0000000000000001",
+            "spanID": "0000000000000002"
+          }
+        ],
+        "startTime": 1485467191639875,
+        "duration": 5,
+        "tags": [],
+        "logs": [],
+        "processID": "p2",
+        "warnings": [
+          "some span warning"
+        ]
+      },
+      {
+        "traceID": "0000000000000001",
+        "spanID": "0000000000000005",
+        "operationName": "preserveParentID-test",
+        "flags": 0,
+        "references": [
+          {
+            "refType": "CHILD_OF",
+            "traceID": "0000000000000001",
+            "spanID": "0000000000000004"
+          }
+        ],
+        "startTime": 1485467191639875,
+        "duration": 4,
+        "tags": [],
+        "logs": [],
+        "processID": "p2",
+        "warnings": [
+          "some span warning"
+        ]
+      }
+    ],
+    "processes": {
+      "p1": {
+        "serviceName": "service-x",
+        "key": "p1",
+        "tags": []
+      },
+      "p2": {
+        "serviceName": "service-y",
+        "key": "p2",
+        "tags": []
+      }
+    },
+    "warnings": [
+    ]
+  }
diff --git a/quickwit/quickwit-serve/src/grpc.rs b/quickwit/quickwit-serve/src/grpc.rs
@@ -23,7 +23,6 @@ use std::sync::Arc;
 
 use quickwit_common::tower::BoxFutureInfaillible;
 use quickwit_config::service::QuickwitService;
-use quickwit_jaeger::JaegerService;
 use quickwit_opentelemetry::otlp::{OtlpGrpcLogsService, OtlpGrpcTracesService};
 use quickwit_proto::indexing::IndexingServiceClient;
 use quickwit_proto::jaeger::storage::v1::span_reader_plugin_server::SpanReaderPluginServer;
@@ -153,18 +152,11 @@ pub(crate) async fn start_grpc_server(
     } else {
         None
     };
-    let enable_jaeger_endpoint = services.node_config.jaeger_config.enable_endpoint;
-    let jaeger_grpc_service = if enable_jaeger_endpoint
-        && services
-            .node_config
-            .is_service_enabled(QuickwitService::Searcher)
-    {
+
+    // Mount gRPC jaeger service if present.
+    let jaeger_grpc_service = if let Some(jaeger_service) = services.jaeger_service_opt.clone() {
         enabled_grpc_services.insert("jaeger");
-        let search_service = services.search_service.clone();
-        Some(SpanReaderPluginServer::new(JaegerService::new(
-            services.node_config.jaeger_config.clone(),
-            search_service,
-        )))
+        Some(SpanReaderPluginServer::new(jaeger_service))
     } else {
         None
     };

diff --git a/quickwit/quickwit-serve/src/jaeger_api/mod.rs b/quickwit/quickwit-serve/src/jaeger_api/mod.rs
@@ -0,0 +1,23 @@
+// Copyright (C) 2023 Quickwit, Inc.
+//
+// Quickwit is offered under the AGPL v3.0 and as commercial software.
+// For commercial licensing, contact us at [email protected].
+//
+// AGPL:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+mod model;
+mod parse_duration;
+mod rest_handler;
+pub(crate) use rest_handler::{jaeger_api_handlers, JaegerApi};