diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index ed1b00fa202..589cbf98bf1 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -5791,6 +5791,7 @@ dependencies = [ "anyhow", "async-trait", "base64 0.21.5", + "hex", "once_cell", "prost", "quickwit-actors", diff --git a/quickwit/quickwit-opentelemetry/Cargo.toml b/quickwit/quickwit-opentelemetry/Cargo.toml index 84353ebb017..8ed34fcc7c5 100644 --- a/quickwit/quickwit-opentelemetry/Cargo.toml +++ b/quickwit/quickwit-opentelemetry/Cargo.toml @@ -13,6 +13,7 @@ documentation = "https://quickwit.io/docs/" anyhow = { workspace = true } async-trait = { workspace = true } base64 = { workspace = true } +hex = { workspace = true } once_cell = { workspace = true } prost = { workspace = true } serde = { workspace = true } diff --git a/quickwit/quickwit-opentelemetry/src/otlp/span_id.rs b/quickwit/quickwit-opentelemetry/src/otlp/span_id.rs index 55f2fb2c21f..0bd8b814552 100644 --- a/quickwit/quickwit-opentelemetry/src/otlp/span_id.rs +++ b/quickwit/quickwit-opentelemetry/src/otlp/span_id.rs @@ -17,15 +17,13 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use base64::prelude::BASE64_STANDARD; -use base64::Engine; use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] pub struct SpanId([u8; 8]); impl SpanId { - pub const BASE64_LENGTH: usize = 12; + pub const HEX_LENGTH: usize = 16; pub fn new(bytes: [u8; 8]) -> Self { Self(bytes) @@ -42,33 +40,29 @@ impl SpanId { impl Serialize for SpanId { fn serialize(&self, serializer: S) -> Result { - let b64span_id = BASE64_STANDARD.encode(self.0); - serializer.serialize_str(&b64span_id) + let hexspan_id = hex::encode(self.0); + serializer.serialize_str(&hexspan_id) } } impl<'de> Deserialize<'de> for SpanId { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de> { - let b64span_id = String::deserialize(deserializer)?; + let hexspan_id = String::deserialize(deserializer)?; - if b64span_id.len() != SpanId::BASE64_LENGTH { + if hexspan_id.len() != SpanId::HEX_LENGTH { let message = format!( - "base64 span ID must be {} bytes long, got {}", - SpanId::BASE64_LENGTH, - b64span_id.len() + "hex span ID must be {} bytes long, got {}", + SpanId::HEX_LENGTH, + hexspan_id.len() ); return Err(de::Error::custom(message)); } let mut span_id = [0u8; 8]; - BASE64_STANDARD - // Using the unchecked version here because otherwise the engine gets the wrong size - // estimate and fails. - .decode_slice_unchecked(b64span_id.as_bytes(), &mut span_id) - .map_err(|error| { - let message = format!("failed to decode base64 span ID: {:?}", error); - de::Error::custom(message) - })?; + hex::decode_to_slice(hexspan_id, &mut span_id).map_err(|error| { + let message = format!("failed to decode hex span ID: {:?}", error); + de::Error::custom(message) + })?; Ok(SpanId(span_id)) } } @@ -104,7 +98,7 @@ mod tests { fn test_span_id_serde() { let expected_span_id = SpanId::new([1; 8]); let span_id_json = serde_json::to_string(&expected_span_id).unwrap(); - assert_eq!(span_id_json, r#""AQEBAQEBAQE=""#); + assert_eq!(span_id_json, r#""0101010101010101""#); let span_id = serde_json::from_str::(&span_id_json).unwrap(); assert_eq!(span_id, expected_span_id,); diff --git a/quickwit/quickwit-opentelemetry/src/otlp/trace_id.rs b/quickwit/quickwit-opentelemetry/src/otlp/trace_id.rs index 6e30b64b295..9c14f8904d1 100644 --- a/quickwit/quickwit-opentelemetry/src/otlp/trace_id.rs +++ b/quickwit/quickwit-opentelemetry/src/otlp/trace_id.rs @@ -20,14 +20,13 @@ use base64::display::Base64Display; use base64::engine::GeneralPurpose; use base64::prelude::BASE64_STANDARD; -use base64::Engine; use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] pub struct TraceId([u8; 16]); impl TraceId { - pub const BASE64_LENGTH: usize = 24; + pub const HEX_LENGTH: usize = 32; pub fn new(bytes: [u8; 16]) -> Self { Self(bytes) @@ -49,8 +48,8 @@ impl TraceId { impl Serialize for TraceId { fn serialize(&self, serializer: S) -> Result { if serializer.is_human_readable() { - let b64trace_id = BASE64_STANDARD.encode(self.0); - serializer.serialize_str(&b64trace_id) + let hextrace_id = hex::encode(self.0); + serializer.serialize_str(&hextrace_id) } else { self.0.serialize(serializer) } @@ -61,24 +60,20 @@ impl<'de> Deserialize<'de> for TraceId { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de> { if deserializer.is_human_readable() { - let b64trace_id = String::deserialize(deserializer)?; - if b64trace_id.len() != TraceId::BASE64_LENGTH { + let hextrace_id = String::deserialize(deserializer)?; + if hextrace_id.len() != TraceId::HEX_LENGTH { let message = format!( - "base64 trace ID must be {} bytes long, got {}", - TraceId::BASE64_LENGTH, - b64trace_id.len() + "hex trace ID must be {} bytes long, got {}", + TraceId::HEX_LENGTH, + hextrace_id.len() ); return Err(de::Error::custom(message)); } let mut trace_id_bytes = [0u8; 16]; - BASE64_STANDARD - // Using the unchecked version here because otherwise the engine gets the wrong size - // estimate and fails. - .decode_slice_unchecked(b64trace_id.as_bytes(), &mut trace_id_bytes) - .map_err(|error| { - let message = format!("failed to decode base64 trace ID: {:?}", error); - de::Error::custom(message) - })?; + hex::decode_to_slice(hextrace_id, &mut trace_id_bytes).map_err(|error| { + let message = format!("failed to decode hex span ID: {:?}", error); + de::Error::custom(message) + })?; Ok(TraceId(trace_id_bytes)) } else { let trace_id_bytes: [u8; 16] = <[u8; 16]>::deserialize(deserializer)?; @@ -118,7 +113,7 @@ mod tests { fn test_trace_id_serde() { let expected_trace_id = TraceId::new([1; 16]); let trace_id_json = serde_json::to_string(&expected_trace_id).unwrap(); - assert_eq!(trace_id_json, r#""AQEBAQEBAQEBAQEBAQEBAQ==""#); + assert_eq!(trace_id_json, r#""01010101010101010101010101010101""#); let trace_id = serde_json::from_str::(&trace_id_json).unwrap(); assert_eq!(trace_id, expected_trace_id,); diff --git a/quickwit/quickwit-search/src/find_trace_ids_collector.rs b/quickwit/quickwit-search/src/find_trace_ids_collector.rs index 2b27c9b7381..0f6a8e4fd33 100644 --- a/quickwit/quickwit-search/src/find_trace_ids_collector.rs +++ b/quickwit/quickwit-search/src/find_trace_ids_collector.rs @@ -226,7 +226,7 @@ impl SegmentCollector for FindTraceIdsSegmentCollector { } fn harvest(self) -> Self::Fruit { - let mut buffer = Vec::with_capacity(TraceId::BASE64_LENGTH); + let mut buffer = Vec::with_capacity(TraceId::HEX_LENGTH); self.select_trace_ids .harvest() .into_iter() diff --git a/quickwit/quickwit-search/src/tests.rs b/quickwit/quickwit-search/src/tests.rs index ec559d2a4d5..3f282ec674d 100644 --- a/quickwit/quickwit-search/src/tests.rs +++ b/quickwit/quickwit-search/src/tests.rs @@ -479,7 +479,7 @@ async fn test_single_node_without_timestamp_with_query_start_timestamp_enabled( let start_timestamp = OffsetDateTime::now_utc().unix_timestamp(); for i in 0..30 { let body = format!("info @ t:{}", i + 1); - docs.push(json!({"body": body})); + docs.push(json!({ "body": body })); } test_sandbox.add_documents(docs).await?; @@ -1782,6 +1782,8 @@ async fn test_single_node_find_trace_ids_collector() { - name: trace_id type: bytes fast: true + input_format: hex + output_format: hex - name: span_timestamp_secs type: datetime fast: true