Add sort format for timestamp field to support sort and search values…

… in millis. (#4153)
quickwit-oss · Dec 11, 2023 · fea1b0e · fea1b0e
1 parent 12686fb
commit fea1b0e
Show file tree

Hide file tree

Showing 15 changed files with 814 additions and 96 deletions.
diff --git a/quickwit/quickwit-cli/src/index.rs b/quickwit/quickwit-cli/src/index.rs
@@ -994,6 +994,7 @@ pub async fn search_index(args: SearchIndexArgs) -> anyhow::Result<SearchRespons
             sort_fields: vec![SortField {
                 field_name: "_score".to_string(),
                 sort_order: SortOrder::Desc as i32,
+                sort_datetime_format: None,
             }],
         })
         .unwrap_or_default();

diff --git a/quickwit/quickwit-proto/protos/quickwit/search.proto b/quickwit/quickwit-proto/protos/quickwit/search.proto
@@ -185,6 +185,10 @@ enum CountHits {
 message SortField {
   string field_name = 1;
   SortOrder sort_order = 2;
+  // Optional sort value format for datetime field only.
+  // If none, the default output format for datetime field is
+  // unix_timestamp_nanos.
+  optional SortDatetimeFormat sort_datetime_format = 3;
 }
 
 enum SortOrder {
@@ -194,6 +198,13 @@ enum SortOrder {
   DESC = 1; //< This will be the default value;
 }
 
+// Sort value format for datetime field.
+// We keep an enum with only one format
+// for future extension.
+enum SortDatetimeFormat {
+  UNIX_TIMESTAMP_MILLIS = 0;
+}
+
 message SearchResponse {
   // Number of hits matching the query.
   uint64 num_hits = 1;

diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.search.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.search.rs
diff --git a/quickwit/quickwit-proto/src/search/mod.rs b/quickwit/quickwit-proto/src/search/mod.rs
@@ -104,7 +104,19 @@ impl SortByValue {
                     return None;
                 }
             }
-            String(_) | Array(_) | Object(_) => return None,
+            // Strings that can be converted to a number are accepted.
+            // Some clients (like JS clients) can't easily handle large integers
+            // without losing precision, so we accept them as strings.
+            String(value) => {
+                if let Ok(number) = value.parse::<i64>() {
+                    Some(SortValue::I64(number))
+                } else if let Ok(number) = value.parse::<u64>() {
+                    Some(SortValue::U64(number))
+                } else {
+                    return None;
+                }
+            }
+            Array(_) | Object(_) => return None,
         };
         Some(SortByValue { sort_value })
     }

diff --git a/quickwit/quickwit-search/src/collector.rs b/quickwit/quickwit-search/src/collector.rs
@@ -1308,11 +1308,13 @@ mod tests {
                         SortField {
                             field_name: field.to_string(),
                             sort_order: SortOrder::Asc.into(),
+                            sort_datetime_format: None,
                         }
                     } else {
                         SortField {
                             field_name: field.to_string(),
                             sort_order: SortOrder::Desc.into(),
+                            sort_datetime_format: None,
                         }
                     }
                 })
@@ -1538,10 +1540,12 @@ mod tests {
                     SortField {
                         field_name: "sort1".to_string(),
                         sort_order: SortOrder::Desc.into(),
+                        sort_datetime_format: None,
                     },
                     SortField {
                         field_name: "sort2".to_string(),
                         sort_order: SortOrder::Asc.into(),
+                        sort_datetime_format: None,
                     },
                 ],
                 search_after: Some(search_after),
@@ -1580,6 +1584,7 @@ mod tests {
                 sort_fields: vec![SortField {
                     field_name: "_shard_doc".to_string(),
                     sort_order: SortOrder::Desc.into(),
+                    sort_datetime_format: None,
                 }],
                 search_after: Some(search_after),
                 ..SearchRequest::default()
@@ -1660,6 +1665,7 @@ mod tests {
                 sort_fields: vec![SortField {
                     field_name: "timestamp".to_string(),
                     sort_order: SortOrder::Desc as i32,
+                    sort_datetime_format: None,
                 }],
                 aggregation_request: None,
                 ..Default::default()
@@ -1703,6 +1709,7 @@ mod tests {
                 sort_fields: vec![SortField {
                     field_name: "timestamp".to_string(),
                     sort_order: SortOrder::Desc as i32,
+                    sort_datetime_format: None,
                 }],
                 aggregation_request: None,
                 ..Default::default()
@@ -1788,6 +1795,7 @@ mod tests {
                 sort_fields: vec![SortField {
                     field_name: "timestamp".to_string(),
                     sort_order: SortOrder::Asc as i32,
+                    sort_datetime_format: None,
                 }],
                 aggregation_request: None,
                 ..Default::default()