remove unused deps (#4819)

and fix some tests/bench compilation issues

quickwit/Cargo.toml

@@ -85,7 +85,6 @@ assert-json-diff = "2"
 async-compression = { version = "0.4", features = ["tokio", "gzip"] }
 async-speed-limit = "0.4"
 async-trait = "0.1"
-backoff = { version = "0.4", features = ["tokio"] }
 base64 = "0.22"
 bytes = { version = "1", features = ["serde"] }
 bytesize = { version = "1.3.0", features = ["serde"] }
@@ -113,7 +112,6 @@ flate2 = "1.0"
 futures = "0.3"
 futures-util = { version = "0.3.25", default-features = false }
 google-cloud-auth = "0.12.0"
-google-cloud-default = { version = "0.3.0", features = ["pubsub"] }
 google-cloud-gax = "0.15.0"
 google-cloud-googleapis = { version = "0.10.0", features = ["pubsub"] }
 google-cloud-pubsub = "0.18.0"
@@ -123,7 +121,6 @@ home = "0.5.4"
 hostname = "0.3"
 http = "0.2.9"
 http-serde = "1.1.2"
-humansize = "2.1.3"
 humantime = "2.1.0"
 hyper = { version = "0.14", features = [
   "client",
@@ -227,7 +224,6 @@ syn = { version = "2.0.11", features = ["extra-traits", "full", "parsing"] }
 sync_wrapper = "0.1.2"
 tabled = { version = "0.14", features = ["color"] }
 tempfile = "3"
-termcolor = "1"
 thiserror = "1"
 thousands = "0.2.0"
 tikv-jemalloc-ctl = "0.5"
@@ -280,7 +276,6 @@ aws-sdk-s3 = "0.28.0"
 aws-smithy-async = "0.55.0"
 aws-smithy-client = "0.55.0"
 aws-smithy-http = "0.55.0"
-aws-smithy-types = "0.55.0"
 aws-types = "0.55.0"
 
 azure_core = { version = "0.13.0", features = ["enable_reqwest_rustls"] }

quickwit/quickwit-aws/Cargo.toml

@@ -18,15 +18,10 @@ aws-smithy-async = { workspace = true }
 aws-smithy-client = { workspace = true }
 aws-types = { workspace = true }
 
-async-trait = { workspace = true }
 futures = { workspace = true }
 hyper = { workspace = true }
 hyper-rustls = { workspace = true }
-once_cell = { workspace = true }
-rand = { workspace = true }
 tokio = { workspace = true }
-tower = { workspace = true }
-tracing = { workspace = true }
 
 quickwit-common = { workspace = true }
 

quickwit/quickwit-cli/Cargo.toml

@@ -22,8 +22,6 @@ path = "src/generate_markdown.rs"
 
 [dependencies]
 anyhow = { workspace = true }
-async-trait = { workspace = true }
-bytes = { workspace = true }
 bytesize = { workspace = true }
 chrono = { workspace = true }
 clap = { workspace = true }
@@ -39,7 +37,6 @@ once_cell = { workspace = true }
 openssl-probe = { workspace = true, optional = true }
 opentelemetry = { workspace = true }
 opentelemetry-otlp = { workspace = true }
-regex = { workspace = true }
 reqwest = { workspace = true }
 serde_json = { workspace = true }
 tabled = { workspace = true }
@@ -50,9 +47,7 @@ tikv-jemalloc-ctl = { workspace = true, optional = true }
 tikv-jemallocator = { workspace = true, optional = true }
 time = { workspace = true }
 tokio = { workspace = true }
-tokio-util = { workspace = true }
 toml = { workspace = true }
-tonic = { workspace = true }
 tracing = { workspace = true }
 tracing-opentelemetry = { workspace = true }
 tracing-subscriber = { workspace = true }
@@ -61,7 +56,6 @@ quickwit-actors = { workspace = true }
 quickwit-cluster = { workspace = true }
 quickwit-common = { workspace = true }
 quickwit-config = { workspace = true }
-quickwit-directories = { workspace = true }
 quickwit-doc-mapper = { workspace = true }
 quickwit-index-management = { workspace = true }
 quickwit-indexing = { workspace = true }
@@ -126,3 +120,7 @@ release-macos-feature-vendored-set = [
   "quickwit-metastore/postgres",
   "quickwit-doc-mapper/multilang",
 ]
+
+[package.metadata.cargo-machete]
+# used to enable the `multilang` feature
+ignored = ["quickwit-doc-mapper"]

quickwit/quickwit-cluster/Cargo.toml

@@ -22,13 +22,11 @@ pin-project = { workspace = true }
 rand = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
-thiserror = { workspace = true }
 time = { workspace = true }
 tokio = { workspace = true }
 tokio-stream = { workspace = true }
 tonic = { workspace = true }
 tracing = { workspace = true }
-ulid = { workspace = true }
 utoipa = { workspace = true }
 
 quickwit-common = { workspace = true }
@@ -40,8 +38,11 @@ testsuite = []
 
 [dev-dependencies]
 rand = { workspace = true }
-tempfile = { workspace = true }
 
 chitchat = { workspace = true, features = ["testsuite"] }
 quickwit-common = { workspace = true, features = ["testsuite"] }
 quickwit-proto = { workspace = true, features = ["testsuite"] }
+
+[package.metadata.cargo-machete]
+# used inside code generated by utoipa
+ignored = ["serde_json"]

quickwit/quickwit-codegen/Cargo.toml

@@ -15,7 +15,6 @@ anyhow = { workspace = true }
 heck = { workspace = true }
 prettyplease = { workspace = true }
 proc-macro2 = { workspace = true }
-prost = { workspace = true }
 prost-build = { workspace = true }
 quote = { workspace = true }
 syn = { workspace = true }

quickwit/quickwit-codegen/example/Cargo.toml

@@ -17,7 +17,6 @@ bytesize = { workspace = true }
 dyn-clone = { workspace = true }
 futures = { workspace = true }
 http = { workspace = true }
-hyper = { workspace = true }
 prost = { workspace = true }
 serde = { workspace = true }
 thiserror = { workspace = true }
@@ -29,7 +28,6 @@ utoipa = { workspace = true }
 
 quickwit-actors = { workspace = true }
 quickwit-common = { workspace = true }
-quickwit-macros = { workspace = true }
 quickwit-proto ={ workspace = true }
 
 [dev-dependencies]

quickwit/quickwit-common/Cargo.toml

@@ -49,5 +49,4 @@ named_tasks = ["tokio/tracing"]
 [dev-dependencies]
 serde_json = { workspace = true }
 tempfile = { workspace = true }
-quickwit-macros = { workspace = true }
 proptest = { workspace = true }

quickwit/quickwit-control-plane/Cargo.toml

@@ -13,23 +13,16 @@ license.workspace = true
 [dependencies]
 anyhow = { workspace = true }
 async-trait = { workspace = true }
-dyn-clone = { workspace = true }
 fnv = { workspace = true }
 futures = { workspace = true }
-http = { workspace = true }
-hyper = { workspace = true }
 itertools = { workspace = true }
 mockall = { workspace = true, optional = true }
 once_cell = { workspace = true }
-prost = { workspace = true }
 rand = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
-thiserror = { workspace = true }
 time = { workspace = true }
 tokio = { workspace = true }
-tokio-stream = { workspace = true }
-tonic = { workspace = true }
 tower = { workspace = true }
 tracing = { workspace = true }
 ulid = { workspace = true }
@@ -55,7 +48,6 @@ quickwit-config = { workspace = true, features = ["testsuite"] }
 quickwit-indexing = { workspace = true }
 quickwit-metastore = { workspace = true, features = ["testsuite"] }
 quickwit-proto = { workspace = true, features = ["testsuite"] }
-quickwit-storage = { workspace = true, features = ["testsuite"] }
 
 [features]
 testsuite = ["mockall"]

quickwit/quickwit-directories/Cargo.toml

@@ -13,13 +13,9 @@ license.workspace = true
 [dependencies]
 anyhow = { workspace = true }
 async-trait = { workspace = true }
-futures = { workspace = true }
-once_cell = { workspace = true }
 postcard = { workspace = true }
 serde = { workspace = true }
-serde_json = { workspace = true }
 tantivy = { workspace = true }
-thiserror = { workspace = true }
 time = { workspace = true }
 tokio = { workspace = true }
 tracing = { workspace = true }

quickwit/quickwit-doc-mapper/Cargo.toml

@@ -18,7 +18,6 @@ fnv = { workspace = true }
 hex = { workspace = true }
 indexmap = { workspace = true }
 itertools = { workspace = true }
-mockall = { workspace = true, optional = true }
 nom = { workspace = true }
 once_cell = { workspace = true }
 regex = { workspace = true }
@@ -39,11 +38,9 @@ quickwit-query = { workspace = true }
 [dev-dependencies]
 criterion = { workspace = true }
 matches = { workspace = true }
-proptest = { workspace = true }
 serde_yaml = { workspace = true }
 time = { workspace = true }
 
-quickwit-proto = { workspace = true }
 quickwit-query = { workspace = true, features = ["multilang"] }
 
 [features]

quickwit/quickwit-index-management/Cargo.toml

@@ -12,34 +12,22 @@ license.workspace = true
 
 [dependencies]
 anyhow = { workspace = true }
-async-trait = { workspace = true }
 futures = { workspace = true }
 futures-util = { workspace = true }
 itertools = { workspace = true }
-rand = { workspace = true }
-serde = { workspace = true }
-serde_json =  { workspace = true }
-tantivy = { workspace = true }
-tempfile = { workspace = true }
 thiserror = { workspace = true }
 time = { workspace = true }
 tokio = { workspace = true }
-tokio-stream = { workspace = true }
-tokio-util = { workspace = true }
 tracing = { workspace = true }
 
 quickwit-common = { workspace = true }
 quickwit-config = { workspace = true }
-quickwit-directories = { workspace = true }
-quickwit-doc-mapper = { workspace = true }
 quickwit-indexing = { workspace = true }
 quickwit-metastore = { workspace = true }
 quickwit-proto = { workspace = true }
 quickwit-storage = { workspace = true }
 
 [dev-dependencies]
-mockall = { workspace = true }
-serde_yaml = { workspace = true }
 
 quickwit-common = { workspace = true, features = ["testsuite"] }
 quickwit-metastore = { workspace = true, features = ["testsuite"] }

quickwit/quickwit-indexing/Cargo.toml

@@ -11,23 +11,19 @@ authors.workspace = true
 license.workspace = true
 
 [dependencies]
-aws-config = { workspace = true, optional = true }
 aws-sdk-kinesis = { workspace = true, optional = true }
-aws-smithy-client = { workspace = true, optional = true }
 
 anyhow = { workspace = true }
 arc-swap = { workspace = true }
 async-compression = { workspace = true }
 async-trait = { workspace = true }
-backoff = { workspace = true, optional = true }
 bytes = { workspace = true }
 bytesize = { workspace = true }
 fail = { workspace = true }
 flume = { workspace = true }
 fnv = { workspace = true }
 futures = { workspace = true }
 google-cloud-auth = { workspace = true, optional = true }
-google-cloud-default = { workspace = true, optional = true }
 google-cloud-gax = { workspace = true, optional = true }
 google-cloud-googleapis = { workspace = true, optional = true }
 google-cloud-pubsub = { workspace = true, optional = true }
@@ -46,12 +42,10 @@ tempfile = { workspace = true }
 thiserror = { workspace = true }
 time = { workspace = true }
 tokio = { workspace = true }
-tokio-stream = { workspace = true }
 tracing = { workspace = true }
 ulid = { workspace = true }
 utoipa = { workspace = true }
 vrl = { workspace = true, optional = true }
-zstd = { workspace = true }
 
 quickwit-actors = { workspace = true }
 quickwit-aws = { workspace = true }
@@ -69,17 +63,14 @@ quickwit-storage = { workspace = true }
 [features]
 gcp-pubsub = [
   "dep:google-cloud-auth",
-  "dep:google-cloud-default",
   "dep:google-cloud-gax",
   "dep:google-cloud-googleapis",
   "dep:google-cloud-pubsub",
 ]
 gcp-pubsub-emulator-tests = []
-kafka = ["rdkafka", "backoff"]
+kafka = ["rdkafka"]
 kafka-broker-tests = []
 kinesis = [
-  "aws-config",
-  "aws-smithy-client",
   "aws-sdk-kinesis",
   "quickwit-aws/kinesis",
 ]
@@ -97,6 +88,9 @@ testsuite = [
   "quickwit-actors/testsuite",
   "quickwit-cluster/testsuite",
   "quickwit-common/testsuite",
+  "quickwit-config/testsuite",
+  "quickwit-proto/testsuite",
+  "quickwit-storage/testsuite"
 ]
 vrl = ["dep:vrl", "quickwit-config/vrl"]
 
@@ -129,3 +123,7 @@ required-features = ["fail/failpoints"]
 [[bench]]
 name = "doc_process_vrl_bench"
 harness = false
+
+[package.metadata.cargo-machete]
+# used to vendor/static build native dependancies
+ignored = ["libz-sys", "openssl"]

quickwit/quickwit-indexing/benches/doc_process_vrl_bench.rs

@@ -32,11 +32,7 @@ macro_rules! bench_func {
                 |docs| async {
                     let (mailbox, handle, universe) = $func;
                     mailbox
-                        .send_message(RawDocBatch {
-                            docs,
-                            checkpoint_delta: checkpoint_delta.clone(),
-                            force_commit: false,
-                        })
+                        .send_message(RawDocBatch::new(docs, checkpoint_delta.clone(), false))
                         .await
                         .unwrap();
 

quickwit/quickwit-ingest/Cargo.toml

@@ -17,10 +17,8 @@ bytes = { workspace = true }
 bytesize = { workspace = true }
 dyn-clone = { workspace = true }
 fail = { workspace = true, optional = true }
-flume = { workspace = true }
 futures = { workspace = true }
 http = { workspace = true }
-hyper = { workspace = true }
 mockall = { workspace = true, optional = true }
 mrecordlog = { workspace = true }
 once_cell = { workspace = true }

quickwit/quickwit-integration-tests/Cargo.toml

@@ -14,30 +14,21 @@ license.workspace = true
 
 [dev-dependencies]
 anyhow = { workspace = true }
-bytes = { workspace = true }
-chitchat = { workspace = true }
 futures-util = { workspace = true }
 hyper = { workspace = true }
 itertools = { workspace = true }
-rand = { workspace = true }
 reqwest = { workspace = true }
-serde = { workspace = true }
 serde_json = { workspace = true }
 tempfile = { workspace = true }
 tokio = { workspace = true }
-tokio-stream = { workspace = true }
 tonic = { workspace = true }
 tracing = { workspace = true }
 
 quickwit-actors = { workspace = true, features = ["testsuite"] }
-quickwit-cluster = { workspace = true, features = ["testsuite"] }
 quickwit-common = { workspace = true, features = ["testsuite"] }
 quickwit-config = { workspace = true, features = ["testsuite"] }
-quickwit-indexing = { workspace = true, features = ["testsuite"] }
-quickwit-janitor = { workspace = true, features = ["testsuite"] }
 quickwit-metastore = { workspace = true, features = ["testsuite"] }
 quickwit-proto = { workspace = true }
 quickwit-rest-client = { workspace = true }
-quickwit-search = { workspace = true, features = ["testsuite"] }
 quickwit-serve = { workspace = true }
 quickwit-storage = { workspace = true, features = ["testsuite"] }