Merge branch 'main' into no-lifo-slots

.github/workflows/coverage.yml

@@ -153,7 +153,8 @@ jobs:
         run: |
           cargo llvm-cov clean --workspace
           cargo llvm-cov nextest --no-report --test failpoints --features fail/failpoints --retries 2
-          CARGO_BUILD_JOBS=4 cargo llvm-cov nextest --no-report --all-features --retries 2
+          # increase stack size for test_all_with_s3_localstack_cli, see quickwit#4963
+          RUST_MIN_STACK=67108864 CARGO_BUILD_JOBS=4 cargo llvm-cov nextest --no-report --all-features --retries 2
           cargo llvm-cov report --lcov --output-path lcov.info
         working-directory: ./quickwit
 
@@ -162,3 +163,32 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
           files: ./quickwit/lcov.info
+
+  on-failure:
+    if: ${{ github.repository_owner == 'quickwit-oss' && failure() }}
+    name: On Failure
+    needs: [test]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Send Message
+        uses: sarisia/actions-status-discord@v1
+        with:
+          webhook: ${{ secrets.DISCORD_WEBHOOK }}
+          nodetail: true
+          color: "#FF0000"
+          title: ""
+          description: |
+            ### ❌ [${{ github.event.pull_request.title }}](${{ github.event.pull_request.html_url }})
+
+            @${{ github.actor }} quickwit coverage CI failed on your PR.
+
+            Coverage CI contains tests that are not running in the regular CI because they are too lengthy.
+            For this reason it is possible for it to break even if the tests were passing on your PR.
+            This is not a catastrophy, but you are responsible for fixing it!
+
+            You can run the full test suite locally with `make test-all`.
+
+            Please report in this channel that you are working on it/fixed it/or if it is a flaky test/
+            or if you need help.
+
+            **[View logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})**

config/quickwit.yaml

@@ -83,6 +83,7 @@ version: 0.8
 # default_index_root_uri: s3://your-bucket/indexes
 #
 # -------------------------------- Storage settings --------------------------------
+# https://quickwit.io/docs/configuration/node-config#storage-configuration
 #
 # Hardcoding credentials into configuration files is not secure and strongly
 # discouraged. Prefer the alternative authentication methods that your storage
@@ -103,6 +104,7 @@ version: 0.8
 #     disable_multipart_upload: false
 #
 # -------------------------------- Metastore settings --------------------------------
+# https://quickwit.io/docs/configuration/node-config#metastore-configuration
 #
 # metastore:
 #   postgres:
@@ -113,6 +115,7 @@ version: 0.8
 #     max_connection_lifetime: 30min
 #
 # -------------------------------- Indexer settings --------------------------------
+# https://quickwit.io/docs/configuration/node-config#indexer-configuration
 
 indexer:
   enable_otlp_endpoint: ${QW_ENABLE_OTLP_ENDPOINT:-true}
@@ -122,20 +125,27 @@ indexer:
 #
 #
 # -------------------------------- Ingest API settings ------------------------------
+# https://quickwit.io/docs/configuration/node-config#ingest-api-configuration
 #
 # ingest_api:
 #   max_queue_memory_usage: 2GiB
 #   max_queue_disk_usage: 4GiB
 #
 # -------------------------------- Searcher settings --------------------------------
+# https://quickwit.io/docs/configuration/node-config#searcher-configuration
 #
 # searcher:
 #   fast_field_cache_capacity: 1G
 #   split_footer_cache_capacity: 500M
-#   max_num_concurrent_split_streams: 100
 #   partial_request_cache_capacity: 64M
+#   max_num_concurrent_split_streams: 100
 #   max_num_concurrent_split_searches: 100
-#
+#   aggregation_memory_limit: 500M
+#   aggregation_bucket_limit: 65000
+#   split_cache:
+#      max_num_bytes: 1G
+#      max_num_splits: 10000
+#      num_concurrent_downloads: 1
 # -------------------------------- Jaeger settings --------------------------------
 
 jaeger:

quickwit/Cargo.toml

@@ -155,7 +155,6 @@ mockall = "0.11"
 mrecordlog = { git = "https://github.com/quickwit-oss/mrecordlog", rev = "306c0a7" }
 new_string_template = "1.5.1"
 nom = "7.1.3"
-num_cpus = "1"
 numfmt = "1.1.1"
 once_cell = "1"
 oneshot = "0.1.5"
@@ -188,7 +187,7 @@ pulsar = { git = "https://github.com/quickwit-oss/pulsar-rs.git", rev = "f9eff04
 quote = "1.0.23"
 rand = "0.8"
 rand_distr = "0.4"
-rayon = "1"
+rayon = "1.10"
 rdkafka = { version = "0.33", default-features = false, features = [
   "cmake-build",
   "libz",
@@ -203,8 +202,8 @@ reqwest = { version = "0.11", default-features = false, features = [
   "rustls-tls",
 ] }
 rust-embed = "6.8.1"
-sea-query = { version = "0" }
-sea-query-binder = { version = "0", features = [
+sea-query = { version = "0.30" }
+sea-query-binder = { version = "0.5", features = [
   "runtime-tokio-rustls",
   "sqlx-postgres",
 ] }
@@ -270,16 +269,12 @@ wiremock = "0.5"
 zstd = "0.13.0"
 
 aws-config = "1.2"
-aws-credential-types = { version = "1.2", features = [
-  "hardcoded-credentials",
-] }
+aws-credential-types = { version = "1.2", features = ["hardcoded-credentials"] }
 aws-sdk-kinesis = "1.21"
 aws-sdk-s3 = "1.24"
 aws-smithy-async = "1.2"
-aws-smithy-runtime = "1.3"
-aws-smithy-types = { version = "1.1", features = [
-  "byte-stream-poll-next"
-] }
+aws-smithy-runtime = "1.5"
+aws-smithy-types = { version = "1.1", features = ["byte-stream-poll-next"] }
 aws-types = "1.2"
 
 azure_core = { version = "0.13.0", features = ["enable_reqwest_rustls"] }
@@ -322,7 +317,7 @@ quickwit-serve = { path = "quickwit-serve" }
 quickwit-storage = { path = "quickwit-storage" }
 quickwit-telemetry = { path = "quickwit-telemetry" }
 
-tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "92b5526", default-features = false, features = [
+tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "6181c1e", default-features = false, features = [
   "lz4-compression",
   "mmap",
   "quickwit",

quickwit/Makefile

@@ -29,6 +29,7 @@ test-all:
 	QW_S3_ENDPOINT=http://localhost:4566 \
 	QW_S3_FORCE_PATH_STYLE_ACCESS=1 \
 	QW_TEST_DATABASE_URL=postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev \
+	RUST_MIN_STACK=67108864 \
 	cargo test --all-features
 	cargo test --test failpoints --features fail/failpoints
 

quickwit/quickwit-aws/src/lib.rs

@@ -62,7 +62,7 @@ pub async fn get_aws_config() -> &'static aws_config::SdkConfig {
                 .hyper_builder(hyper_client_builder)
                 .build(https_connector);
 
-            aws_config::defaults(BehaviorVersion::v2023_11_09())
+            aws_config::defaults(BehaviorVersion::v2024_03_28())
                 .http_client(hyper_client)
                 // Currently handle this ourselves so probably best for now to leave it as is.
                 .retry_config(RetryConfig::disabled())