diff --git a/docs/configuration/source-config.md b/docs/configuration/source-config.md
index 46fea0b58f8..48e517406c6 100644
--- a/docs/configuration/source-config.md
+++ b/docs/configuration/source-config.md
@@ -64,11 +64,14 @@ The Kafka source consumes a `topic` using the client library [librdkafka](https:
 - `bootstrap.servers`
 Comma-separated list of host and port pairs that are the addresses of a subset of the Kafka brokers in the Kafka cluster.
 
+- `auto.offset.reset`
+Defines the behavior of the source when consuming a partition for which there is no initial offset saved in the checkpoint. `earliest` consumes from the beginning of the partition, whereas `latest` (default) consumes from the end.
+
 - `enable.auto.commit`
 The Kafka source manages commit offsets manually using the [checkpoint API](../overview/concepts/indexing.md#checkpoint) and disables auto-commit.
 
 - `group.id`
-Kafka-based distributed indexing relies on consumer groups. Unless overridden in the client parameters, the default group ID assigned to each consumer managed by the source is `quickwit-{index_uid}-{source_id}`
+Kafka-based distributed indexing relies on consumer groups. Unless overridden in the client parameters, the default group ID assigned to each consumer managed by the source is `quickwit-{index_uid}-{source_id}`.
 
 - `max.poll.interval.ms`
 Short max poll interval durations may cause a source to crash when back pressure from the indexer occurs. Therefore, Quickwit recommends using the default value of `300000` (5 minutes).
diff --git a/docs/operating/monitoring.md b/docs/operating/monitoring.md
index 0734d8dd839..335be31b52c 100644
--- a/docs/operating/monitoring.md
+++ b/docs/operating/monitoring.md
@@ -5,12 +5,12 @@ sidebar_position: 2
 
 You can monitor your Quickwit cluster with Grafana.
 
-We provide two Grafana dashboards to help you monitor:
+We provide three Grafana dashboards to help you monitor:
 - [indexers performance](https://github.com/quickwit-oss/quickwit/blob/main/monitoring/grafana/dashboards/indexers.json)
-- [indexers performance](https://github.com/quickwit-oss/quickwit/blob/main/monitoring/grafana/dashboards/searchers.json)
+- [searchers performance](https://github.com/quickwit-oss/quickwit/blob/main/monitoring/grafana/dashboards/searchers.json)
 - [metastore queries](https://github.com/quickwit-oss/quickwit/blob/main/monitoring/grafana/dashboards/metastore.json)
 
-Both dashboards relies on a prometheus datasource fed with [Quickwit metrics](../reference/metrics.md).
+Dashboards rely on a prometheus datasource fed with [Quickwit metrics](../reference/metrics.md).
 
 ## Screenshots
 
diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock
index b360b6e849a..ba6daccd9f2 100644
--- a/quickwit/Cargo.lock
+++ b/quickwit/Cargo.lock
@@ -33,6 +33,16 @@ dependencies = [
  "winapi-build",
 ]
 
+[[package]]
+name = "aead"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0"
+dependencies = [
+ "crypto-common",
+ "generic-array",
+]
+
 [[package]]
 name = "aes"
 version = "0.8.3"
@@ -50,7 +60,7 @@ version = "0.7.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5a824f2aa7e75a0c98c5a504fceb80649e9c35265d44525b5f94de4771a395cd"
 dependencies = [
- "getrandom 0.2.10",
+ "getrandom 0.2.11",
  "once_cell",
  "version_check",
 ]
@@ -62,7 +72,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a"
 dependencies = [
  "cfg-if",
- "getrandom 0.2.10",
+ "getrandom 0.2.11",
  "once_cell",
  "version_check",
  "zerocopy",
@@ -291,7 +301,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -302,7 +312,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -780,7 +790,7 @@ dependencies = [
  "bytes",
  "dyn-clone",
  "futures",
- "getrandom 0.2.10",
+ "getrandom 0.2.11",
  "http-types",
  "log",
  "paste",
@@ -846,7 +856,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1"
 dependencies = [
  "futures-core",
- "getrandom 0.2.10",
+ "getrandom 0.2.11",
  "instant",
  "pin-project-lite",
  "rand 0.8.5",
@@ -941,6 +951,14 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "bitpacking"
+version = "0.8.3"
+source = "git+https://github.com/quickwit-oss/bitpacking?rev=f730b75#f730b75598589d7c2a9c4eee3fc17025c59e2b2c"
+dependencies = [
+ "crunchy",
+]
+
 [[package]]
 name = "bitpacking"
 version = "0.8.4"
@@ -1031,16 +1049,6 @@ version = "3.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
 
-[[package]]
-name = "byte-unit"
-version = "4.0.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da78b32057b8fdfc352504708feeba7216dcd65a2c9ab02978cbd288d1279b6c"
-dependencies = [
- "serde",
- "utf8-width",
-]
-
 [[package]]
 name = "bytecheck"
 version = "0.6.11"
@@ -1095,10 +1103,19 @@ dependencies = [
 ]
 
 [[package]]
-name = "bytestring"
+name = "bytesize"
 version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "238e4886760d98c4f899360c834fa93e62cf7f721ac3c2da375cbdf4b8679aae"
+checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "bytestring"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72"
 dependencies = [
  "bytes",
 ]
@@ -1170,6 +1187,30 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "chacha20"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818"
+dependencies = [
+ "cfg-if",
+ "cipher",
+ "cpufeatures",
+]
+
+[[package]]
+name = "chacha20poly1305"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35"
+dependencies = [
+ "aead",
+ "chacha20",
+ "cipher",
+ "poly1305",
+ "zeroize",
+]
+
 [[package]]
 name = "charset"
 version = "0.1.3"
@@ -1182,9 +1223,9 @@ dependencies = [
 
 [[package]]
 name = "chitchat"
-version = "0.6.0"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e5c649e5309f040f18aa0d660e63fe913d3d642e4771dd3cb66515ad75d239d"
+checksum = "cb13c7104e3c77027520e984339ab3e805fd19c05c8555cc6338b2a4f455cbfb"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -1214,9 +1255,9 @@ dependencies = [
 
 [[package]]
 name = "chrono-tz"
-version = "0.8.3"
+version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1369bc6b9e9a7dfdae2055f6ec151fe9c554a9d23d357c0237cee2e25eaabb7"
+checksum = "e23185c0e21df6ed832a12e2bda87c7d1def6842881fb634a8511ced741b0d76"
 dependencies = [
  "chrono",
  "chrono-tz-build",
@@ -1225,9 +1266,9 @@ dependencies = [
 
 [[package]]
 name = "chrono-tz-build"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2f5ebdc942f57ed96d560a6d1a459bae5851102a25d5bf89dc04ae453e31ecf"
+checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f"
 dependencies = [
  "parse-zoneinfo",
  "phf",
@@ -1282,6 +1323,7 @@ checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
 dependencies = [
  "crypto-common",
  "inout",
+ "zeroize",
 ]
 
 [[package]]
@@ -1353,6 +1395,20 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "community-id"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f6af96839c04974cf381e427792a99913ecf3f7bfb348f153dc8a8e5f9803ad"
+dependencies = [
+ "anyhow",
+ "base64 0.21.5",
+ "hex",
+ "lazy_static",
+ "num_enum 0.6.1",
+ "sha1",
+]
+
 [[package]]
 name = "concurrent-queue"
 version = "2.3.0"
@@ -1465,9 +1521,9 @@ dependencies = [
 
 [[package]]
 name = "crc-catalog"
-version = "2.2.0"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484"
+checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
 
 [[package]]
 name = "crc32c"
@@ -1602,9 +1658,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
 dependencies = [
  "generic-array",
+ "rand_core 0.6.4",
  "typenum",
 ]
 
+[[package]]
+name = "crypto_secretbox"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9d6cf87adf719ddf43a805e92c6870a531aedda35ff640442cbaf8674e141e1"
+dependencies = [
+ "aead",
+ "cipher",
+ "generic-array",
+ "poly1305",
+ "salsa20",
+ "subtle",
+ "zeroize",
+]
+
 [[package]]
 name = "csv"
 version = "1.3.0"
@@ -1680,7 +1752,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -1702,7 +1774,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
 dependencies = [
  "darling_core 0.20.3",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -1772,20 +1844,10 @@ version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3"
 dependencies = [
+ "powerfmt",
  "serde",
 ]
 
-[[package]]
-name = "derivative"
-version = "2.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
 [[package]]
 name = "derive_more"
 version = "0.99.17"
@@ -1858,14 +1920,14 @@ dependencies = [
 
 [[package]]
 name = "dns-lookup"
-version = "1.0.8"
+version = "2.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53ecafc952c4528d9b51a458d1a8904b81783feff9fde08ab6ed2545ff396872"
+checksum = "e5766087c2235fec47fafa4cfecc81e494ee679d0fd4a59887ea0919bfb0e4fc"
 dependencies = [
  "cfg-if",
  "libc",
- "socket2 0.4.10",
- "winapi 0.3.9",
+ "socket2 0.5.5",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -1912,9 +1974,9 @@ dependencies = [
 
 [[package]]
 name = "dyn-clone"
-version = "1.0.14"
+version = "1.0.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd"
+checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d"
 
 [[package]]
 name = "either"
@@ -2057,7 +2119,7 @@ checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -2090,9 +2152,9 @@ dependencies = [
 
 [[package]]
 name = "errno"
-version = "0.3.5"
+version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860"
+checksum = "7c18ee0ed65a5f1f81cac6b1d213b69c35fa47d4252ad41f1486dbd8226fe36e"
 dependencies = [
  "libc",
  "windows-sys 0.48.0",
@@ -2348,7 +2410,7 @@ checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -2408,6 +2470,7 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
 dependencies = [
  "typenum",
  "version_check",
+ "zeroize",
 ]
 
 [[package]]
@@ -2423,9 +2486,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.10"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
+checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f"
 dependencies = [
  "cfg-if",
  "js-sys",
@@ -2613,6 +2676,16 @@ dependencies = [
  "async-trait",
 ]
 
+[[package]]
+name = "grok"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "273797968160270573071022613fc4aa28b91fe68f3eef6c96a1b2a1947ddfbd"
+dependencies = [
+ "glob",
+ "onig",
+]
+
 [[package]]
 name = "h2"
 version = "0.3.21"
@@ -3028,7 +3101,7 @@ checksum = "ce243b1bfa62ffc028f1cc3b6034ec63d649f3031bc8a4fbbb004e1ac17d1f68"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -3055,9 +3128,9 @@ dependencies = [
 
 [[package]]
 name = "inventory"
-version = "0.3.12"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1be380c410bf0595e94992a648ea89db4dd3f3354ba54af206fd2a68cf5ac8e"
+checksum = "0508c56cfe9bfd5dfeb0c22ab9a6abfda2f27bdca422132e494266351ed8d83c"
 
 [[package]]
 name = "io-lifetimes"
@@ -3131,18 +3204,18 @@ dependencies = [
 
 [[package]]
 name = "js-sys"
-version = "0.3.64"
+version = "0.3.65"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a"
+checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8"
 dependencies = [
  "wasm-bindgen",
 ]
 
 [[package]]
 name = "json_comments"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41ee439ee368ba4a77ac70d04f14015415af8600d6c894dc1f11bd79758c57d5"
+checksum = "9dbbfed4e59ba9750e15ba154fdfd9329cee16ff3df539c2666b70f58cc32105"
 
 [[package]]
 name = "jsonwebtoken"
@@ -3178,9 +3251,9 @@ dependencies = [
 
 [[package]]
 name = "lalrpop"
-version = "0.19.12"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a1cbf952127589f2851ab2046af368fd20645491bb4b376f04b7f94d7a9837b"
+checksum = "da4081d44f4611b66c6dd725e6de3169f9f63905421e8626fcb86b6a898998b8"
 dependencies = [
  "ascii-canvas",
  "bit-set",
@@ -3191,7 +3264,7 @@ dependencies = [
  "lalrpop-util",
  "petgraph",
  "regex",
- "regex-syntax 0.6.29",
+ "regex-syntax 0.7.5",
  "string_cache",
  "term",
  "tiny-keccak",
@@ -3200,9 +3273,9 @@ dependencies = [
 
 [[package]]
 name = "lalrpop-util"
-version = "0.19.12"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3c48237b9604c5a4702de6b824e02006c3214327564636aef27c1028a8fa0ed"
+checksum = "3f35c735096c0293d313e8f2a641627472b83d01b937177fe76e5e2708d31e0d"
 
 [[package]]
 name = "lazy_static"
@@ -3221,9 +3294,9 @@ checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25"
 
 [[package]]
 name = "libc"
-version = "0.2.149"
+version = "0.2.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b"
+checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c"
 
 [[package]]
 name = "libm"
@@ -3231,6 +3304,17 @@ version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058"
 
+[[package]]
+name = "libredox"
+version = "0.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"
+dependencies = [
+ "bitflags 2.4.1",
+ "libc",
+ "redox_syscall 0.4.1",
+]
+
 [[package]]
 name = "libsqlite3-sys"
 version = "0.26.0"
@@ -3495,9 +3579,9 @@ checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.4.10"
+version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f"
+checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829"
 
 [[package]]
 name = "lock_api"
@@ -3713,7 +3797,7 @@ dependencies = [
 [[package]]
 name = "mrecordlog"
 version = "0.4.0"
-source = "git+https://github.com/quickwit-oss/mrecordlog?rev=0d1a7aa#0d1a7aa2bdf11aec832919503cefaf282d70e0d8"
+source = "git+https://github.com/quickwit-oss/mrecordlog?rev=ebee0fd#ebee0fdb6556a18f7cb73eca32e40bf44b0c4c6c"
 dependencies = [
  "async-trait",
  "bytes",
@@ -3761,7 +3845,7 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3"
 dependencies = [
- "getrandom 0.2.10",
+ "getrandom 0.2.11",
 ]
 
 [[package]]
@@ -3905,7 +3989,16 @@ version = "0.5.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9"
 dependencies = [
- "num_enum_derive",
+ "num_enum_derive 0.5.11",
+]
+
+[[package]]
+name = "num_enum"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a015b430d3c108a207fd776d2e2196aaf8b1cf8cf93253e3a097ff3085076a1"
+dependencies = [
+ "num_enum_derive 0.6.1",
 ]
 
 [[package]]
@@ -3920,6 +4013,18 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "num_enum_derive"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96667db765a921f7b295ffee8b60472b686a51d4f21c2ee4ffdb94c7013b65a6"
+dependencies = [
+ "proc-macro-crate 1.3.1",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.39",
+]
+
 [[package]]
 name = "num_threads"
 version = "0.1.6"
@@ -3943,7 +4048,7 @@ checksum = "c38841cdd844847e3e7c8d29cef9dcfed8877f8f56f9071f77843ecf3baf937f"
 dependencies = [
  "base64 0.13.1",
  "chrono",
- "getrandom 0.2.10",
+ "getrandom 0.2.11",
  "http",
  "rand 0.8.5",
  "reqwest",
@@ -3988,12 +4093,40 @@ dependencies = [
  "loom",
 ]
 
+[[package]]
+name = "onig"
+version = "6.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f"
+dependencies = [
+ "bitflags 1.3.2",
+ "libc",
+ "once_cell",
+ "onig_sys",
+]
+
+[[package]]
+name = "onig_sys"
+version = "69.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
+
 [[package]]
 name = "oorandom"
 version = "11.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
 
+[[package]]
+name = "opaque-debug"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
+
 [[package]]
 name = "openidconnect"
 version = "2.5.1"
@@ -4023,9 +4156,9 @@ dependencies = [
 
 [[package]]
 name = "openssl"
-version = "0.10.57"
+version = "0.10.59"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c"
+checksum = "7a257ad03cd8fb16ad4172fedf8094451e1af1c4b70097636ef2eac9a5f0cc33"
 dependencies = [
  "bitflags 2.4.1",
  "cfg-if",
@@ -4044,7 +4177,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -4064,9 +4197,9 @@ dependencies = [
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.93"
+version = "0.9.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d"
+checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9"
 dependencies = [
  "cc",
  "libc",
@@ -4165,9 +4298,9 @@ dependencies = [
 
 [[package]]
 name = "ordered-float"
-version = "3.9.2"
+version = "4.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc"
+checksum = "536900a8093134cf9ccf00a27deb3532421099e958d9dd431135d0c7543ca1e8"
 dependencies = [
  "num-traits",
 ]
@@ -4204,7 +4337,7 @@ dependencies = [
  "proc-macro-error",
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -4222,7 +4355,7 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
 [[package]]
 name = "ownedbytes"
 version = "0.6.0"
-source = "git+https://github.com/quickwit-oss/tantivy/?rev=ecb9a89#ecb9a89a9ff0573fba5c2cf8b83f353f35022310"
+source = "git+https://github.com/quickwit-oss/tantivy/?rev=927b443#927b4432c935b55cb8a2275daea3f77fc6823bd6"
 dependencies = [
  "stable_deref_trait",
 ]
@@ -4236,7 +4369,7 @@ dependencies = [
  "ansi-str",
  "bytecount",
  "fnv",
- "strip-ansi-escapes",
+ "strip-ansi-escapes 0.1.1",
  "unicode-width",
 ]
 
@@ -4295,17 +4428,6 @@ version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c"
 
-[[package]]
-name = "path"
-version = "0.1.0"
-source = "git+https://github.com/vectordotdev/vrl?rev=v0.3.0#113005bcee6cd7b5ea0a53a7db2fc45ba4bc4125"
-dependencies = [
- "once_cell",
- "regex",
- "serde",
- "snafu",
-]
-
 [[package]]
 name = "pbkdf2"
 version = "0.11.0"
@@ -4318,6 +4440,12 @@ dependencies = [
  "sha2",
 ]
 
+[[package]]
+name = "peeking_take_while"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e9ed2178b0575fff8e1b83b58ba6f75e727aafac2e1b6c795169ad3b17eb518"
+
 [[package]]
 name = "pem"
 version = "1.1.1"
@@ -4342,6 +4470,51 @@ version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
 
+[[package]]
+name = "pest"
+version = "2.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5"
+dependencies = [
+ "memchr",
+ "thiserror",
+ "ucd-trie",
+]
+
+[[package]]
+name = "pest_derive"
+version = "2.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81d78524685f5ef2a3b3bd1cafbc9fcabb036253d9b1463e726a91cd16e2dfc2"
+dependencies = [
+ "pest",
+ "pest_generator",
+]
+
+[[package]]
+name = "pest_generator"
+version = "2.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68bd1206e71118b5356dae5ddc61c8b11e28b09ef6a31acbd15ea48a28e0c227"
+dependencies = [
+ "pest",
+ "pest_meta",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.39",
+]
+
+[[package]]
+name = "pest_meta"
+version = "2.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c747191d4ad9e4a4ab9c8798f1e82a39affe7ef9648390b7e5548d18e099de6"
+dependencies = [
+ "once_cell",
+ "pest",
+ "sha2",
+]
+
 [[package]]
 name = "petgraph"
 version = "0.6.4"
@@ -4416,7 +4589,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -4577,11 +4750,22 @@ dependencies = [
  "pnet_sys",
 ]
 
+[[package]]
+name = "poly1305"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf"
+dependencies = [
+ "cpufeatures",
+ "opaque-debug",
+ "universal-hash",
+]
+
 [[package]]
 name = "portable-atomic"
-version = "1.5.0"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b559898e0b4931ed2d3b959ab0c2da4d99cc644c4b0b1a35b4d344027f474023"
+checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b"
 
 [[package]]
 name = "postcard"
@@ -4594,6 +4778,12 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "powerfmt"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.17"
@@ -4678,7 +4868,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d"
 dependencies = [
  "proc-macro2",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -4765,9 +4955,9 @@ dependencies = [
 
 [[package]]
 name = "proptest"
-version = "1.3.1"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c003ac8c77cb07bb74f5f198bce836a689bcd5a42574612bf14d17bfd08c20e"
+checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf"
 dependencies = [
  "bit-set",
  "bit-vec",
@@ -4777,7 +4967,7 @@ dependencies = [
  "rand 0.8.5",
  "rand_chacha 0.3.1",
  "rand_xorshift",
- "regex-syntax 0.7.5",
+ "regex-syntax 0.8.2",
  "rusty-fork",
  "tempfile",
  "unarray",
@@ -4965,8 +5155,8 @@ version = "0.6.5-dev"
 dependencies = [
  "anyhow",
  "async-trait",
- "byte-unit",
  "bytes",
+ "bytesize",
  "chitchat",
  "clap",
  "colored",
@@ -5055,7 +5245,7 @@ dependencies = [
  "prost-build",
  "quote",
  "serde",
- "syn 2.0.38",
+ "syn 2.0.39",
  "tonic-build",
 ]
 
@@ -5091,7 +5281,7 @@ dependencies = [
  "anyhow",
  "async-speed-limit",
  "async-trait",
- "byte-unit",
+ "bytesize",
  "dyn-clone",
  "env_logger",
  "fnv",
@@ -5125,8 +5315,8 @@ name = "quickwit-config"
 version = "0.6.5-dev"
 dependencies = [
  "anyhow",
- "byte-unit",
  "bytes",
+ "bytesize",
  "chrono",
  "cron",
  "enum-iterator",
@@ -5134,6 +5324,7 @@ dependencies = [
  "itertools 0.11.0",
  "json_comments",
  "new_string_template",
+ "num_cpus",
  "once_cell",
  "quickwit-common",
  "quickwit-doc-mapper",
@@ -5143,13 +5334,12 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_with 3.4.0",
- "serde_yaml 0.9.25",
+ "serde_yaml 0.9.27",
  "tokio",
  "toml 0.7.8",
  "tracing",
  "utoipa",
  "vrl",
- "vrl-stdlib",
 ]
 
 [[package]]
@@ -5251,7 +5441,7 @@ dependencies = [
  "regex",
  "serde",
  "serde_json",
- "serde_yaml 0.9.25",
+ "serde_yaml 0.9.27",
  "siphasher",
  "tantivy",
  "thiserror",
@@ -5267,7 +5457,6 @@ version = "0.6.5-dev"
 dependencies = [
  "anyhow",
  "async-trait",
- "byte-unit",
  "futures",
  "futures-util",
  "itertools 0.11.0",
@@ -5283,7 +5472,7 @@ dependencies = [
  "rand 0.8.5",
  "serde",
  "serde_json",
- "serde_yaml 0.9.25",
+ "serde_yaml 0.9.27",
  "tantivy",
  "tempfile",
  "thiserror",
@@ -5305,8 +5494,8 @@ dependencies = [
  "aws-sdk-kinesis",
  "aws-smithy-client",
  "backoff",
- "byte-unit",
  "bytes",
+ "bytesize",
  "chitchat",
  "criterion",
  "fail",
@@ -5355,7 +5544,6 @@ dependencies = [
  "ulid",
  "utoipa",
  "vrl",
- "vrl-stdlib",
 ]
 
 [[package]]
@@ -5364,8 +5552,8 @@ version = "0.6.5-dev"
 dependencies = [
  "anyhow",
  "async-trait",
- "byte-unit",
  "bytes",
+ "bytesize",
  "dyn-clone",
  "flume",
  "futures",
@@ -5505,7 +5693,7 @@ dependencies = [
  "proc-macro2",
  "quickwit-macros-impl",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -5515,7 +5703,7 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -5524,7 +5712,7 @@ version = "0.6.5-dev"
 dependencies = [
  "anyhow",
  "async-trait",
- "byte-unit",
+ "bytesize",
  "dotenv",
  "futures",
  "http",
@@ -5724,8 +5912,8 @@ dependencies = [
  "anyhow",
  "assert-json-diff 2.0.2",
  "async-trait",
- "byte-unit",
  "bytes",
+ "bytesize",
  "chitchat",
  "elasticsearch-dsl",
  "futures",
@@ -5795,8 +5983,8 @@ dependencies = [
  "azure_storage",
  "azure_storage_blobs",
  "base64 0.21.5",
- "byte-unit",
  "bytes",
+ "bytesize",
  "fnv",
  "futures",
  "hyper",
@@ -5853,9 +6041,9 @@ dependencies = [
 
 [[package]]
 name = "quoted_printable"
-version = "0.4.8"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a3866219251662ec3b26fc217e3e05bf9c4f84325234dfb96bf0bf840889e49"
+checksum = "79ec282e887b434b68c18fe5c121d38e72a5cf35119b59e54ec5b992ea9c8eb0"
 
 [[package]]
 name = "radium"
@@ -5922,7 +6110,7 @@ version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
 dependencies = [
- "getrandom 0.2.10",
+ "getrandom 0.2.11",
 ]
 
 [[package]]
@@ -5993,29 +6181,20 @@ dependencies = [
 
 [[package]]
 name = "rdkafka-sys"
-version = "4.6.0+2.2.0"
+version = "4.7.0+2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad63c279fca41a27c231c450a2d2ad18288032e9cbb159ad16c9d96eba35aaaf"
+checksum = "55e0d2f9ba6253f6ec72385e453294f8618e9e15c2c6aba2a5c01ccf9622d615"
 dependencies = [
  "cmake",
  "libc",
  "libz-sys",
- "num_enum",
+ "num_enum 0.5.11",
  "openssl-sys",
  "pkg-config",
  "sasl2-sys",
  "zstd-sys",
 ]
 
-[[package]]
-name = "redox_syscall"
-version = "0.2.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
-dependencies = [
- "bitflags 1.3.2",
-]
-
 [[package]]
 name = "redox_syscall"
 version = "0.3.5"
@@ -6036,12 +6215,12 @@ dependencies = [
 
 [[package]]
 name = "redox_users"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
+checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4"
 dependencies = [
- "getrandom 0.2.10",
- "redox_syscall 0.2.16",
+ "getrandom 0.2.11",
+ "libredox",
  "thiserror",
 ]
 
@@ -6177,7 +6356,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b"
 dependencies = [
  "cc",
- "getrandom 0.2.10",
+ "getrandom 0.2.11",
  "libc",
  "spin 0.9.8",
  "untrusted 0.9.0",
@@ -6270,7 +6449,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rust-embed-utils",
- "syn 2.0.38",
+ "syn 2.0.39",
  "walkdir",
 ]
 
@@ -6354,7 +6533,7 @@ dependencies = [
  "bitflags 2.4.1",
  "errno",
  "libc",
- "linux-raw-sys 0.4.10",
+ "linux-raw-sys 0.4.11",
  "windows-sys 0.48.0",
 ]
 
@@ -6447,6 +6626,15 @@ version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
 
+[[package]]
+name = "salsa20"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213"
+dependencies = [
+ "cipher",
+]
+
 [[package]]
 name = "same-file"
 version = "1.0.6"
@@ -6521,14 +6709,14 @@ dependencies = [
 
 [[package]]
 name = "sea-query-derive"
-version = "0.4.0"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd78f2e0ee8e537e9195d1049b752e0433e2cac125426bccb7b5c3e508096117"
+checksum = "25a82fcb49253abcb45cdcb2adf92956060ec0928635eb21b4f7a6d8f25ab0bc"
 dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn 2.0.39",
  "thiserror",
 ]
 
@@ -6569,9 +6757,9 @@ checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090"
 
 [[package]]
 name = "serde"
-version = "1.0.171"
+version = "1.0.192"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9"
+checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001"
 dependencies = [
  "serde_derive",
 ]
@@ -6588,20 +6776,20 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.171"
+version = "1.0.192"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682"
+checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.107"
+version = "1.0.108"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65"
+checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b"
 dependencies = [
  "itoa",
  "ryu",
@@ -6721,7 +6909,7 @@ dependencies = [
  "darling 0.20.3",
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -6738,9 +6926,9 @@ dependencies = [
 
 [[package]]
 name = "serde_yaml"
-version = "0.9.25"
+version = "0.9.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574"
+checksum = "3cc7a1570e38322cfe4154732e5110f887ea57e22b76f4bfd32b5bdd3368666c"
 dependencies = [
  "indexmap 2.1.0",
  "itoa",
@@ -7212,7 +7400,16 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "011cbb39cf7c1f62871aea3cc46e5817b0937b49e9447370c93cacbe93a766d8"
 dependencies = [
- "vte",
+ "vte 0.10.1",
+]
+
+[[package]]
+name = "strip-ansi-escapes"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55ff8ef943b384c414f54aefa961dd2bd853add74ec75e7ac74cf91dba62bcfa"
+dependencies = [
+ "vte 0.11.1",
 ]
 
 [[package]]
@@ -7240,9 +7437,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.38"
+version = "2.0.39"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b"
+checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -7257,9 +7454,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
 
 [[package]]
 name = "syslog_loose"
-version = "0.18.0"
+version = "0.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97fb75f176928530867b2a659e470f9c9ff71904695bab6556f7ad30f9039efd"
+checksum = "acf5252d1adec0a489a0225f867c1a7fd445e41674530a396d0629cff0c4b211"
 dependencies = [
  "chrono",
  "nom",
@@ -7314,13 +7511,13 @@ dependencies = [
 [[package]]
 name = "tantivy"
 version = "0.21.0"
-source = "git+https://github.com/quickwit-oss/tantivy/?rev=ecb9a89#ecb9a89a9ff0573fba5c2cf8b83f353f35022310"
+source = "git+https://github.com/quickwit-oss/tantivy/?rev=927b443#927b4432c935b55cb8a2275daea3f77fc6823bd6"
 dependencies = [
  "aho-corasick",
  "arc-swap",
  "async-trait",
  "base64 0.21.5",
- "bitpacking",
+ "bitpacking 0.8.3",
  "byteorder",
  "census",
  "crc32fast",
@@ -7369,15 +7566,15 @@ dependencies = [
 [[package]]
 name = "tantivy-bitpacker"
 version = "0.5.0"
-source = "git+https://github.com/quickwit-oss/tantivy/?rev=ecb9a89#ecb9a89a9ff0573fba5c2cf8b83f353f35022310"
+source = "git+https://github.com/quickwit-oss/tantivy/?rev=927b443#927b4432c935b55cb8a2275daea3f77fc6823bd6"
 dependencies = [
- "bitpacking",
+ "bitpacking 0.8.4",
 ]
 
 [[package]]
 name = "tantivy-columnar"
 version = "0.2.0"
-source = "git+https://github.com/quickwit-oss/tantivy/?rev=ecb9a89#ecb9a89a9ff0573fba5c2cf8b83f353f35022310"
+source = "git+https://github.com/quickwit-oss/tantivy/?rev=927b443#927b4432c935b55cb8a2275daea3f77fc6823bd6"
 dependencies = [
  "fastdivide",
  "fnv",
@@ -7392,7 +7589,7 @@ dependencies = [
 [[package]]
 name = "tantivy-common"
 version = "0.6.0"
-source = "git+https://github.com/quickwit-oss/tantivy/?rev=ecb9a89#ecb9a89a9ff0573fba5c2cf8b83f353f35022310"
+source = "git+https://github.com/quickwit-oss/tantivy/?rev=927b443#927b4432c935b55cb8a2275daea3f77fc6823bd6"
 dependencies = [
  "async-trait",
  "byteorder",
@@ -7415,7 +7612,7 @@ dependencies = [
 [[package]]
 name = "tantivy-query-grammar"
 version = "0.21.0"
-source = "git+https://github.com/quickwit-oss/tantivy/?rev=ecb9a89#ecb9a89a9ff0573fba5c2cf8b83f353f35022310"
+source = "git+https://github.com/quickwit-oss/tantivy/?rev=927b443#927b4432c935b55cb8a2275daea3f77fc6823bd6"
 dependencies = [
  "nom",
 ]
@@ -7423,7 +7620,7 @@ dependencies = [
 [[package]]
 name = "tantivy-sstable"
 version = "0.2.0"
-source = "git+https://github.com/quickwit-oss/tantivy/?rev=ecb9a89#ecb9a89a9ff0573fba5c2cf8b83f353f35022310"
+source = "git+https://github.com/quickwit-oss/tantivy/?rev=927b443#927b4432c935b55cb8a2275daea3f77fc6823bd6"
 dependencies = [
  "tantivy-common",
  "tantivy-fst",
@@ -7433,7 +7630,7 @@ dependencies = [
 [[package]]
 name = "tantivy-stacker"
 version = "0.2.0"
-source = "git+https://github.com/quickwit-oss/tantivy/?rev=ecb9a89#ecb9a89a9ff0573fba5c2cf8b83f353f35022310"
+source = "git+https://github.com/quickwit-oss/tantivy/?rev=927b443#927b4432c935b55cb8a2275daea3f77fc6823bd6"
 dependencies = [
  "murmurhash32",
  "tantivy-common",
@@ -7442,7 +7639,7 @@ dependencies = [
 [[package]]
 name = "tantivy-tokenizer-api"
 version = "0.2.0"
-source = "git+https://github.com/quickwit-oss/tantivy/?rev=ecb9a89#ecb9a89a9ff0573fba5c2cf8b83f353f35022310"
+source = "git+https://github.com/quickwit-oss/tantivy/?rev=927b443#927b4432c935b55cb8a2275daea3f77fc6823bd6"
 dependencies = [
  "serde",
 ]
@@ -7520,7 +7717,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -7572,14 +7769,15 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.26"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a79d09ac6b08c1ab3906a2f7cc2e81a0e27c7ae89c63812df75e52bef0751e07"
+checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5"
 dependencies = [
  "deranged",
  "itoa",
  "libc",
  "num_threads",
+ "powerfmt",
  "serde",
  "time-core",
  "time-macros",
@@ -7587,9 +7785,9 @@ dependencies = [
 
 [[package]]
 name = "time-core"
-version = "0.1.1"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
+checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
 
 [[package]]
 name = "time-fmt"
@@ -7603,9 +7801,9 @@ dependencies = [
 
 [[package]]
 name = "time-macros"
-version = "0.2.12"
+version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75c65469ed6b3a4809d987a41eb1dc918e9bc1d92211cbad7ae82931846f7451"
+checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20"
 dependencies = [
  "time-core",
 ]
@@ -7646,9 +7844,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.33.0"
+version = "1.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653"
+checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9"
 dependencies = [
  "backtrace",
  "bytes",
@@ -7676,13 +7874,13 @@ dependencies = [
 
 [[package]]
 name = "tokio-macros"
-version = "2.1.0"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
+checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -7963,7 +8161,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -8091,7 +8289,7 @@ checksum = "bfc13d450dc4a695200da3074dacf43d449b968baee95e341920e47f61a3b40f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -8108,6 +8306,12 @@ dependencies = [
  "serde_yaml 0.8.26",
 ]
 
+[[package]]
+name = "ucd-trie"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9"
+
 [[package]]
 name = "ulid"
 version = "1.1.0"
@@ -8178,6 +8382,16 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
 
+[[package]]
+name = "universal-hash"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea"
+dependencies = [
+ "crypto-common",
+ "subtle",
+]
+
 [[package]]
 name = "unsafe-libyaml"
 version = "0.2.9"
@@ -8269,7 +8483,7 @@ dependencies = [
  "proc-macro-error",
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -8278,9 +8492,10 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc"
 dependencies = [
- "getrandom 0.2.10",
+ "getrandom 0.2.11",
  "rand 0.8.5",
  "serde",
+ "wasm-bindgen",
 ]
 
 [[package]]
@@ -8289,23 +8504,6 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
 
-[[package]]
-name = "value"
-version = "0.1.0"
-source = "git+https://github.com/vectordotdev/vrl?rev=v0.3.0#113005bcee6cd7b5ea0a53a7db2fc45ba4bc4125"
-dependencies = [
- "bytes",
- "chrono",
- "once_cell",
- "ordered-float 3.9.2",
- "path",
- "regex",
- "serde",
- "serde_json",
- "snafu",
- "tracing",
-]
-
 [[package]]
 name = "vcpkg"
 version = "0.2.15"
@@ -8320,113 +8518,52 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
 [[package]]
 name = "vrl"
-version = "0.1.0"
-source = "git+https://github.com/vectordotdev/vrl?rev=v0.3.0#113005bcee6cd7b5ea0a53a7db2fc45ba4bc4125"
-dependencies = [
- "value",
- "vrl-compiler",
- "vrl-diagnostic",
-]
-
-[[package]]
-name = "vrl-compiler"
-version = "0.1.0"
-source = "git+https://github.com/vectordotdev/vrl?rev=v0.3.0#113005bcee6cd7b5ea0a53a7db2fc45ba4bc4125"
-dependencies = [
- "anymap",
- "bytes",
- "chrono",
- "chrono-tz",
- "dyn-clone",
- "getrandom 0.2.10",
- "indoc",
- "lalrpop-util",
- "ordered-float 3.9.2",
- "paste",
- "path",
- "regex",
- "serde",
- "snafu",
- "thiserror",
- "tracing",
- "value",
- "vrl-diagnostic",
- "vrl-parser",
-]
-
-[[package]]
-name = "vrl-core"
-version = "0.1.0"
-source = "git+https://github.com/vectordotdev/vrl?rev=v0.3.0#113005bcee6cd7b5ea0a53a7db2fc45ba4bc4125"
-dependencies = [
- "bytes",
- "chrono",
- "chrono-tz",
- "derivative",
- "nom",
- "ordered-float 3.9.2",
- "path",
- "serde",
- "serde_json",
- "snafu",
- "value",
- "vrl-diagnostic",
-]
-
-[[package]]
-name = "vrl-diagnostic"
-version = "0.1.0"
-source = "git+https://github.com/vectordotdev/vrl?rev=v0.3.0#113005bcee6cd7b5ea0a53a7db2fc45ba4bc4125"
-dependencies = [
- "codespan-reporting",
- "termcolor",
-]
-
-[[package]]
-name = "vrl-parser"
-version = "0.1.0"
-source = "git+https://github.com/vectordotdev/vrl?rev=v0.3.0#113005bcee6cd7b5ea0a53a7db2fc45ba4bc4125"
-dependencies = [
- "lalrpop",
- "lalrpop-util",
- "ordered-float 3.9.2",
- "paste",
- "path",
- "thiserror",
- "vrl-diagnostic",
-]
-
-[[package]]
-name = "vrl-stdlib"
-version = "0.1.0"
-source = "git+https://github.com/vectordotdev/vrl?rev=v0.3.0#113005bcee6cd7b5ea0a53a7db2fc45ba4bc4125"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8a93ee342590c4df0ff63961d7d76a347e0c7b6e6c0be4c001317ca1ff11b53"
 dependencies = [
  "aes",
+ "anymap",
  "base16",
  "base64 0.21.5",
  "bytes",
  "cbc",
  "cfb-mode",
+ "cfg-if",
+ "chacha20poly1305",
  "charset",
  "chrono",
+ "chrono-tz",
  "cidr-utils",
+ "codespan-reporting",
+ "community-id",
+ "crypto_secretbox",
  "csv",
  "ctr",
  "data-encoding",
  "dns-lookup",
+ "dyn-clone",
  "flate2",
+ "grok",
  "hex",
  "hmac",
  "hostname",
- "indexmap 1.9.3",
+ "indexmap 2.1.0",
  "indoc",
+ "itertools 0.11.0",
+ "lalrpop",
+ "lalrpop-util",
  "md-5",
  "nom",
  "ofb",
  "once_cell",
- "ordered-float 3.9.2",
- "path",
+ "onig",
+ "ordered-float 4.1.1",
+ "paste",
+ "peeking_take_while",
  "percent-encoding",
+ "pest",
+ "pest_derive",
  "quoted_printable",
  "rand 0.8.5",
  "regex",
@@ -8438,19 +8575,18 @@ dependencies = [
  "sha-1",
  "sha2",
  "sha3",
- "strip-ansi-escapes",
+ "snafu",
+ "strip-ansi-escapes 0.2.0",
  "syslog_loose",
+ "termcolor",
+ "thiserror",
  "tracing",
  "uaparser",
  "url",
  "utf8-width",
  "uuid",
- "value",
- "vrl-compiler",
- "vrl-core",
- "vrl-diagnostic",
  "woothee",
- "zstd 0.12.4",
+ "zstd 0.13.0",
 ]
 
 [[package]]
@@ -8470,6 +8606,16 @@ dependencies = [
  "vte_generate_state_changes",
 ]
 
+[[package]]
+name = "vte"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197"
+dependencies = [
+ "utf8parse",
+ "vte_generate_state_changes",
+]
+
 [[package]]
 name = "vte_generate_state_changes"
 version = "0.1.1"
@@ -8559,9 +8705,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.87"
+version = "0.2.88"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342"
+checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce"
 dependencies = [
  "cfg-if",
  "wasm-bindgen-macro",
@@ -8569,24 +8715,24 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.87"
+version = "0.2.88"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd"
+checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217"
 dependencies = [
  "bumpalo",
  "log",
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.37"
+version = "0.4.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03"
+checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02"
 dependencies = [
  "cfg-if",
  "js-sys",
@@ -8596,9 +8742,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.87"
+version = "0.2.88"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d"
+checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -8606,22 +8752,22 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.87"
+version = "0.2.88"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
+checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.87"
+version = "0.2.88"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
+checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b"
 
 [[package]]
 name = "wasm-streams"
@@ -8638,9 +8784,9 @@ dependencies = [
 
 [[package]]
 name = "web-sys"
-version = "0.3.64"
+version = "0.3.65"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b"
+checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -8898,9 +9044,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
 
 [[package]]
 name = "winnow"
-version = "0.5.17"
+version = "0.5.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3b801d0e0a6726477cc207f60162da452f3a95adb368399bef20a946e06f65c"
+checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b"
 dependencies = [
  "memchr",
 ]
@@ -8917,9 +9063,9 @@ dependencies = [
 
 [[package]]
 name = "wiremock"
-version = "0.5.19"
+version = "0.5.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6f71803d3a1c80377a06221e0530be02035d5b3e854af56c6ece7ac20ac441d"
+checksum = "079aee011e8a8e625d16df9e785de30a6b77f80a6126092d76a57375f96448da"
 dependencies = [
  "assert-json-diff 2.0.2",
  "async-trait",
@@ -8994,22 +9140,22 @@ checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec"
 
 [[package]]
 name = "zerocopy"
-version = "0.7.16"
+version = "0.7.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c552e97c5a9b90bc8ddc545b5106e798807376356688ebaa3aee36f44f8c4b9e"
+checksum = "8cd369a67c0edfef15010f980c3cbe45d7f651deac2cd67ce097cd801de16557"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.7.16"
+version = "0.7.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "964bc0588d7ac1c0243d0427ef08482618313702bbb014806cb7ab3da34d3d99"
+checksum = "c2f140bda219a26ccc0cdb03dba58af72590c53b22642577d88a927bc5c87d6b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -9047,15 +9193,6 @@ dependencies = [
  "zstd-safe 5.0.2+zstd.1.5.2",
 ]
 
-[[package]]
-name = "zstd"
-version = "0.12.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c"
-dependencies = [
- "zstd-safe 6.0.6",
-]
-
 [[package]]
 name = "zstd"
 version = "0.13.0"
@@ -9075,16 +9212,6 @@ dependencies = [
  "zstd-sys",
 ]
 
-[[package]]
-name = "zstd-safe"
-version = "6.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581"
-dependencies = [
- "libc",
- "zstd-sys",
-]
-
 [[package]]
 name = "zstd-safe"
 version = "7.0.0"
diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml
index 6ded4a0bb37..2b0d55fad81 100644
--- a/quickwit/Cargo.toml
+++ b/quickwit/Cargo.toml
@@ -46,11 +46,14 @@ async-speed-limit = "0.4"
 async-trait = "0.1"
 backoff = { version = "0.4", features = ["tokio"] }
 base64 = "0.21"
-byte-unit = { version = "4", default-features = false, features = ["serde", "std"] }
 bytes = { version = "1", features = ["serde"] }
+bytesize = { version = "1.3.0", features = ["serde"] }
 bytestring = "1.3.0"
-chitchat = "0.6"
-chrono = { version = "0.4.23", default-features = false, features = ["clock", "std"] }
+chitchat = "0.7"
+chrono = { version = "0.4.23", default-features = false, features = [
+  "clock",
+  "std",
+] }
 clap = { version = "4.4.1", features = ["env", "string"] }
 colored = "2.0.0"
 console-subscriber = "0.1.8"
@@ -96,12 +99,19 @@ libz-sys = "1.1.8"
 lru = "0.12"
 lindera-core = "0.27.0"
 lindera-dictionary = "0.27.0"
-lindera-tokenizer = { version = "0.27.0", features = ["ipadic", "ipadic-compress", "cc-cedict", "cc-cedict-compress", "ko-dic", "ko-dic-compress"] }
+lindera-tokenizer = { version = "0.27.0", features = [
+  "cc-cedict-compress",
+  "cc-cedict",
+  "ipadic-compress",
+  "ipadic",
+  "ko-dic-compress",
+  "ko-dic",
+] }
 matches = "0.1.9"
 md5 = "0.7"
 mime_guess = "2.0.4"
 mockall = "0.11"
-mrecordlog = { git = "https://github.com/quickwit-oss/mrecordlog", rev = "0d1a7aa" }
+mrecordlog = { git = "https://github.com/quickwit-oss/mrecordlog", rev = "ebee0fd" }
 new_string_template = "1.4.0"
 nom = "7.1.3"
 num_cpus = "1"
@@ -113,7 +123,9 @@ opentelemetry = { version = "0.19", features = ["rt-tokio"] }
 opentelemetry-otlp = "0.12.0"
 pin-project = "1.1.0"
 pnet = { version = "0.33.0", features = ["std"] }
-postcard = { version = "1.0.4", features = ["use-std"], default-features = false}
+postcard = { version = "1.0.4", features = [
+  "use-std",
+], default-features = false }
 predicates = "3"
 prettyplease = "0.2.0"
 proc-macro2 = "1.0.50"
@@ -124,7 +136,11 @@ prost = { version = "0.11.6", default-features = false, features = [
 ] }
 prost-build = "0.11.6"
 prost-types = "0.11.6"
-pulsar = { git = "https://github.com/quickwit-oss/pulsar-rs.git", rev = "f9eff04", default-features = false, features = ["compression", "tokio-runtime", "auth-oauth2"] }
+pulsar = { git = "https://github.com/quickwit-oss/pulsar-rs.git", rev = "f9eff04", default-features = false, features = [
+  "auth-oauth2",
+  "compression",
+  "tokio-runtime",
+] }
 quote = "1.0.23"
 rand = "0.8"
 rand_distr = "0.4"
@@ -143,20 +159,24 @@ reqwest = { version = "0.11", default-features = false, features = [
 ] }
 rust-embed = "6.8.1"
 sea-query = { version = "0" }
-sea-query-binder = { version = "0", features = ["sqlx-postgres", "runtime-tokio-rustls",] }
-serde = { version = "= 1.0.171", features = ["derive", "rc"] }
+sea-query-binder = { version = "0", features = [
+  "runtime-tokio-rustls",
+  "sqlx-postgres",
+] }
+# ^1.0.184 due to serde-rs/serde#2538
+serde = { version = "1.0.184", features = ["derive", "rc"] }
 serde_json = "1.0"
 serde_qs = { version = "0.12", features = ["warp"] }
 serde_with = "3.4.0"
 serde_yaml = "0.9"
 siphasher = "0.3"
 sqlx = { version = "0.7", features = [
-  "runtime-tokio-rustls",
-  "postgres",
   "migrate",
+  "postgres",
+  "runtime-tokio-rustls",
   "time",
 ] }
-syn = { version = "2.0.11", features = [ "extra-traits", "full", "parsing" ]}
+syn = { version = "2.0.11", features = ["extra-traits", "full", "parsing"] }
 sync_wrapper = "0.1.2"
 tabled = { version = "0.8", features = ["color"] }
 tempfile = "3"
@@ -166,20 +186,26 @@ thousands = "0.2.0"
 tikv-jemalloc-ctl = "0.5"
 tikv-jemallocator = "0.5"
 time = { version = "0.3.17", features = ["std", "formatting", "macros"] }
-tokio = { version = "1.33", features = ["full"] }
+tokio = { version = "1.34", features = ["full"] }
 tokio-stream = { version = "0.1", features = ["sync"] }
 tokio-util = { version = "0.7", features = ["full"] }
 toml = "0.7.6"
 tonic = { version = "0.9.0", features = ["gzip"] }
 tonic-build = "0.9.0"
-tower = { version = "0.4.13", features = ["balance", "buffer", "load", "retry", "util"] }
+tower = { version = "0.4.13", features = [
+  "balance",
+  "buffer",
+  "load",
+  "retry",
+  "util",
+] }
 tower-http = { version = "0.4.0", features = ["compression-gzip", "cors"] }
 tracing = "0.1.37"
 tracing-opentelemetry = "0.19.0"
 tracing-subscriber = { version = "0.3.16", features = [
-  "time",
-  "std",
   "env-filter",
+  "std",
+  "time",
 ] }
 ttl_cache = "0.5"
 typetag = "0.2"
@@ -187,12 +213,20 @@ ulid = "1.1"
 username = "0.2"
 utoipa = "3.5.0"
 uuid = { version = "1.5", features = ["v4", "serde"] }
+vrl = { version = "0.8.1", default-features = false, features = [
+  "compiler",
+  "diagnostic",
+  "stdlib",
+  "value",
+] }
 warp = "0.3"
 whichlang = { git = "https://github.com/quickwit-oss/whichlang", rev = "fe406416" }
 wiremock = "0.5"
 
 aws-config = "0.55.0"
-aws-credential-types = { version = "0.55.0", features = ["hardcoded-credentials"] }
+aws-credential-types = { version = "0.55.0", features = [
+  "hardcoded-credentials",
+] }
 aws-sdk-kinesis = "0.28.0"
 aws-sdk-s3 = "0.28.0"
 aws-smithy-async = "0.55.0"
@@ -202,8 +236,12 @@ aws-smithy-types = "0.55.0"
 aws-types = "0.55.0"
 
 azure_core = { version = "0.13.0", features = ["enable_reqwest_rustls"] }
-azure_storage = { version = "0.13.0", default-features = false, features = ["enable_reqwest_rustls"] }
-azure_storage_blobs = { version = "0.13.0", default-features = false, features = ["enable_reqwest_rustls"] }
+azure_storage = { version = "0.13.0", default-features = false, features = [
+  "enable_reqwest_rustls",
+] }
+azure_storage_blobs = { version = "0.13.0", default-features = false, features = [
+  "enable_reqwest_rustls",
+] }
 
 quickwit-actors = { version = "0.6.5-dev", path = "./quickwit-actors" }
 quickwit-aws = { version = "0.6.5-dev", path = "./quickwit-aws" }
@@ -234,179 +272,17 @@ quickwit-serve = { version = "0.6.5-dev", path = "./quickwit-serve" }
 quickwit-storage = { version = "0.6.5-dev", path = "./quickwit-storage" }
 quickwit-telemetry = { version = "0.6.5-dev", path = "./quickwit-telemetry" }
 
-tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "ecb9a89", default-features = false, features = [
-  "mmap",
+tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "927b443", default-features = false, features = [
   "lz4-compression",
-  "zstd-compression",
+  "mmap",
   "quickwit",
+  "zstd-compression",
 ] }
 
 # This is actually not used directly the goal is to fix the version
 # used by reqwest.
 encoding_rs = "=0.8.32"
 
-# vrl deps, at the end because the feature list for vrl-stdlib is long
-vrl = { git = "https://github.com/vectordotdev/vrl", rev = "v0.3.0", default-features = false, features = [
-  "compiler",
-  "diagnostic",
-  "value",
-] }
-# opt out of vrl-stdlib parse-grok* default features
-vrl-stdlib = { git = "https://github.com/vectordotdev/vrl", rev = "v0.3.0", default-features = false, features = [
-  "abs",
-  "append",
-  "array",
-  "assert",
-  "assert_eq",
-  "boolean",
-  "ceil",
-  "chunks",
-  "compact",
-  "contains",
-  "decode_base16",
-  "decode_base64",
-  "decode_gzip",
-  "decode_mime_q",
-  "decode_percent",
-  "decode_zlib",
-  "decode_zstd",
-  "decrypt",
-  "del",
-  "downcase",
-  "encode_base16",
-  "encode_base64",
-  "encode_gzip",
-  "encode_json",
-  "encode_key_value",
-  "encode_logfmt",
-  "encode_percent",
-  "encode_zlib",
-  "encode_zstd",
-  "encrypt",
-  "ends_with",
-  "exists",
-  "filter",
-  "find",
-  "flatten",
-  "float",
-  "floor",
-  "for_each",
-  "format_int",
-  "format_number",
-  "format_timestamp",
-  "get",
-  "get_env_var",
-  "get_hostname",
-  "hmac",
-  "includes",
-  "integer",
-  "ip_aton",
-  "ip_cidr_contains",
-  "ip_ntoa",
-  "ip_ntop",
-  "ip_pton",
-  "ip_subnet",
-  "ip_to_ipv6",
-  "ipv6_to_ipv4",
-  "is_array",
-  "is_boolean",
-  "is_empty",
-  "is_float",
-  "is_integer",
-  "is_ipv4",
-  "is_ipv6",
-  "is_json",
-  "is_null",
-  "is_nullish",
-  "is_object",
-  "is_regex",
-  "is_string",
-  "is_timestamp",
-  "join",
-  "keys",
-  "length",
-  "log",
-  "map_keys",
-  "map_values",
-  "match",
-  "match_any",
-  "match_array",
-  "md5",
-  "merge",
-  "mod",
-  "now",
-  "object",
-  "parse_apache_log",
-  "parse_aws_alb_log",
-  "parse_aws_cloudwatch_log_subscription_message",
-  "parse_aws_vpc_flow_log",
-  "parse_cef",
-  "parse_common_log",
-  "parse_csv",
-  "parse_duration",
-  "parse_glog",
-  "parse_int",
-  "parse_json",
-  "parse_key_value",
-  "parse_klog",
-  "parse_linux_authorization",
-  "parse_logfmt",
-  "parse_nginx_log",
-  "parse_query_string",
-  "parse_regex",
-  "parse_regex_all",
-  "parse_ruby_hash",
-  "parse_syslog",
-  "parse_timestamp",
-  "parse_tokens",
-  "parse_url",
-  "parse_user_agent",
-  "parse_xml",
-  "push",
-  "random_bool",
-  "random_bytes",
-  "random_float",
-  "random_int",
-  "redact",
-  "remove",
-  "replace",
-  "reverse_dns",
-  "round",
-  "seahash",
-  "set",
-  "sha1",
-  "sha2",
-  "sha3",
-  "slice",
-  "split",
-  "starts_with",
-  "string",
-  "strip_ansi_escape_codes",
-  "strip_whitespace",
-  "strlen",
-  "tag_types_externally",
-  "tally",
-  "tally_value",
-  "timestamp",
-  "to_bool",
-  "to_float",
-  "to_int",
-  "to_regex",
-  "to_string",
-  "to_syslog_facility",
-  "to_syslog_level",
-  "to_syslog_severity",
-  "to_timestamp",
-  "to_unix_timestamp",
-  "truncate",
-  "type_def",
-  "unique",
-  "unnest",
-  "upcase",
-  "uuid_v4",
-  "values"
-]}
-
 [patch.crates-io]
 sasl2-sys = { git = "https://github.com/quickwit-oss/rust-sasl/", rev = "daca921" }
 
diff --git a/quickwit/quickwit-cli/Cargo.toml b/quickwit/quickwit-cli/Cargo.toml
index 6fc245b41aa..30d0d44a8de 100644
--- a/quickwit/quickwit-cli/Cargo.toml
+++ b/quickwit/quickwit-cli/Cargo.toml
@@ -21,7 +21,7 @@ path = "src/generate_markdown.rs"
 [dependencies]
 anyhow = { workspace = true }
 async-trait = { workspace = true }
-byte-unit = { workspace = true }
+bytesize = { workspace = true }
 bytes = { workspace = true }
 chitchat = { workspace = true }
 clap = { workspace = true }
diff --git a/quickwit/quickwit-cli/src/index.rs b/quickwit/quickwit-cli/src/index.rs
index ce07f236224..809622a302b 100644
--- a/quickwit/quickwit-cli/src/index.rs
+++ b/quickwit/quickwit-cli/src/index.rs
@@ -25,9 +25,9 @@ use std::str::FromStr;
 use std::time::{Duration, Instant};
 use std::{fmt, io};
 
-use anyhow::{bail, Context};
-use byte_unit::Byte;
+use anyhow::{anyhow, bail, Context};
 use bytes::Bytes;
+use bytesize::ByteSize;
 use clap::{arg, Arg, ArgAction, ArgMatches, Command};
 use colored::{ColoredString, Colorize};
 use humantime::format_duration;
@@ -130,6 +130,12 @@ pub fn build_index_command() -> Command {
                         .short('w')
                         .help("Wait for all documents to be commited and available for search before exiting")
                         .action(ArgAction::SetTrue),
+                    // TODO remove me after Quickwit 0.7.
+                    Arg::new("v2")
+                        .long("v2")
+                        .help("Ingest v2 (experimental! Do not use me.)")
+                        .hide(true)
+                        .action(ArgAction::SetTrue),
                     Arg::new("force")
                         .long("force")
                         .short('f')
@@ -205,7 +211,7 @@ pub struct IngestDocsArgs {
     pub client_args: ClientArgs,
     pub index_id: String,
     pub input_path_opt: Option<PathBuf>,
-    pub batch_size_limit_opt: Option<Byte>,
+    pub batch_size_limit_opt: Option<ByteSize>,
     pub commit_type: CommitType,
 }
 
@@ -333,8 +339,9 @@ impl IndexCliCommand {
 
         let batch_size_limit_opt = matches
             .remove_one::<String>("batch-size-limit")
-            .map(Byte::from_str)
-            .transpose()?;
+            .map(|limit| limit.parse::<ByteSize>())
+            .transpose()
+            .map_err(|error| anyhow!(error))?;
         let commit_type = match (matches.get_flag("wait"), matches.get_flag("force")) {
             (false, false) => CommitType::Auto,
             (false, true) => CommitType::Force,
@@ -527,9 +534,9 @@ pub struct IndexStats {
     pub index_id: String,
     pub index_uri: Uri,
     pub num_published_splits: usize,
-    pub size_published_splits: Byte,
+    pub size_published_splits: ByteSize,
     pub num_published_docs: u64,
-    pub size_published_docs_uncompressed: Byte,
+    pub size_published_docs_uncompressed: ByteSize,
     pub timestamp_field_name: Option<String>,
     pub timestamp_range: Option<(i64, i64)>,
     pub num_docs_descriptive: Option<DescriptiveStats>,
@@ -544,13 +551,9 @@ impl Tabled for IndexStats {
             self.index_id.clone(),
             self.index_uri.to_string(),
             self.num_published_docs.to_string(),
-            self.size_published_docs_uncompressed
-                .get_appropriate_unit(false)
-                .to_string(),
+            self.size_published_docs_uncompressed.to_string(),
             self.num_published_splits.to_string(),
-            self.size_published_splits
-                .get_appropriate_unit(false)
-                .to_string(),
+            self.size_published_splits.to_string(),
             display_option_in_table(&self.timestamp_field_name),
             display_timestamp_range(&self.timestamp_range),
         ]
@@ -653,9 +656,9 @@ impl IndexStats {
             index_id: index_config.index_id.clone(),
             index_uri: index_config.index_uri.clone(),
             num_published_splits: published_splits.len(),
-            size_published_splits: Byte::from(total_num_bytes),
+            size_published_splits: ByteSize(total_num_bytes),
             num_published_docs: total_num_docs,
-            size_published_docs_uncompressed: Byte::from(total_uncompressed_num_bytes),
+            size_published_docs_uncompressed: ByteSize(total_uncompressed_num_bytes),
             timestamp_field_name: index_config.doc_mapping.timestamp_field,
             timestamp_range,
             num_docs_descriptive,
@@ -804,7 +807,7 @@ pub async fn ingest_docs_cli(args: IngestDocsArgs) -> anyhow::Result<()> {
     };
     let batch_size_limit_opt = args
         .batch_size_limit_opt
-        .map(|batch_size_limit| batch_size_limit.get_bytes() as usize);
+        .map(|batch_size_limit| batch_size_limit.as_u64() as usize);
     qw_client
         .ingest(
             &args.index_id,
@@ -1126,13 +1129,13 @@ mod test {
 
         let split_data_1 = Split {
             split_metadata: split_metadata_1,
-            split_state: quickwit_metastore::SplitState::Published,
+            split_state: SplitState::Published,
             update_timestamp: 0,
             publish_timestamp: Some(10),
         };
         let split_data_2 = Split {
             split_metadata: split_metadata_2,
-            split_state: quickwit_metastore::SplitState::MarkedForDeletion,
+            split_state: SplitState::MarkedForDeletion,
             update_timestamp: 0,
             publish_timestamp: Some(10),
         };
@@ -1143,14 +1146,11 @@ mod test {
         assert_eq!(index_stats.index_id, index_id);
         assert_eq!(index_stats.index_uri.as_str(), index_uri);
         assert_eq!(index_stats.num_published_splits, 1);
-        assert_eq!(
-            index_stats.size_published_splits,
-            Byte::from(15_000_000usize)
-        );
+        assert_eq!(index_stats.size_published_splits, ByteSize::mb(15));
         assert_eq!(index_stats.num_published_docs, 100_000);
         assert_eq!(
             index_stats.size_published_docs_uncompressed,
-            Byte::from(19_000_000usize)
+            ByteSize::mb(19)
         );
         assert_eq!(
             index_stats.timestamp_field_name,
@@ -1165,7 +1165,7 @@ mod test {
     fn test_descriptive_stats() -> anyhow::Result<()> {
         let split_id = "stat-test-split".to_string();
         let template_split = Split {
-            split_state: quickwit_metastore::SplitState::Published,
+            split_state: SplitState::Published,
             update_timestamp: 10,
             publish_timestamp: Some(10),
             split_metadata: SplitMetadata::default(),
@@ -1201,12 +1201,12 @@ mod test {
 
         let num_docs_descriptive = DescriptiveStats::maybe_new(&splits_num_docs);
         let num_bytes_descriptive = DescriptiveStats::maybe_new(&splits_bytes);
-        let desciptive_stats_none = DescriptiveStats::maybe_new(&[]);
+        let descriptive_stats_none = DescriptiveStats::maybe_new(&[]);
 
         assert!(num_docs_descriptive.is_some());
         assert!(num_bytes_descriptive.is_some());
 
-        assert!(desciptive_stats_none.is_none());
+        assert!(descriptive_stats_none.is_none());
 
         Ok(())
     }
diff --git a/quickwit/quickwit-cli/src/lib.rs b/quickwit/quickwit-cli/src/lib.rs
index 3c5cdafc366..675aa20245e 100644
--- a/quickwit/quickwit-cli/src/lib.rs
+++ b/quickwit/quickwit-cli/src/lib.rs
@@ -106,6 +106,7 @@ pub struct ClientArgs {
     pub connect_timeout: Option<Timeout>,
     pub timeout: Option<Timeout>,
     pub commit_timeout: Option<Timeout>,
+    pub ingest_v2: bool,
 }
 
 impl Default for ClientArgs {
@@ -115,6 +116,7 @@ impl Default for ClientArgs {
             connect_timeout: None,
             timeout: None,
             commit_timeout: None,
+            ingest_v2: false,
         }
     }
 }
@@ -128,6 +130,9 @@ impl ClientArgs {
         if let Some(timeout) = self.timeout {
             builder = builder.timeout(timeout);
         }
+        if self.ingest_v2 {
+            builder = builder.enable_ingest_v2();
+        }
         builder.build()
     }
 
@@ -180,6 +185,11 @@ impl ClientArgs {
         } else {
             None
         };
+        let ingest_v2 = if process_ingest {
+            matches.get_flag("v2")
+        } else {
+            false
+        };
         let commit_timeout = if process_ingest {
             if let Some(duration) = matches.remove_one::<String>("commit-timeout") {
                 Some(parse_duration_or_none(&duration)?)
@@ -194,6 +204,7 @@ impl ClientArgs {
             connect_timeout,
             timeout,
             commit_timeout,
+            ingest_v2,
         })
     }
 }
diff --git a/quickwit/quickwit-cli/src/main.rs b/quickwit/quickwit-cli/src/main.rs
index 5d8ec6676da..bbe3bd90f72 100644
--- a/quickwit/quickwit-cli/src/main.rs
+++ b/quickwit/quickwit-cli/src/main.rs
@@ -92,7 +92,7 @@ mod tests {
     use std::str::FromStr;
     use std::time::Duration;
 
-    use byte_unit::Byte;
+    use bytesize::ByteSize;
     use quickwit_cli::cli::{build_cli, CliCommand};
     use quickwit_cli::index::{
         ClearIndexArgs, CreateIndexArgs, DeleteIndexArgs, DescribeIndexArgs, IndexCliCommand,
@@ -181,6 +181,32 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn test_parse_ingest_v2_args() {
+        let app = build_cli().no_binary_name(true);
+        let matches = app
+            .try_get_matches_from(["index", "ingest", "--index", "wikipedia", "--v2"])
+            .unwrap();
+        let command = CliCommand::parse_cli_args(matches).unwrap();
+        assert!(matches!(
+            command,
+            CliCommand::Index(IndexCliCommand::Ingest(
+                IngestDocsArgs {
+                    client_args,
+                    index_id,
+                    input_path_opt: None,
+                    batch_size_limit_opt: None,
+                    commit_type: CommitType::Auto,
+                })) if &index_id == "wikipedia"
+                && client_args.timeout.is_none()
+                && client_args.connect_timeout.is_none()
+                && client_args.commit_timeout.is_none()
+                && client_args.cluster_endpoint == Url::from_str("http://127.0.0.1:7280").unwrap()
+                && client_args.ingest_v2
+
+        ));
+    }
+
     #[test]
     fn test_parse_ingest_args() -> anyhow::Result<()> {
         let app = build_cli().no_binary_name(true);
@@ -207,6 +233,7 @@ mod tests {
                 && client_args.connect_timeout.is_none()
                 && client_args.commit_timeout.is_none()
                 && client_args.cluster_endpoint == Url::from_str("http://127.0.0.1:8000").unwrap()
+                && !client_args.ingest_v2
         ));
 
         let app = build_cli().no_binary_name(true);
@@ -234,8 +261,8 @@ mod tests {
                         && client_args.timeout.is_none()
                         && client_args.connect_timeout.is_none()
                         && client_args.commit_timeout.is_none()
-                        && batch_size_limit == Byte::from_str("8MB").unwrap()
-
+                        && !client_args.ingest_v2
+                        && batch_size_limit == ByteSize::mb(8)
         ));
 
         let app = build_cli().no_binary_name(true);
@@ -263,7 +290,8 @@ mod tests {
                     && client_args.timeout.is_none()
                     && client_args.connect_timeout.is_none()
                     && client_args.commit_timeout.is_none()
-                    && batch_size_limit == Byte::from_str("4KB").unwrap()
+                    && !client_args.ingest_v2
+                    && batch_size_limit == ByteSize::kb(4)
         ));
 
         let app = build_cli().no_binary_name(true);
diff --git a/quickwit/quickwit-cli/src/tool.rs b/quickwit/quickwit-cli/src/tool.rs
index bcc5eff3868..ecacae7167b 100644
--- a/quickwit/quickwit-cli/src/tool.rs
+++ b/quickwit/quickwit-cli/src/tool.rs
@@ -49,6 +49,7 @@ use quickwit_indexing::models::{
 use quickwit_indexing::IndexingPipeline;
 use quickwit_ingest::IngesterPool;
 use quickwit_metastore::IndexMetadataResponseExt;
+use quickwit_proto::indexing::CpuCapacity;
 use quickwit_proto::metastore::{IndexMetadataRequest, MetastoreService, MetastoreServiceClient};
 use quickwit_proto::search::{CountHits, SearchResponse};
 use quickwit_proto::types::NodeId;
@@ -690,7 +691,7 @@ pub async fn garbage_collect_index_cli(args: GarbageCollectIndexArgs) -> anyhow:
     let deleted_bytes: u64 = removal_info
         .removed_split_entries
         .iter()
-        .map(|split_info| split_info.file_size_bytes.get_bytes())
+        .map(|split_info| split_info.file_size_bytes.as_u64())
         .sum();
     println!(
         "{}MB of storage garbage collected.",
@@ -931,15 +932,16 @@ impl ThroughputCalculator {
 
 async fn create_empty_cluster(config: &NodeConfig) -> anyhow::Result<Cluster> {
     let node_id: NodeId = config.node_id.clone().into();
-    let self_node = ClusterMember::new(
+    let self_node = ClusterMember {
         node_id,
-        quickwit_cluster::GenerationId::now(),
-        false,
-        HashSet::new(),
-        config.gossip_advertise_addr,
-        config.grpc_advertise_addr,
-        Vec::new(),
-    );
+        generation_id: quickwit_cluster::GenerationId::now(),
+        is_ready: false,
+        enabled_services: HashSet::new(),
+        gossip_advertise_addr: config.gossip_advertise_addr,
+        grpc_advertise_addr: config.grpc_advertise_addr,
+        indexing_cpu_capacity: CpuCapacity::zero(),
+        indexing_tasks: Vec::new(),
+    };
     let cluster = Cluster::join(
         config.cluster_id.clone(),
         self_node,
@@ -949,5 +951,6 @@ async fn create_empty_cluster(config: &NodeConfig) -> anyhow::Result<Cluster> {
         &ChannelTransport::default(),
     )
     .await?;
+
     Ok(cluster)
 }
diff --git a/quickwit/quickwit-cluster/src/cluster.rs b/quickwit/quickwit-cluster/src/cluster.rs
index 53a6c0dbbd5..0c9f95533cf 100644
--- a/quickwit/quickwit-cluster/src/cluster.rs
+++ b/quickwit/quickwit-cluster/src/cluster.rs
@@ -18,7 +18,7 @@
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 use std::collections::{BTreeMap, HashMap, HashSet};
-use std::fmt::Debug;
+use std::fmt::{Debug, Display};
 use std::net::SocketAddr;
 use std::sync::Arc;
 use std::time::Duration;
@@ -229,7 +229,7 @@ impl Cluster {
     }
 
     /// Sets a key-value pair on the cluster node's state.
-    pub async fn set_self_key_value<K: Into<String>, V: Into<String>>(&self, key: K, value: V) {
+    pub async fn set_self_key_value(&self, key: impl Display, value: impl Display) {
         self.chitchat()
             .await
             .lock()
@@ -517,17 +517,19 @@ pub async fn create_cluster_for_test_with_id(
     transport: &dyn Transport,
     self_node_readiness: bool,
 ) -> anyhow::Result<Cluster> {
+    use quickwit_proto::indexing::PIPELINE_FULL_CAPACITY;
     let gossip_advertise_addr: SocketAddr = ([127, 0, 0, 1], node_id).into();
     let node_id: NodeId = format!("node_{node_id}").into();
-    let self_node = ClusterMember::new(
+    let self_node = ClusterMember {
         node_id,
-        crate::GenerationId(1),
-        self_node_readiness,
-        enabled_services.clone(),
+        generation_id: crate::GenerationId(1),
+        is_ready: self_node_readiness,
+        enabled_services: enabled_services.clone(),
         gossip_advertise_addr,
-        grpc_addr_from_listen_addr_for_test(gossip_advertise_addr),
-        Vec::new(),
-    );
+        grpc_advertise_addr: grpc_addr_from_listen_addr_for_test(gossip_advertise_addr),
+        indexing_tasks: Vec::new(),
+        indexing_cpu_capacity: PIPELINE_FULL_CAPACITY,
+    };
     let failure_detector_config = create_failure_detector_config_for_test();
     let cluster = Cluster::join(
         cluster_id,
diff --git a/quickwit/quickwit-cluster/src/lib.rs b/quickwit/quickwit-cluster/src/lib.rs
index e1e4afc180c..5821d41c65d 100644
--- a/quickwit/quickwit-cluster/src/lib.rs
+++ b/quickwit/quickwit-cluster/src/lib.rs
@@ -30,6 +30,7 @@ use chitchat::transport::UdpTransport;
 use chitchat::FailureDetectorConfig;
 use quickwit_config::service::QuickwitService;
 use quickwit_config::NodeConfig;
+use quickwit_proto::indexing::CpuCapacity;
 use quickwit_proto::types::NodeId;
 use time::OffsetDateTime;
 
@@ -37,7 +38,7 @@ pub use crate::change::ClusterChange;
 #[cfg(any(test, feature = "testsuite"))]
 pub use crate::cluster::{create_cluster_for_test, grpc_addr_from_listen_addr_for_test};
 pub use crate::cluster::{Cluster, ClusterSnapshot, NodeIdSchema};
-pub use crate::member::ClusterMember;
+pub use crate::member::{ClusterMember, INDEXING_CPU_CAPACITY_KEY};
 pub use crate::node::ClusterNode;
 
 #[derive(Debug, Clone, Copy, Eq, PartialEq)]
@@ -68,15 +69,21 @@ pub async fn start_cluster_service(node_config: &NodeConfig) -> anyhow::Result<C
     let node_id: NodeId = node_config.node_id.clone().into();
     let generation_id = GenerationId::now();
     let is_ready = false;
-    let self_node = ClusterMember::new(
+    let indexing_cpu_capacity = if node_config.is_service_enabled(QuickwitService::Indexer) {
+        node_config.indexer_config.cpu_capacity
+    } else {
+        CpuCapacity::zero()
+    };
+    let self_node = ClusterMember {
         node_id,
         generation_id,
         is_ready,
-        node_config.enabled_services.clone(),
-        node_config.gossip_advertise_addr,
-        node_config.grpc_advertise_addr,
+        enabled_services: node_config.enabled_services.clone(),
+        gossip_advertise_addr: node_config.gossip_advertise_addr,
+        grpc_advertise_addr: node_config.grpc_advertise_addr,
         indexing_tasks,
-    );
+        indexing_cpu_capacity,
+    };
     let cluster = Cluster::join(
         cluster_id,
         self_node,
@@ -86,5 +93,13 @@ pub async fn start_cluster_service(node_config: &NodeConfig) -> anyhow::Result<C
         &UdpTransport,
     )
     .await?;
+    if node_config
+        .enabled_services
+        .contains(&QuickwitService::Indexer)
+    {
+        cluster
+            .set_self_key_value(INDEXING_CPU_CAPACITY_KEY, indexing_cpu_capacity)
+            .await;
+    }
     Ok(cluster)
 }
diff --git a/quickwit/quickwit-cluster/src/member.rs b/quickwit/quickwit-cluster/src/member.rs
index 728546134e7..2bdfb085d6d 100644
--- a/quickwit/quickwit-cluster/src/member.rs
+++ b/quickwit/quickwit-cluster/src/member.rs
@@ -19,13 +19,14 @@
 
 use std::collections::HashSet;
 use std::net::SocketAddr;
+use std::str::FromStr;
 
 use anyhow::{anyhow, Context};
 use chitchat::{ChitchatId, NodeState};
 use itertools::Itertools;
-use quickwit_proto::indexing::IndexingTask;
+use quickwit_proto::indexing::{CpuCapacity, IndexingTask};
 use quickwit_proto::types::NodeId;
-use tracing::warn;
+use tracing::{error, warn};
 
 use crate::{GenerationId, QuickwitService};
 
@@ -43,6 +44,8 @@ pub(crate) const READINESS_KEY: &str = "readiness";
 pub(crate) const READINESS_VALUE_READY: &str = "READY";
 pub(crate) const READINESS_VALUE_NOT_READY: &str = "NOT_READY";
 
+pub const INDEXING_CPU_CAPACITY_KEY: &str = "indexing_cpu_capacity";
+
 pub(crate) trait NodeStateExt {
     fn grpc_advertise_addr(&self) -> anyhow::Result<SocketAddr>;
 
@@ -91,30 +94,12 @@ pub struct ClusterMember {
     /// None if the node is not an indexer or the indexer has not yet started some indexing
     /// pipelines.
     pub indexing_tasks: Vec<IndexingTask>,
+    /// Indexing cpu capacity of the node expressed in milli cpu.
+    pub indexing_cpu_capacity: CpuCapacity,
     pub is_ready: bool,
 }
 
 impl ClusterMember {
-    pub fn new(
-        node_id: NodeId,
-        generation_id: GenerationId,
-        is_ready: bool,
-        enabled_services: HashSet<QuickwitService>,
-        gossip_advertise_addr: SocketAddr,
-        grpc_advertise_addr: SocketAddr,
-        indexing_tasks: Vec<IndexingTask>,
-    ) -> Self {
-        Self {
-            node_id,
-            generation_id,
-            is_ready,
-            enabled_services,
-            gossip_advertise_addr,
-            grpc_advertise_addr,
-            indexing_tasks,
-        }
-    }
-
     pub fn chitchat_id(&self) -> ChitchatId {
         ChitchatId::new(
             self.node_id.clone().into(),
@@ -130,6 +115,18 @@ impl From<ClusterMember> for ChitchatId {
     }
 }
 
+fn parse_indexing_cpu_capacity(node_state: &NodeState) -> CpuCapacity {
+    let Some(indexing_capacity_str) = node_state.get(INDEXING_CPU_CAPACITY_KEY) else {
+        return CpuCapacity::zero();
+    };
+    if let Ok(indexing_capacity) = CpuCapacity::from_str(indexing_capacity_str) {
+        indexing_capacity
+    } else {
+        error!(indexing_capacity=?indexing_capacity_str, "Received an unparseable indexing capacity from node.");
+        CpuCapacity::zero()
+    }
+}
+
 // Builds a cluster member from a [`NodeState`].
 pub(crate) fn build_cluster_member(
     chitchat_id: ChitchatId,
@@ -150,15 +147,17 @@ pub(crate) fn build_cluster_member(
         })?;
     let grpc_advertise_addr = node_state.grpc_advertise_addr()?;
     let indexing_tasks = parse_indexing_tasks(node_state, &chitchat_id.node_id);
-    let member = ClusterMember::new(
-        chitchat_id.node_id.into(),
-        chitchat_id.generation_id.into(),
+    let indexing_cpu_capacity = parse_indexing_cpu_capacity(node_state);
+    let member = ClusterMember {
+        node_id: chitchat_id.node_id.into(),
+        generation_id: chitchat_id.generation_id.into(),
         is_ready,
         enabled_services,
-        chitchat_id.gossip_advertise_addr,
+        gossip_advertise_addr: chitchat_id.gossip_advertise_addr,
         grpc_advertise_addr,
         indexing_tasks,
-    );
+        indexing_cpu_capacity,
+    };
     Ok(member)
 }
 
diff --git a/quickwit/quickwit-cluster/src/node.rs b/quickwit/quickwit-cluster/src/node.rs
index 1091ade7068..0b7d39dc7ce 100644
--- a/quickwit/quickwit-cluster/src/node.rs
+++ b/quickwit/quickwit-cluster/src/node.rs
@@ -24,7 +24,7 @@ use std::sync::Arc;
 
 use chitchat::{ChitchatId, NodeState};
 use quickwit_config::service::QuickwitService;
-use quickwit_proto::indexing::IndexingTask;
+use quickwit_proto::indexing::{CpuCapacity, IndexingTask};
 use tonic::transport::Channel;
 
 use crate::member::build_cluster_member;
@@ -49,6 +49,7 @@ impl ClusterNode {
             enabled_services: member.enabled_services,
             grpc_advertise_addr: member.grpc_advertise_addr,
             indexing_tasks: member.indexing_tasks,
+            indexing_capacity: member.indexing_cpu_capacity,
             is_ready: member.is_ready,
             is_self_node,
         };
@@ -112,6 +113,10 @@ impl ClusterNode {
         &self.inner.indexing_tasks
     }
 
+    pub fn indexing_capacity(&self) -> CpuCapacity {
+        self.inner.indexing_capacity
+    }
+
     pub fn is_ready(&self) -> bool {
         self.inner.is_ready
     }
@@ -149,6 +154,7 @@ struct InnerNode {
     enabled_services: HashSet<QuickwitService>,
     grpc_advertise_addr: SocketAddr,
     indexing_tasks: Vec<IndexingTask>,
+    indexing_capacity: CpuCapacity,
     is_ready: bool,
     is_self_node: bool,
 }
diff --git a/quickwit/quickwit-common/Cargo.toml b/quickwit/quickwit-common/Cargo.toml
index 4b0daef9a4f..bc86a752c64 100644
--- a/quickwit/quickwit-common/Cargo.toml
+++ b/quickwit/quickwit-common/Cargo.toml
@@ -13,7 +13,7 @@ documentation = "https://quickwit.io/docs/"
 anyhow = { workspace = true }
 async-speed-limit = { workspace = true }
 async-trait = { workspace = true }
-byte-unit = { workspace = true }
+bytesize = { workspace = true }
 dyn-clone = { workspace = true }
 env_logger = { workspace = true }
 fnv = { workspace = true }
diff --git a/quickwit/quickwit-common/src/tower/rate.rs b/quickwit/quickwit-common/src/tower/rate.rs
index d885deacc17..81e7f72fe11 100644
--- a/quickwit/quickwit-common/src/tower/rate.rs
+++ b/quickwit/quickwit-common/src/tower/rate.rs
@@ -19,7 +19,7 @@
 
 use std::time::Duration;
 
-use byte_unit::Byte;
+use bytesize::ByteSize;
 
 pub trait Rate: Clone {
     /// Returns the amount of work per time period.
@@ -41,18 +41,21 @@ impl ConstantRate {
     ///
     /// # Panics
     ///
-    /// This function panics if `work` is equal to zero or `period` is < 1ms.
+    /// This function panics if `period` is < 1ms.
     pub fn new(work: u64, period: Duration) -> Self {
-        assert!(work > 0);
         assert!(period.as_millis() > 0);
 
         Self { work, period }
     }
 
-    pub fn from_bytes(bytes: Byte, period: Duration) -> Self {
-        let work = bytes.get_bytes();
+    pub fn bytes_per_period(bytes: ByteSize, period: Duration) -> Self {
+        let work = bytes.as_u64();
         Self::new(work, period)
     }
+
+    pub fn bytes_per_sec(bytes: ByteSize) -> Self {
+        Self::bytes_per_period(bytes, Duration::from_secs(1))
+    }
 }
 
 impl Rate for ConstantRate {
diff --git a/quickwit/quickwit-config/Cargo.toml b/quickwit/quickwit-config/Cargo.toml
index 585c36d5ad2..8e6912d0ace 100644
--- a/quickwit/quickwit-config/Cargo.toml
+++ b/quickwit/quickwit-config/Cargo.toml
@@ -11,8 +11,8 @@ documentation = "https://quickwit.io/docs/"
 
 [dependencies]
 anyhow = { workspace = true }
-byte-unit = { workspace = true }
 bytes = { workspace = true }
+bytesize = { workspace = true }
 chrono = { workspace = true }
 cron = { workspace = true }
 enum-iterator = { workspace = true }
@@ -20,6 +20,7 @@ humantime = { workspace = true }
 itertools = { workspace = true }
 json_comments = { workspace = true }
 new_string_template = { workspace = true }
+num_cpus = { workspace = true }
 once_cell = { workspace = true }
 regex = { workspace = true }
 serde = { workspace = true }
@@ -29,8 +30,7 @@ serde_yaml = { workspace = true }
 toml = { workspace = true }
 tracing = { workspace = true }
 utoipa = { workspace = true }
-vrl = { workspace = true, optional=true }
-vrl-stdlib = { workspace = true, optional=true }
+vrl = { workspace = true, optional = true }
 
 quickwit-common = { workspace = true }
 quickwit-doc-mapper = { workspace = true }
@@ -42,4 +42,4 @@ tokio = { workspace = true }
 
 [features]
 testsuite = []
-vrl = ["dep:vrl", "vrl-stdlib"]
+vrl = ["dep:vrl"]
diff --git a/quickwit/quickwit-config/src/index_config/mod.rs b/quickwit/quickwit-config/src/index_config/mod.rs
index 68654049593..7f12b1b6a51 100644
--- a/quickwit/quickwit-config/src/index_config/mod.rs
+++ b/quickwit/quickwit-config/src/index_config/mod.rs
@@ -26,7 +26,7 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use anyhow::Context;
-use byte_unit::Byte;
+use bytesize::ByteSize;
 use chrono::Utc;
 use cron::Schedule;
 use humantime::parse_duration;
@@ -96,7 +96,7 @@ pub struct DocMapping {
 pub struct IndexingResources {
     #[schema(value_type = String, default = "2 GB")]
     #[serde(default = "IndexingResources::default_heap_size")]
-    pub heap_size: Byte,
+    pub heap_size: ByteSize,
     /// Sets the maximum write IO throughput in bytes/sec for the merge and delete pipelines.
     /// The IO limit is applied both to the downloader and to the merge executor.
     /// On hardware where IO is limited, this parameter can help limiting the impact of
@@ -104,7 +104,7 @@ pub struct IndexingResources {
     #[schema(value_type = String)]
     #[serde(default)]
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub max_merge_write_throughput: Option<Byte>,
+    pub max_merge_write_throughput: Option<ByteSize>,
 }
 
 impl PartialEq for IndexingResources {
@@ -114,14 +114,14 @@ impl PartialEq for IndexingResources {
 }
 
 impl IndexingResources {
-    fn default_heap_size() -> Byte {
-        Byte::from_bytes(2_000_000_000) // 2GB
+    fn default_heap_size() -> ByteSize {
+        ByteSize::gb(2)
     }
 
     #[cfg(any(test, feature = "testsuite"))]
     pub fn for_test() -> Self {
         Self {
-            heap_size: Byte::from_bytes(20_000_000), // 20MB
+            heap_size: ByteSize::mb(20),
             ..Default::default()
         }
     }
@@ -465,7 +465,7 @@ impl TestableForRegression for IndexConfig {
         };
         let merge_policy = MergePolicyConfig::StableLog(stable_log_config);
         let indexing_resources = IndexingResources {
-            heap_size: Byte::from_bytes(3),
+            heap_size: ByteSize::mb(50),
             ..Default::default()
         };
         let indexing_settings = IndexingSettings {
@@ -609,7 +609,7 @@ mod tests {
         assert_eq!(
             index_config.indexing_settings.resources,
             IndexingResources {
-                heap_size: Byte::from_bytes(3_000_000_000),
+                heap_size: ByteSize::gb(3),
                 ..Default::default()
             }
         );
@@ -637,7 +637,7 @@ mod tests {
     }
 
     #[test]
-    fn test_index_config_default_values() {
+    fn test_indexer_config_default_values() {
         let default_index_root_uri = Uri::for_test("s3://defaultbucket/");
         {
             let index_config_filepath = get_index_config_filepath("minimal-hdfs-logs.yaml");
diff --git a/quickwit/quickwit-config/src/node_config/mod.rs b/quickwit/quickwit-config/src/node_config/mod.rs
index 051208bef00..c381d396b14 100644
--- a/quickwit/quickwit-config/src/node_config/mod.rs
+++ b/quickwit/quickwit-config/src/node_config/mod.rs
@@ -27,9 +27,10 @@ use std::path::PathBuf;
 use std::time::Duration;
 
 use anyhow::{bail, ensure};
-use byte_unit::Byte;
+use bytesize::ByteSize;
 use quickwit_common::net::HostAddr;
 use quickwit_common::uri::Uri;
+use quickwit_proto::indexing::CpuCapacity;
 use serde::{Deserialize, Serialize};
 use tracing::warn;
 
@@ -44,7 +45,7 @@ pub const DEFAULT_QW_CONFIG_PATH: &str = "config/quickwit.yaml";
 #[serde(deny_unknown_fields)]
 pub struct IndexerConfig {
     #[serde(default = "IndexerConfig::default_split_store_max_num_bytes")]
-    pub split_store_max_num_bytes: Byte,
+    pub split_store_max_num_bytes: ByteSize,
     #[serde(default = "IndexerConfig::default_split_store_max_num_splits")]
     pub split_store_max_num_splits: usize,
     #[serde(default = "IndexerConfig::default_max_concurrent_split_uploads")]
@@ -55,6 +56,8 @@ pub struct IndexerConfig {
     pub enable_otlp_endpoint: bool,
     #[serde(default = "IndexerConfig::default_enable_cooperative_indexing")]
     pub enable_cooperative_indexing: bool,
+    #[serde(default = "IndexerConfig::default_cpu_capacity")]
+    pub cpu_capacity: CpuCapacity,
 }
 
 impl IndexerConfig {
@@ -77,22 +80,28 @@ impl IndexerConfig {
         12
     }
 
-    pub fn default_split_store_max_num_bytes() -> Byte {
-        Byte::from_bytes(100_000_000_000) // 100G
+    pub fn default_split_store_max_num_bytes() -> ByteSize {
+        ByteSize::gb(100)
     }
 
     pub fn default_split_store_max_num_splits() -> usize {
         1_000
     }
 
+    fn default_cpu_capacity() -> CpuCapacity {
+        CpuCapacity::one_cpu_thread() * (num_cpus::get() as u32)
+    }
+
     #[cfg(any(test, feature = "testsuite"))]
     pub fn for_test() -> anyhow::Result<Self> {
+        use quickwit_proto::indexing::PIPELINE_FULL_CAPACITY;
         let indexer_config = IndexerConfig {
             enable_cooperative_indexing: false,
             enable_otlp_endpoint: true,
-            split_store_max_num_bytes: Byte::from_bytes(1_000_000),
+            split_store_max_num_bytes: ByteSize::mb(1),
             split_store_max_num_splits: 3,
             max_concurrent_split_uploads: 4,
+            cpu_capacity: PIPELINE_FULL_CAPACITY * 4u32,
         };
         Ok(indexer_config)
     }
@@ -106,6 +115,7 @@ impl Default for IndexerConfig {
             split_store_max_num_bytes: Self::default_split_store_max_num_bytes(),
             split_store_max_num_splits: Self::default_split_store_max_num_splits(),
             max_concurrent_split_uploads: Self::default_max_concurrent_split_uploads(),
+            cpu_capacity: Self::default_cpu_capacity(),
         }
     }
 }
@@ -113,7 +123,7 @@ impl Default for IndexerConfig {
 #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct SplitCacheLimits {
-    pub max_num_bytes: Byte,
+    pub max_num_bytes: ByteSize,
     #[serde(default = "SplitCacheLimits::default_max_num_splits")]
     pub max_num_splits: NonZeroU32,
     #[serde(default = "SplitCacheLimits::default_num_concurrent_downloads")]
@@ -133,7 +143,7 @@ impl SplitCacheLimits {
 impl Default for SplitCacheLimits {
     fn default() -> SplitCacheLimits {
         SplitCacheLimits {
-            max_num_bytes: Byte::from_bytes(1_000_000_000), // 1 GB.
+            max_num_bytes: ByteSize::gb(1),
             max_num_splits: NonZeroU32::new(100).unwrap(),
             num_concurrent_downloads: NonZeroU32::new(1).unwrap(),
         }
@@ -143,11 +153,11 @@ impl Default for SplitCacheLimits {
 #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields, default)]
 pub struct SearcherConfig {
-    pub aggregation_memory_limit: Byte,
+    pub aggregation_memory_limit: ByteSize,
     pub aggregation_bucket_limit: u32,
-    pub fast_field_cache_capacity: Byte,
-    pub split_footer_cache_capacity: Byte,
-    pub partial_request_cache_capacity: Byte,
+    pub fast_field_cache_capacity: ByteSize,
+    pub split_footer_cache_capacity: ByteSize,
+    pub partial_request_cache_capacity: ByteSize,
     pub max_num_concurrent_split_searches: usize,
     pub max_num_concurrent_split_streams: usize,
     // Strangely, if None, this will also have the effect of not forwarding
@@ -160,12 +170,12 @@ pub struct SearcherConfig {
 impl Default for SearcherConfig {
     fn default() -> Self {
         Self {
-            fast_field_cache_capacity: Byte::from_bytes(1_000_000_000), // 1G
-            split_footer_cache_capacity: Byte::from_bytes(500_000_000), // 500M
-            partial_request_cache_capacity: Byte::from_bytes(64_000_000), // 64M
+            fast_field_cache_capacity: ByteSize::gb(1),
+            split_footer_cache_capacity: ByteSize::mb(500),
+            partial_request_cache_capacity: ByteSize::mb(64),
             max_num_concurrent_split_streams: 100,
             max_num_concurrent_split_searches: 100,
-            aggregation_memory_limit: Byte::from_bytes(500_000_000), // 500M
+            aggregation_memory_limit: ByteSize::mb(500),
             aggregation_bucket_limit: 65000,
             split_cache: None,
         }
@@ -175,19 +185,19 @@ impl Default for SearcherConfig {
 #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields, default)]
 pub struct IngestApiConfig {
-    pub max_queue_memory_usage: Byte,
-    pub max_queue_disk_usage: Byte,
+    pub max_queue_memory_usage: ByteSize,
+    pub max_queue_disk_usage: ByteSize,
     pub replication_factor: usize,
-    pub content_length_limit: u64,
+    pub content_length_limit: ByteSize,
 }
 
 impl Default for IngestApiConfig {
     fn default() -> Self {
         Self {
-            max_queue_memory_usage: Byte::from_bytes(2 * 1024 * 1024 * 1024), /* 2 GiB // TODO maybe we want more? */
-            max_queue_disk_usage: Byte::from_bytes(4 * 1024 * 1024 * 1024), /* 4 GiB // TODO maybe we want more? */
+            max_queue_memory_usage: ByteSize::gib(2), // TODO maybe we want more?
+            max_queue_disk_usage: ByteSize::gib(4),   // TODO maybe we want more?
             replication_factor: 1,
-            content_length_limit: 10 * 1024 * 1024, // 10 MiB
+            content_length_limit: ByteSize::mib(10),
         }
     }
 }
@@ -368,3 +378,54 @@ impl NodeConfig {
         serialize::node_config_for_test()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use quickwit_proto::indexing::CpuCapacity;
+
+    use crate::IndexerConfig;
+
+    #[test]
+    fn test_index_config_serialization() {
+        {
+            let indexer_config: IndexerConfig = serde_json::from_str(r#"{}"#).unwrap();
+            assert_eq!(&indexer_config, &IndexerConfig::default());
+            assert!(indexer_config.cpu_capacity.cpu_millis() > 0);
+            assert_eq!(indexer_config.cpu_capacity.cpu_millis() % 1_000, 0);
+        }
+        {
+            let indexer_config: IndexerConfig =
+                serde_yaml::from_str(r#"cpu_capacity: 1.5"#).unwrap();
+            assert_eq!(
+                indexer_config.cpu_capacity,
+                CpuCapacity::from_cpu_millis(1500)
+            );
+            let indexer_config_json = serde_json::to_value(&indexer_config).unwrap();
+            assert_eq!(
+                indexer_config_json
+                    .get("cpu_capacity")
+                    .unwrap()
+                    .as_str()
+                    .unwrap(),
+                "1500m"
+            );
+        }
+        {
+            let indexer_config: IndexerConfig =
+                serde_yaml::from_str(r#"cpu_capacity: 1500m"#).unwrap();
+            assert_eq!(
+                indexer_config.cpu_capacity,
+                CpuCapacity::from_cpu_millis(1500)
+            );
+            let indexer_config_json = serde_json::to_value(&indexer_config).unwrap();
+            assert_eq!(
+                indexer_config_json
+                    .get("cpu_capacity")
+                    .unwrap()
+                    .as_str()
+                    .unwrap(),
+                "1500m"
+            );
+        }
+    }
+}
diff --git a/quickwit/quickwit-config/src/node_config/serialize.rs b/quickwit/quickwit-config/src/node_config/serialize.rs
index 72ccfffdd06..9a99d0fb60e 100644
--- a/quickwit/quickwit-config/src/node_config/serialize.rs
+++ b/quickwit/quickwit-config/src/node_config/serialize.rs
@@ -398,7 +398,7 @@ mod tests {
     use std::num::NonZeroU64;
     use std::path::Path;
 
-    use byte_unit::Byte;
+    use bytesize::ByteSize;
     use itertools::Itertools;
 
     use super::*;
@@ -481,9 +481,10 @@ mod tests {
             config.indexer_config,
             IndexerConfig {
                 enable_otlp_endpoint: true,
-                split_store_max_num_bytes: Byte::from_str("1T").unwrap(),
+                split_store_max_num_bytes: ByteSize::tb(1),
                 split_store_max_num_splits: 10_000,
                 max_concurrent_split_uploads: 8,
+                cpu_capacity: IndexerConfig::default_cpu_capacity(),
                 enable_cooperative_indexing: false,
             }
         );
@@ -497,11 +498,11 @@ mod tests {
         assert_eq!(
             config.searcher_config,
             SearcherConfig {
-                aggregation_memory_limit: Byte::from_str("1G").unwrap(),
+                aggregation_memory_limit: ByteSize::gb(1),
                 aggregation_bucket_limit: 500_000,
-                fast_field_cache_capacity: Byte::from_str("10G").unwrap(),
-                split_footer_cache_capacity: Byte::from_str("1G").unwrap(),
-                partial_request_cache_capacity: Byte::from_str("64M").unwrap(),
+                fast_field_cache_capacity: ByteSize::gb(10),
+                split_footer_cache_capacity: ByteSize::gb(1),
+                partial_request_cache_capacity: ByteSize::mb(64),
                 max_num_concurrent_split_searches: 150,
                 max_num_concurrent_split_streams: 120,
                 split_cache: None,
diff --git a/quickwit/quickwit-config/src/source_config/mod.rs b/quickwit/quickwit-config/src/source_config/mod.rs
index 0fdcb7945b4..bda2e987f8c 100644
--- a/quickwit/quickwit-config/src/source_config/mod.rs
+++ b/quickwit/quickwit-config/src/source_config/mod.rs
@@ -510,7 +510,7 @@ impl TransformConfig {
         // Append "\n." to the script to return the entire document and not only the modified
         // fields.
         let vrl_script = self.vrl_script.clone() + "\n.";
-        let functions = vrl_stdlib::all();
+        let functions = vrl::stdlib::all();
 
         let compilation_res = match vrl::compiler::compile(&vrl_script, &functions) {
             Ok(compilation_res) => compilation_res,
@@ -1161,7 +1161,7 @@ mod tests {
             let transform_config = TransformConfig {
                 vrl_script: r#"
                 . = parse_json!(string!(.message))
-                .timestamp = to_unix_timestamp(to_timestamp!(.timestamp))
+                .timestamp = to_unix_timestamp(timestamp!(.timestamp))
                 del(.username)
                 .message = downcase(string!(.message))
                 "#
diff --git a/quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs b/quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs
index 5832c4a624c..4176535111e 100644
--- a/quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs
+++ b/quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs
@@ -26,7 +26,9 @@ use std::time::{Duration, Instant};
 
 use fnv::{FnvHashMap, FnvHashSet};
 use itertools::Itertools;
-use quickwit_proto::indexing::{ApplyIndexingPlanRequest, IndexingService, IndexingTask};
+use quickwit_proto::indexing::{
+    ApplyIndexingPlanRequest, CpuCapacity, IndexingService, IndexingTask, PIPELINE_FULL_CAPACITY,
+};
 use quickwit_proto::metastore::SourceType;
 use quickwit_proto::types::NodeId;
 use scheduling::{SourceToSchedule, SourceToScheduleType};
@@ -35,12 +37,9 @@ use tracing::{debug, error, info, warn};
 
 use crate::control_plane_model::ControlPlaneModel;
 use crate::indexing_plan::PhysicalIndexingPlan;
-use crate::indexing_scheduler::scheduling::{build_physical_indexing_plan, Load};
+use crate::indexing_scheduler::scheduling::build_physical_indexing_plan;
 use crate::{IndexerNodeInfo, IndexerPool};
 
-const PIPELINE_FULL_LOAD: Load = 1_000u32;
-const LOAD_PER_NODE: Load = 4_000u32;
-
 pub(crate) const MIN_DURATION_BETWEEN_SCHEDULING: Duration =
     if cfg!(any(test, feature = "testsuite")) {
         Duration::from_millis(50)
@@ -158,7 +157,8 @@ fn get_sources_to_schedule(model: &ControlPlaneModel) -> Vec<SourceToSchedule> {
                     source_type: SourceToScheduleType::NonSharded {
                         num_pipelines: source_config.desired_num_pipelines.get() as u32,
                         // FIXME
-                        load_per_pipeline: NonZeroU32::new(PIPELINE_FULL_LOAD).unwrap(),
+                        load_per_pipeline: NonZeroU32::new(PIPELINE_FULL_CAPACITY.cpu_millis())
+                            .unwrap(),
                     },
                 });
             }
@@ -185,7 +185,7 @@ impl IndexingScheduler {
     // has happened.
     pub(crate) fn schedule_indexing_plan_if_needed(&mut self, model: &ControlPlaneModel) {
         crate::metrics::CONTROL_PLANE_METRICS.schedule_total.inc();
-        let mut indexers = self.get_indexers_from_indexer_pool();
+        let mut indexers: Vec<(String, IndexerNodeInfo)> = self.get_indexers_from_indexer_pool();
         if indexers.is_empty() {
             warn!("No indexer available, cannot schedule an indexing plan.");
             return;
@@ -193,17 +193,16 @@ impl IndexingScheduler {
 
         let sources = get_sources_to_schedule(model);
 
-        let indexer_max_loads: FnvHashMap<String, Load> = indexers
+        let indexer_id_to_cpu_capacities: FnvHashMap<String, CpuCapacity> = indexers
             .iter()
-            .map(|(indexer_id, _)| {
-                // TODO Get info from chitchat.
-                (indexer_id.to_string(), LOAD_PER_NODE)
+            .map(|(indexer_id, indexer_node_info)| {
+                (indexer_id.to_string(), indexer_node_info.indexing_capacity)
             })
             .collect();
 
         let new_physical_plan = build_physical_indexing_plan(
             &sources,
-            &indexer_max_loads,
+            &indexer_id_to_cpu_capacities,
             self.state.last_applied_physical_plan.as_ref(),
         );
         if let Some(last_applied_plan) = &self.state.last_applied_physical_plan {
@@ -741,8 +740,8 @@ mod tests {
             },
         ];
         let mut indexer_max_loads = FnvHashMap::default();
-        indexer_max_loads.insert("indexer1".to_string(), 3_000);
-        indexer_max_loads.insert("indexer2".to_string(), 3_000);
+        indexer_max_loads.insert("indexer1".to_string(), mcpu(3_000));
+        indexer_max_loads.insert("indexer2".to_string(), mcpu(3_000));
         let physical_plan = build_physical_indexing_plan(&sources[..], &indexer_max_loads, None);
         assert_eq!(physical_plan.indexing_tasks_per_indexer().len(), 2);
         let indexing_tasks_1 = physical_plan.indexer("indexer1").unwrap();
@@ -771,7 +770,7 @@ mod tests {
             let mut indexer_max_loads = FnvHashMap::default();
             for i in 0..num_indexers {
                 let indexer_id = format!("indexer-{i}");
-                indexer_max_loads.insert(indexer_id, 4_000);
+                indexer_max_loads.insert(indexer_id, mcpu(4_000));
             }
             let physical_indexing_plan = build_physical_indexing_plan(&sources, &indexer_max_loads, None);
             let source_map: FnvHashMap<&SourceUid, &SourceToSchedule> = sources
@@ -802,12 +801,13 @@ mod tests {
                         }
                     }
                 }
-                assert!(load_in_node <= *indexer_max_loads.get(node_id).unwrap());
+                assert!(load_in_node <= indexer_max_loads.get(node_id).unwrap().cpu_millis());
             }
         }
     }
 
     use quickwit_config::SourceInputFormat;
+    use quickwit_proto::indexing::mcpu;
 
     fn kafka_source_params_for_test() -> SourceParams {
         SourceParams::Kafka(KafkaSourceParams {
diff --git a/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/mod.rs b/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/mod.rs
index 7dce1cb841f..22cd4a6ebd6 100644
--- a/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/mod.rs
+++ b/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/mod.rs
@@ -24,9 +24,8 @@ use std::collections::hash_map::Entry;
 use std::num::NonZeroU32;
 
 use fnv::FnvHashMap;
-use quickwit_proto::indexing::IndexingTask;
+use quickwit_proto::indexing::{CpuCapacity, IndexingTask};
 use quickwit_proto::types::{IndexUid, ShardId};
-pub use scheduling_logic_model::Load;
 use scheduling_logic_model::{IndexerOrd, SourceOrd};
 use tracing::error;
 use tracing::log::warn;
@@ -35,7 +34,6 @@ use crate::indexing_plan::PhysicalIndexingPlan;
 use crate::indexing_scheduler::scheduling::scheduling_logic_model::{
     SchedulingProblem, SchedulingSolution,
 };
-use crate::indexing_scheduler::PIPELINE_FULL_LOAD;
 use crate::SourceUid;
 
 /// If we have several pipelines below this threshold we
@@ -49,10 +47,10 @@ use crate::SourceUid;
 ///
 /// Coming back to a single pipeline requires having a load per pipeline
 /// of 30%. Which translates into an overall load of 60%.
-const LOAD_PER_PIPELINE_LOW_THRESHOLD: u32 = PIPELINE_FULL_LOAD * 3 / 10;
+const CPU_PER_PIPELINE_LOAD_THRESHOLD: CpuCapacity = CpuCapacity::from_cpu_millis(1_200);
 
 /// That's 80% of a period
-const MAX_LOAD_PER_PIPELINE: u32 = PIPELINE_FULL_LOAD * 8 / 10;
+const MAX_LOAD_PER_PIPELINE: CpuCapacity = CpuCapacity::from_cpu_millis(3_200);
 
 fn indexing_task(source_uid: SourceUid, shard_ids: Vec<ShardId>) -> IndexingTask {
     IndexingTask {
@@ -260,35 +258,37 @@ fn group_shards_into_pipelines(
     source_uid: &SourceUid,
     shard_ids: &[ShardId],
     previous_indexing_tasks: &[IndexingTask],
-    load_per_shard: Load,
+    cpu_load_per_shard: CpuCapacity,
 ) -> Vec<IndexingTask> {
     let num_shards = shard_ids.len() as u32;
     if num_shards == 0 {
         return Vec::new();
     }
     let max_num_shards_per_pipeline: NonZeroU32 =
-        NonZeroU32::new(MAX_LOAD_PER_PIPELINE / load_per_shard).unwrap_or_else(|| {
-            // We throttle shard at ingestion to ensure that a shard does not
-            // exceed 5MB/s.
-            //
-            // This value has been chosen to make sure that one full pipeline
-            // should always be able to handle the load of one shard.
-            //
-            // However it is possible for the system to take more than this
-            // when it is playing catch up.
-            //
-            // This is a transitory state, and not a problem per se.
-            warn!("load per shard is higher than `MAX_LOAD_PER_PIPELINE`");
-            NonZeroU32::new(1).unwrap()
-        });
+        NonZeroU32::new(MAX_LOAD_PER_PIPELINE.cpu_millis() / cpu_load_per_shard.cpu_millis())
+            .unwrap_or_else(|| {
+                // We throttle shard at ingestion to ensure that a shard does not
+                // exceed 5MB/s.
+                //
+                // This value has been chosen to make sure that one full pipeline
+                // should always be able to handle the load of one shard.
+                //
+                // However it is possible for the system to take more than this
+                // when it is playing catch up.
+                //
+                // This is a transitory state, and not a problem per se.
+                warn!("load per shard is higher than `MAX_LOAD_PER_PIPELINE`");
+                NonZeroU32::MIN // also colloquially known as `1`
+            });
 
     // We compute the number of pipelines we will create, cooking in some hysteresis effect here.
     // We have two different threshold to increase and to decrease the number of pipelines.
     let min_num_pipelines: u32 =
         (num_shards + max_num_shards_per_pipeline.get() - 1) / max_num_shards_per_pipeline;
     assert!(min_num_pipelines > 0);
-    let max_num_pipelines: u32 =
-        min_num_pipelines.max(num_shards * load_per_shard / LOAD_PER_PIPELINE_LOW_THRESHOLD);
+    let max_num_pipelines: u32 = min_num_pipelines.max(
+        num_shards * cpu_load_per_shard.cpu_millis() / CPU_PER_PIPELINE_LOAD_THRESHOLD.cpu_millis(),
+    );
     let previous_num_pipelines = previous_indexing_tasks.len() as u32;
     let num_pipelines: u32 = if previous_num_pipelines > min_num_pipelines {
         previous_num_pipelines.min(max_num_pipelines)
@@ -409,7 +409,7 @@ fn convert_scheduling_solution_to_physical_plan(
                         &source.source_uid,
                         &shard_ids_for_node,
                         indexing_tasks,
-                        load_per_shard.get(),
+                        CpuCapacity::from_cpu_millis(load_per_shard.get()),
                     );
                     for indexing_task in indexing_tasks {
                         physical_indexing_plan.add_indexing_task(node_id, indexing_task);
@@ -471,7 +471,7 @@ fn convert_scheduling_solution_to_physical_plan(
 /// TODO cut into pipelines.
 pub fn build_physical_indexing_plan(
     sources: &[SourceToSchedule],
-    indexer_id_to_max_load: &FnvHashMap<String, Load>,
+    indexer_id_to_cpu_capacities: &FnvHashMap<String, CpuCapacity>,
     previous_plan_opt: Option<&PhysicalIndexingPlan>,
 ) -> PhysicalIndexingPlan {
     // TODO make the load per node something that can be configured on each node.
@@ -480,14 +480,15 @@ pub fn build_physical_indexing_plan(
     let mut id_to_ord_map = IdToOrdMap::default();
 
     // We use a Vec as a `IndexOrd` -> Max load map.
-    let mut indexer_max_loads: Vec<Load> = Vec::with_capacity(indexer_id_to_max_load.len());
-    for (indexer_id, &max_load) in indexer_id_to_max_load {
+    let mut indexer_cpu_capacities: Vec<CpuCapacity> =
+        Vec::with_capacity(indexer_id_to_cpu_capacities.len());
+    for (indexer_id, &cpu_capacity) in indexer_id_to_cpu_capacities {
         let indexer_ord = id_to_ord_map.add_indexer_id(indexer_id.clone());
-        assert_eq!(indexer_ord, indexer_max_loads.len() as IndexerOrd);
-        indexer_max_loads.push(max_load);
+        assert_eq!(indexer_ord, indexer_cpu_capacities.len() as IndexerOrd);
+        indexer_cpu_capacities.push(cpu_capacity);
     }
 
-    let mut problem = SchedulingProblem::with_indexer_maximum_load(indexer_max_loads);
+    let mut problem = SchedulingProblem::with_indexer_cpu_capacities(indexer_cpu_capacities);
     for source in sources {
         if let Some(source_ord) = populate_problem(source, &mut problem) {
             let registered_source_ord = id_to_ord_map.add_source_uid(source.source_uid.clone());
@@ -528,7 +529,7 @@ mod tests {
     use std::sync::atomic::{AtomicUsize, Ordering};
 
     use fnv::FnvHashMap;
-    use quickwit_proto::indexing::IndexingTask;
+    use quickwit_proto::indexing::{mcpu, IndexingTask};
     use quickwit_proto::types::{IndexUid, ShardId};
 
     use super::{
@@ -573,25 +574,28 @@ mod tests {
             source_uid: source_uid0.clone(),
             source_type: SourceToScheduleType::Sharded {
                 shards: vec![0, 1, 2, 3, 4, 5, 6, 7],
-                load_per_shard: NonZeroU32::new(250).unwrap(),
+                load_per_shard: NonZeroU32::new(1_000).unwrap(),
             },
         };
         let source_1 = SourceToSchedule {
             source_uid: source_uid1.clone(),
             source_type: SourceToScheduleType::NonSharded {
                 num_pipelines: 2,
-                load_per_pipeline: NonZeroU32::new(800).unwrap(),
+                load_per_pipeline: NonZeroU32::new(3_200).unwrap(),
             },
         };
         let source_2 = SourceToSchedule {
             source_uid: source_uid2.clone(),
             source_type: SourceToScheduleType::IngestV1,
         };
-        let mut indexer_max_load = FnvHashMap::default();
-        indexer_max_load.insert(indexer1.clone(), 4_000);
-        indexer_max_load.insert(indexer2.clone(), 4_000);
-        let indexing_plan =
-            build_physical_indexing_plan(&[source_0, source_1, source_2], &indexer_max_load, None);
+        let mut indexer_id_to_cpu_capacities = FnvHashMap::default();
+        indexer_id_to_cpu_capacities.insert(indexer1.clone(), mcpu(16_000));
+        indexer_id_to_cpu_capacities.insert(indexer2.clone(), mcpu(16_000));
+        let indexing_plan = build_physical_indexing_plan(
+            &[source_0, source_1, source_2],
+            &indexer_id_to_cpu_capacities,
+            None,
+        );
         assert_eq!(indexing_plan.indexing_tasks_per_indexer().len(), 2);
 
         let node1_plan = indexing_plan.indexer(&indexer1).unwrap();
@@ -633,7 +637,7 @@ mod tests {
         let indexer1 = "indexer1".to_string();
         let mut indexer_max_loads = FnvHashMap::default();
         {
-            indexer_max_loads.insert(indexer1.clone(), 1_999);
+            indexer_max_loads.insert(indexer1.clone(), mcpu(1_999));
             // This test what happens when there isn't enough capacity on the cluster.
             let physical_plan = build_physical_indexing_plan(&sources, &indexer_max_loads, None);
             assert_eq!(physical_plan.indexing_tasks_per_indexer().len(), 1);
@@ -644,7 +648,7 @@ mod tests {
             );
         }
         {
-            indexer_max_loads.insert(indexer1.clone(), 2_000);
+            indexer_max_loads.insert(indexer1.clone(), mcpu(2_000));
             // This test what happens when there isn't enough capacity on the cluster.
             let physical_plan = build_physical_indexing_plan(&sources, &indexer_max_loads, None);
             assert_eq!(physical_plan.indexing_tasks_per_indexer().len(), 1);
@@ -662,7 +666,7 @@ mod tests {
     #[test]
     fn test_group_shards_empty() {
         let source_uid = source_id();
-        let indexing_tasks = group_shards_into_pipelines(&source_uid, &[], &[], 250);
+        let indexing_tasks = group_shards_into_pipelines(&source_uid, &[], &[], mcpu(250));
         assert!(indexing_tasks.is_empty());
     }
 
@@ -690,7 +694,7 @@ mod tests {
             &source_uid,
             &[0, 1, 3, 4, 5],
             &previous_indexing_tasks,
-            250,
+            mcpu(1_000),
         );
         assert_eq!(indexing_tasks.len(), 2);
         assert_eq!(&indexing_tasks[0].shard_ids, &[0, 1]);
@@ -700,7 +704,7 @@ mod tests {
     #[test]
     fn test_group_shards_load_per_shard_too_high() {
         let source_uid = source_id();
-        let indexing_tasks = group_shards_into_pipelines(&source_uid, &[1, 2], &[], 1_000);
+        let indexing_tasks = group_shards_into_pipelines(&source_uid, &[1, 2], &[], mcpu(4_000));
         assert_eq!(indexing_tasks.len(), 2);
     }
 
@@ -712,7 +716,7 @@ mod tests {
             &source_uid,
             &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
             &previous_indexing_tasks,
-            100,
+            mcpu(400),
         );
         assert_eq!(indexing_tasks_1.len(), 2);
         assert_eq!(&indexing_tasks_1[0].shard_ids, &[0, 2, 4, 6, 8, 10]);
@@ -722,7 +726,7 @@ mod tests {
             &source_uid,
             &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
             &indexing_tasks_1,
-            150,
+            mcpu(600),
         );
         assert_eq!(indexing_tasks_2.len(), 3);
         assert_eq!(&indexing_tasks_2[0].shard_ids, &[0, 2, 4, 6, 8]);
@@ -734,7 +738,7 @@ mod tests {
             &source_uid,
             &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
             &indexing_tasks_2,
-            100,
+            mcpu(400),
         );
         assert_eq!(indexing_tasks_3.len(), 3);
         assert_eq!(&indexing_tasks_3[0].shard_ids, &[0, 2, 4, 6, 8]);
@@ -745,7 +749,7 @@ mod tests {
             &source_uid,
             &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
             &indexing_tasks_3,
-            80,
+            mcpu(320),
         );
         assert_eq!(indexing_tasks_4.len(), 2);
         assert_eq!(&indexing_tasks_4[0].shard_ids, &[0, 2, 4, 6, 8, 10]);
diff --git a/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/scheduling_logic.rs b/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/scheduling_logic.rs
index b2f327fd336..82af447012a 100644
--- a/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/scheduling_logic.rs
+++ b/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/scheduling_logic.rs
@@ -21,6 +21,8 @@ use std::cmp::Reverse;
 use std::collections::btree_map::Entry;
 use std::collections::{BTreeMap, BinaryHeap, HashMap};
 
+use quickwit_proto::indexing::CpuCapacity;
+
 use super::scheduling_logic_model::*;
 
 // ------------------------------------------------------------------------------------
@@ -43,7 +45,7 @@ pub fn solve(
     let mut solution = previous_solution;
     check_contract_conditions(problem, &solution);
     remove_extraneous_shards(problem, &mut solution);
-    enforce_nodes_max_load(problem, &mut solution);
+    enforce_indexers_cpu_capacity(problem, &mut solution);
     let still_unassigned = place_unassigned_shards(problem, &mut solution);
     // TODO ideally we should have some smarter logic here to bread first search for a better
     // solution.
@@ -84,7 +86,7 @@ fn remove_extraneous_shards(problem: &SchedulingProblem, solution: &mut Scheduli
         }
     }
 
-    let mut indexer_available_capacity: Vec<Load> = solution
+    let mut indexer_available_capacity: Vec<CpuCapacity> = solution
         .indexer_assignments
         .iter()
         .map(|indexer_assignment| indexer_assignment.indexer_available_capacity(problem))
@@ -136,48 +138,50 @@ fn assert_remove_extraneous_shards_post_condition(
 // Phase 2
 // Releave sources from the node that are exceeding their maximum load.
 
-fn enforce_nodes_max_load(problem: &SchedulingProblem, solution: &mut SchedulingSolution) {
+fn enforce_indexers_cpu_capacity(problem: &SchedulingProblem, solution: &mut SchedulingSolution) {
     for indexer_assignment in solution.indexer_assignments.iter_mut() {
-        let node_max_load: Load = problem.indexer_max_load(indexer_assignment.indexer_ord);
-        enforce_node_max_load(problem, node_max_load, indexer_assignment);
+        let indexer_cpu_capacity: CpuCapacity =
+            problem.indexer_cpu_capacity(indexer_assignment.indexer_ord);
+        enforce_indexer_cpu_capacity(problem, indexer_cpu_capacity, indexer_assignment);
     }
 }
 
-fn enforce_node_max_load(
+fn enforce_indexer_cpu_capacity(
     problem: &SchedulingProblem,
-    node_max_load: Load,
+    indexer_cpu_capacity: CpuCapacity,
     indexer_assignment: &mut IndexerAssignment,
 ) {
-    let total_load = indexer_assignment.total_load(problem);
-    if total_load <= node_max_load {
+    let total_load = indexer_assignment.total_cpu_load(problem);
+    if total_load <= indexer_cpu_capacity {
         return;
     }
-    let mut load_to_remove = total_load - node_max_load;
-    let mut load_sources: Vec<(Load, SourceOrd)> = indexer_assignment
+    let mut load_to_remove: CpuCapacity = total_load - indexer_cpu_capacity;
+    let mut source_cpu_capacities: Vec<(CpuCapacity, SourceOrd)> = indexer_assignment
         .num_shards_per_source
         .iter()
-        .map(|(source_ord, num_shards)| {
-            let load_for_source = problem.source_load_per_shard(*source_ord).get() * num_shards;
-            (load_for_source, *source_ord)
+        .map(|(&source_ord, num_shards)| {
+            let load_for_source = problem.source_load_per_shard(source_ord).get() * num_shards;
+            (CpuCapacity::from_cpu_millis(load_for_source), source_ord)
         })
         .collect();
-    load_sources.sort();
-    for (load, source_ord) in load_sources {
+    source_cpu_capacities.sort();
+    for (source_cpu_capacity, source_ord) in source_cpu_capacities {
         indexer_assignment.num_shards_per_source.remove(&source_ord);
-        load_to_remove = load_to_remove.saturating_sub(load);
-        if load_to_remove == 0 {
+        load_to_remove = if load_to_remove <= source_cpu_capacity {
             break;
-        }
+        } else {
+            load_to_remove - source_cpu_capacity
+        };
     }
-    assert_enforce_nodes_max_load_post_condition(problem, indexer_assignment);
+    assert_enforce_nodes_cpu_capacity_post_condition(problem, indexer_assignment);
 }
 
-fn assert_enforce_nodes_max_load_post_condition(
+fn assert_enforce_nodes_cpu_capacity_post_condition(
     problem: &SchedulingProblem,
     indexer_assignment: &IndexerAssignment,
 ) {
-    let total_load = indexer_assignment.total_load(problem);
-    assert!(total_load <= problem.indexer_max_load(indexer_assignment.indexer_ord));
+    let total_load = indexer_assignment.total_cpu_load(problem);
+    assert!(total_load <= problem.indexer_cpu_capacity(indexer_assignment.indexer_ord));
 }
 
 // ----------------------------------------------------
@@ -199,12 +203,15 @@ fn place_unassigned_shards(
         let load = source.num_shards * source.load_per_shard.get();
         Reverse(load)
     });
-    let mut node_with_least_loads: BinaryHeap<(Load, IndexerOrd)> =
+    let mut indexers_with_most_available_capacity: BinaryHeap<(CpuCapacity, IndexerOrd)> =
         compute_indexer_available_capacity(problem, solution);
     let mut unassignable_shards = BTreeMap::new();
     for source in unassigned_shards {
-        let num_shards_unassigned =
-            place_unassigned_shards_single_source(&source, &mut node_with_least_loads, solution);
+        let num_shards_unassigned = place_unassigned_shards_single_source(
+            &source,
+            &mut indexers_with_most_available_capacity,
+            solution,
+        );
         // We haven't been able to place this source entirely.
         if num_shards_unassigned != 0 {
             unassignable_shards.insert(source.source_ord, num_shards_unassigned);
@@ -237,28 +244,29 @@ fn assert_place_unassigned_shards_post_condition(
     }
     // We make sure that all unassigned shard cannot be placed.
     for indexer_assignment in &solution.indexer_assignments {
-        let available_capacity: Load = indexer_assignment.indexer_available_capacity(problem);
+        let available_capacity: CpuCapacity =
+            indexer_assignment.indexer_available_capacity(problem);
         for (&source_ord, &num_shards) in unassigned_shards {
             assert!(num_shards > 0);
             let source = problem.source(source_ord);
-            assert!(source.load_per_shard.get() > available_capacity);
+            assert!(source.load_per_shard.get() > available_capacity.cpu_millis());
         }
     }
 }
 
 fn place_unassigned_shards_single_source(
     source: &Source,
-    indexer_available_capacities: &mut BinaryHeap<(Load, IndexerOrd)>,
+    indexer_available_capacities: &mut BinaryHeap<(CpuCapacity, IndexerOrd)>,
     solution: &mut SchedulingSolution,
 ) -> u32 {
     let mut num_shards = source.num_shards;
     while num_shards > 0 {
-        let Some(mut node_with_least_load) = indexer_available_capacities.peek_mut() else {
+        let Some(mut node_with_most_capacity) = indexer_available_capacities.peek_mut() else {
             break;
         };
-        let node_id = node_with_least_load.1;
-        let available_capacity = &mut node_with_least_load.0;
-        let num_placable_shards = *available_capacity / source.load_per_shard;
+        let node_id = node_with_most_capacity.1;
+        let available_capacity = &mut node_with_most_capacity.0;
+        let num_placable_shards = available_capacity.cpu_millis() / source.load_per_shard;
         let num_shards_to_place = num_placable_shards.min(num_shards);
         // We cannot place more shards with this load.
         if num_shards_to_place == 0 {
@@ -267,7 +275,8 @@ fn place_unassigned_shards_single_source(
         // TODO take in account colocation.
         // Update the solution, the shard load, and the number of shards to place.
         solution.indexer_assignments[node_id].add_shards(source.source_ord, num_shards_to_place);
-        *available_capacity -= num_shards_to_place * source.load_per_shard.get();
+        *available_capacity = *available_capacity
+            - CpuCapacity::from_cpu_millis(num_shards_to_place * source.load_per_shard.get());
         num_shards -= num_shards_to_place;
     }
     num_shards
@@ -298,8 +307,8 @@ fn compute_unassigned_sources(
 fn compute_indexer_available_capacity(
     problem: &SchedulingProblem,
     solution: &SchedulingSolution,
-) -> BinaryHeap<(Load, IndexerOrd)> {
-    let mut indexer_available_capacity: BinaryHeap<(Load, IndexerOrd)> =
+) -> BinaryHeap<(CpuCapacity, IndexerOrd)> {
+    let mut indexer_available_capacity: BinaryHeap<(CpuCapacity, IndexerOrd)> =
         BinaryHeap::with_capacity(problem.num_indexers());
     for indexer_assignment in &solution.indexer_assignments {
         let available_capacity = indexer_assignment.indexer_available_capacity(problem);
@@ -312,12 +321,14 @@ mod tests {
     use std::num::NonZeroU32;
 
     use proptest::prelude::*;
+    use quickwit_proto::indexing::mcpu;
 
     use super::*;
 
     #[test]
     fn test_remove_extranous_shards() {
-        let mut problem = SchedulingProblem::with_indexer_maximum_load(vec![4_000, 5_000]);
+        let mut problem =
+            SchedulingProblem::with_indexer_cpu_capacities(vec![mcpu(4_000), mcpu(5_000)]);
         problem.add_source(1, NonZeroU32::new(1_000u32).unwrap());
         let mut solution = problem.new_solution();
         solution.indexer_assignments[0].add_shards(0, 3);
@@ -329,7 +340,8 @@ mod tests {
 
     #[test]
     fn test_remove_extranous_shards_2() {
-        let mut problem = SchedulingProblem::with_indexer_maximum_load(vec![5_000, 4_000]);
+        let mut problem =
+            SchedulingProblem::with_indexer_cpu_capacities(vec![mcpu(5_000), mcpu(4_000)]);
         problem.add_source(2, NonZeroU32::new(1_000).unwrap());
         let mut solution = problem.new_solution();
         solution.indexer_assignments[0].add_shards(0, 3);
@@ -341,7 +353,8 @@ mod tests {
 
     #[test]
     fn test_remove_missing_sources() {
-        let mut problem = SchedulingProblem::with_indexer_maximum_load(vec![5_000, 4_000]);
+        let mut problem =
+            SchedulingProblem::with_indexer_cpu_capacities(vec![mcpu(5_000), mcpu(4_000)]);
         // Source 0
         problem.add_source(0, NonZeroU32::new(1_000).unwrap());
         // Source 1
@@ -358,9 +371,14 @@ mod tests {
     }
 
     #[test]
-    fn test_enforce_nodes_max_load() {
-        let mut problem =
-            SchedulingProblem::with_indexer_maximum_load(vec![5_000, 5_000, 5_000, 5_000, 7_000]);
+    fn test_enforce_nodes_cpu_capacity() {
+        let mut problem = SchedulingProblem::with_indexer_cpu_capacities(vec![
+            mcpu(5_000),
+            mcpu(5_000),
+            mcpu(5_000),
+            mcpu(5_000),
+            mcpu(7_000),
+        ]);
         // Source 0
         problem.add_source(10, NonZeroU32::new(3_000).unwrap());
         problem.add_source(10, NonZeroU32::new(2_000).unwrap());
@@ -388,7 +406,7 @@ mod tests {
         solution.indexer_assignments[4].add_shards(1, 1);
         solution.indexer_assignments[4].add_shards(2, 2);
 
-        enforce_nodes_max_load(&problem, &mut solution);
+        enforce_indexers_cpu_capacity(&problem, &mut solution);
 
         assert_eq!(solution.indexer_assignments[0].num_shards(0), 1);
         assert_eq!(solution.indexer_assignments[0].num_shards(1), 0);
@@ -414,7 +432,8 @@ mod tests {
 
     #[test]
     fn test_compute_unassigned_shards_simple() {
-        let mut problem = SchedulingProblem::with_indexer_maximum_load(vec![0, 4_000]);
+        let mut problem =
+            SchedulingProblem::with_indexer_cpu_capacities(vec![mcpu(0), mcpu(4_000)]);
         problem.add_source(4, NonZeroU32::new(1000).unwrap());
         problem.add_source(4, NonZeroU32::new(1_000).unwrap());
         let solution = problem.new_solution();
@@ -431,7 +450,8 @@ mod tests {
 
     #[test]
     fn test_compute_unassigned_shards_with_non_trivial_solution() {
-        let mut problem = SchedulingProblem::with_indexer_maximum_load(vec![50_000, 40_000]);
+        let mut problem =
+            SchedulingProblem::with_indexer_cpu_capacities(vec![mcpu(50_000), mcpu(40_000)]);
         problem.add_source(5, NonZeroU32::new(1_000).unwrap());
         problem.add_source(15, NonZeroU32::new(2_000).unwrap());
         let mut solution = problem.new_solution();
@@ -461,7 +481,7 @@ mod tests {
 
     #[test]
     fn test_place_unassigned_shards_simple() {
-        let mut problem = SchedulingProblem::with_indexer_maximum_load(vec![4_000]);
+        let mut problem = SchedulingProblem::with_indexer_cpu_capacities(vec![mcpu(4_000)]);
         problem.add_source(4, NonZeroU32::new(1_000).unwrap());
         let mut solution = problem.new_solution();
         let unassigned = place_unassigned_shards(&problem, &mut solution);
@@ -471,7 +491,8 @@ mod tests {
 
     #[test]
     fn test_place_unassigned_shards_reach_capacity() {
-        let mut problem = SchedulingProblem::with_indexer_maximum_load(vec![50_000, 40_000]);
+        let mut problem =
+            SchedulingProblem::with_indexer_cpu_capacities(vec![mcpu(50_000), mcpu(40_000)]);
         problem.add_source(5, NonZeroU32::new(1_000).unwrap());
         problem.add_source(15, NonZeroU32::new(2_000).unwrap());
         let mut solution = problem.new_solution();
@@ -504,20 +525,21 @@ mod tests {
 
     #[test]
     fn test_solve() {
-        let mut problem = SchedulingProblem::with_indexer_maximum_load(vec![800]);
+        let mut problem = SchedulingProblem::with_indexer_cpu_capacities(vec![mcpu(800)]);
         problem.add_source(43, NonZeroU32::new(1).unwrap());
         problem.add_source(379, NonZeroU32::new(1).unwrap());
         let previous_solution = problem.new_solution();
         solve(&problem, previous_solution);
     }
 
-    fn node_max_load_strat() -> impl Strategy<Value = Load> {
+    fn indexer_cpu_capacity_strat() -> impl Strategy<Value = CpuCapacity> {
         prop_oneof![
             0u32..10_000u32,
             Just(0u32),
             800u32..1200u32,
             1900u32..2100u32,
         ]
+        .prop_map(CpuCapacity::from_cpu_millis)
     }
 
     fn num_shards() -> impl Strategy<Value = u32> {
@@ -543,10 +565,11 @@ mod tests {
         num_nodes: usize,
         num_sources: usize,
     ) -> impl Strategy<Value = SchedulingProblem> {
-        let node_max_loads_strat = proptest::collection::vec(node_max_load_strat(), num_nodes);
+        let indexer_cpu_capacity_strat =
+            proptest::collection::vec(indexer_cpu_capacity_strat(), num_nodes);
         let sources_strat = proptest::collection::vec(source_strat(), num_sources);
-        (node_max_loads_strat, sources_strat).prop_map(|(node_max_loads, sources)| {
-            let mut problem = SchedulingProblem::with_indexer_maximum_load(node_max_loads);
+        (indexer_cpu_capacity_strat, sources_strat).prop_map(|(node_cpu_capacities, sources)| {
+            let mut problem = SchedulingProblem::with_indexer_cpu_capacities(node_cpu_capacities);
             for (num_shards, load_per_shard) in sources {
                 problem.add_source(num_shards, load_per_shard);
             }
diff --git a/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/scheduling_logic_model.rs b/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/scheduling_logic_model.rs
index 91255aa7f10..7e61a5cb3ca 100644
--- a/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/scheduling_logic_model.rs
+++ b/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/scheduling_logic_model.rs
@@ -21,10 +21,10 @@ use std::collections::btree_map::Entry;
 use std::collections::BTreeMap;
 use std::num::NonZeroU32;
 
+use quickwit_proto::indexing::CpuCapacity;
+
 pub type SourceOrd = u32;
 pub type IndexerOrd = usize;
-pub type Load = u32;
-
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub struct Source {
     pub source_ord: SourceOrd,
@@ -35,22 +35,24 @@ pub struct Source {
 #[derive(Debug)]
 pub struct SchedulingProblem {
     sources: Vec<Source>,
-    indexer_max_loads: Vec<Load>,
+    indexer_cpu_capacities: Vec<CpuCapacity>,
 }
 
 impl SchedulingProblem {
     pub fn new_solution(&self) -> SchedulingSolution {
-        SchedulingSolution::with_num_indexers(self.indexer_max_loads.len())
+        SchedulingSolution::with_num_indexers(self.indexer_cpu_capacities.len())
     }
 
-    pub fn indexer_max_load(&self, indexer_ord: IndexerOrd) -> Load {
-        self.indexer_max_loads[indexer_ord]
+    pub fn indexer_cpu_capacity(&self, indexer_ord: IndexerOrd) -> CpuCapacity {
+        self.indexer_cpu_capacities[indexer_ord]
     }
 
-    pub fn with_indexer_maximum_load(node_max_loads: Vec<Load>) -> SchedulingProblem {
+    pub fn with_indexer_cpu_capacities(
+        indexer_cpu_capacities: Vec<CpuCapacity>,
+    ) -> SchedulingProblem {
         SchedulingProblem {
             sources: Vec::new(),
-            indexer_max_loads: node_max_loads,
+            indexer_cpu_capacities,
         }
     }
 
@@ -81,7 +83,7 @@ impl SchedulingProblem {
     }
 
     pub fn num_indexers(&self) -> usize {
-        self.indexer_max_loads.len()
+        self.indexer_cpu_capacities.len()
     }
 }
 
@@ -99,17 +101,25 @@ impl IndexerAssignment {
         }
     }
 
-    pub fn indexer_available_capacity(&self, problem: &SchedulingProblem) -> Load {
-        problem.indexer_max_loads[self.indexer_ord].saturating_sub(self.total_load(problem))
+    pub fn indexer_available_capacity(&self, problem: &SchedulingProblem) -> CpuCapacity {
+        let total_cpu_capacity = self.total_cpu_load(problem);
+        let indexer_cpu_capacity = problem.indexer_cpu_capacities[self.indexer_ord];
+        if indexer_cpu_capacity <= total_cpu_capacity {
+            CpuCapacity::zero()
+        } else {
+            indexer_cpu_capacity - total_cpu_capacity
+        }
     }
 
-    pub fn total_load(&self, problem: &SchedulingProblem) -> Load {
-        self.num_shards_per_source
-            .iter()
-            .map(|(source_ord, num_shards)| {
-                problem.source_load_per_shard(*source_ord).get() * num_shards
-            })
-            .sum()
+    pub fn total_cpu_load(&self, problem: &SchedulingProblem) -> CpuCapacity {
+        CpuCapacity::from_cpu_millis(
+            self.num_shards_per_source
+                .iter()
+                .map(|(source_ord, num_shards)| {
+                    problem.source_load_per_shard(*source_ord).get() * num_shards
+                })
+                .sum(),
+        )
     }
 
     pub fn num_shards(&self, source_ord: SourceOrd) -> u32 {
diff --git a/quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs b/quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs
index b2292ef6a74..2d086d6d54a 100644
--- a/quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs
+++ b/quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs
@@ -25,12 +25,12 @@ use itertools::Itertools;
 use quickwit_common::{PrettySample, Progress};
 use quickwit_ingest::IngesterPool;
 use quickwit_proto::control_plane::{
-    ClosedShards, ControlPlaneError, ControlPlaneResult, GetOrCreateOpenShardsFailure,
+    ControlPlaneError, ControlPlaneResult, GetOrCreateOpenShardsFailure,
     GetOrCreateOpenShardsFailureReason, GetOrCreateOpenShardsRequest,
     GetOrCreateOpenShardsResponse, GetOrCreateOpenShardsSuccess,
 };
 use quickwit_proto::ingest::ingester::{IngesterService, PingRequest};
-use quickwit_proto::ingest::{IngestV2Error, ShardState};
+use quickwit_proto::ingest::{ClosedShards, IngestV2Error, ShardState};
 use quickwit_proto::metastore;
 use quickwit_proto::metastore::{MetastoreService, MetastoreServiceClient};
 use quickwit_proto::types::{IndexUid, NodeId};
@@ -188,7 +188,7 @@ impl IngestController {
                     index_id=%index_uid.index_id(),
                     source_id=%source_id,
                     shard_ids=?PrettySample::new(&closed_shard_ids, 5),
-                    "closed {} shards reported by router",
+                    "closed {} shard(s) reported by router",
                     closed_shard_ids.len()
                 );
             }
diff --git a/quickwit/quickwit-control-plane/src/lib.rs b/quickwit/quickwit-control-plane/src/lib.rs
index cc0022bef08..d800544f77b 100644
--- a/quickwit/quickwit-control-plane/src/lib.rs
+++ b/quickwit/quickwit-control-plane/src/lib.rs
@@ -25,7 +25,7 @@ pub mod ingest;
 pub(crate) mod metrics;
 
 use quickwit_common::tower::Pool;
-use quickwit_proto::indexing::{IndexingServiceClient, IndexingTask};
+use quickwit_proto::indexing::{CpuCapacity, IndexingServiceClient, IndexingTask};
 use quickwit_proto::types::{IndexUid, SourceId};
 
 /// It can however appear only once in a given index.
@@ -41,6 +41,7 @@ pub struct SourceUid {
 pub struct IndexerNodeInfo {
     pub client: IndexingServiceClient,
     pub indexing_tasks: Vec<IndexingTask>,
+    pub indexing_capacity: CpuCapacity,
 }
 
 pub type IndexerPool = Pool<String, IndexerNodeInfo>;
diff --git a/quickwit/quickwit-control-plane/src/tests.rs b/quickwit/quickwit-control-plane/src/tests.rs
index 384b6681316..2c3d4b8fe29 100644
--- a/quickwit/quickwit-control-plane/src/tests.rs
+++ b/quickwit/quickwit-control-plane/src/tests.rs
@@ -31,7 +31,7 @@ use quickwit_config::service::QuickwitService;
 use quickwit_config::{KafkaSourceParams, SourceConfig, SourceInputFormat, SourceParams};
 use quickwit_indexing::IndexingService;
 use quickwit_metastore::{IndexMetadata, ListIndexesMetadataResponseExt};
-use quickwit_proto::indexing::{ApplyIndexingPlanRequest, IndexingServiceClient};
+use quickwit_proto::indexing::{ApplyIndexingPlanRequest, CpuCapacity, IndexingServiceClient};
 use quickwit_proto::metastore::{
     ListIndexesMetadataResponse, ListShardsResponse, MetastoreServiceClient,
 };
@@ -91,6 +91,7 @@ pub fn test_indexer_change_stream(
                         IndexerNodeInfo {
                             client,
                             indexing_tasks,
+                            indexing_capacity: CpuCapacity::from_cpu_millis(4_000),
                         },
                     ))
                 }
@@ -156,6 +157,7 @@ async fn start_control_plane(
 
 #[tokio::test]
 async fn test_scheduler_scheduling_and_control_loop_apply_plan_again() {
+    quickwit_common::setup_logging_for_tests();
     let transport = ChannelTransport::default();
     let cluster =
         create_cluster_for_test(Vec::new(), &["indexer", "control_plane"], &transport, true)
diff --git a/quickwit/quickwit-doc-mapper/src/default_doc_mapper/default_mapper.rs b/quickwit/quickwit-doc-mapper/src/default_doc_mapper/default_mapper.rs
index a8438fea678..cf4bf30a9d9 100644
--- a/quickwit/quickwit-doc-mapper/src/default_doc_mapper/default_mapper.rs
+++ b/quickwit/quickwit-doc-mapper/src/default_doc_mapper/default_mapper.rs
@@ -25,13 +25,13 @@ use fnv::FnvHashSet;
 use quickwit_common::PathHasher;
 use quickwit_query::create_default_quickwit_tokenizer_manager;
 use quickwit_query::query_ast::QueryAst;
+use quickwit_query::tokenizers::TokenizerManager;
 use serde::{Deserialize, Serialize};
 use serde_json::{self, Value as JsonValue};
 use tantivy::query::Query;
 use tantivy::schema::{
     Field, FieldType, FieldValue, OwnedValue as TantivyValue, Schema, INDEXED, STORED,
 };
-use tantivy::tokenizer::TokenizerManager;
 use tantivy::TantivyDocument as Document;
 
 use super::field_mapping_entry::RAW_TOKENIZER_NAME;
@@ -178,7 +178,7 @@ impl TryFrom<DefaultDocMapperBuilder> for DefaultDocMapper {
                 );
             }
             if tokenizer_manager
-                .get(&tokenizer_config_entry.name)
+                .get_tokenizer(&tokenizer_config_entry.name)
                 .is_some()
             {
                 bail!(
@@ -197,7 +197,12 @@ impl TryFrom<DefaultDocMapperBuilder> for DefaultDocMapper {
                         error
                     )
                 })?;
-            tokenizer_manager.register(&tokenizer_config_entry.name, tokenizer);
+            let does_lowercasing = tokenizer_config_entry
+                .config
+                .filters
+                .iter()
+                .any(|filter| matches!(filter, crate::TokenFilterType::LowerCaser));
+            tokenizer_manager.register(&tokenizer_config_entry.name, tokenizer, does_lowercasing);
             custom_tokenizer_names.insert(&tokenizer_config_entry.name);
         }
         validate_fields_tokenizers(&schema, &tokenizer_manager)?;
@@ -329,7 +334,7 @@ fn validate_fields_tokenizers(
             _ => None,
         };
         if let Some(tokenizer_name) = tokenizer_name_opt {
-            if tokenizer_manager.get(tokenizer_name).is_none() {
+            if tokenizer_manager.get_tokenizer(tokenizer_name).is_none() {
                 bail!(
                     "unknown tokenizer `{}` for field `{}`",
                     tokenizer_name,
@@ -1857,7 +1862,10 @@ mod tests {
             }
             _ => panic!("Expected a text field"),
         }
-        assert!(mapper.tokenizer_manager().get("my_tokenizer").is_some());
+        assert!(mapper
+            .tokenizer_manager()
+            .get_tokenizer("my_tokenizer")
+            .is_some());
     }
 
     #[test]
@@ -1902,7 +1910,10 @@ mod tests {
         }"#,
         )
         .unwrap();
-        let mut tokenizer = mapper.tokenizer_manager().get("my_tokenizer").unwrap();
+        let mut tokenizer = mapper
+            .tokenizer_manager()
+            .get_tokenizer("my_tokenizer")
+            .unwrap();
         let mut token_stream = tokenizer.token_stream("HELLO WORLD");
         assert_eq!(token_stream.next().unwrap().text, "hel");
         assert_eq!(token_stream.next().unwrap().text, "hell");
@@ -1959,8 +1970,11 @@ mod tests {
         }"#,
         )
         .unwrap();
-        let mut default_tokenizer = mapper.tokenizer_manager().get("default").unwrap();
-        let mut tokenizer = mapper.tokenizer_manager().get("my_tokenizer").unwrap();
+        let mut default_tokenizer = mapper.tokenizer_manager().get_tokenizer("default").unwrap();
+        let mut tokenizer = mapper
+            .tokenizer_manager()
+            .get_tokenizer("my_tokenizer")
+            .unwrap();
         let text = "I've seen things... seen things you little people wouldn't believe.";
         let mut default_token_stream = default_tokenizer.token_stream(text);
         let mut token_stream = tokenizer.token_stream(text);
diff --git a/quickwit/quickwit-doc-mapper/src/doc_mapper.rs b/quickwit/quickwit-doc-mapper/src/doc_mapper.rs
index bc81198375d..e47101707d6 100644
--- a/quickwit/quickwit-doc-mapper/src/doc_mapper.rs
+++ b/quickwit/quickwit-doc-mapper/src/doc_mapper.rs
@@ -25,10 +25,10 @@ use std::ops::Bound;
 use anyhow::Context;
 use dyn_clone::{clone_trait_object, DynClone};
 use quickwit_query::query_ast::QueryAst;
+use quickwit_query::tokenizers::TokenizerManager;
 use serde_json::Value as JsonValue;
 use tantivy::query::Query;
 use tantivy::schema::{Field, FieldType, OwnedValue as Value, Schema};
-use tantivy::tokenizer::TokenizerManager;
 use tantivy::{TantivyDocument as Document, Term};
 
 pub type Partition = u64;
diff --git a/quickwit/quickwit-doc-mapper/src/query_builder.rs b/quickwit/quickwit-doc-mapper/src/query_builder.rs
index b0532ec0d9e..9532313178f 100644
--- a/quickwit/quickwit-doc-mapper/src/query_builder.rs
+++ b/quickwit/quickwit-doc-mapper/src/query_builder.rs
@@ -22,13 +22,13 @@ use std::convert::Infallible;
 use std::ops::Bound;
 
 use quickwit_query::query_ast::{
-    FieldPresenceQuery, FullTextMode, FullTextQuery, PhrasePrefixQuery, QueryAst, QueryAstVisitor,
-    RangeQuery, TermSetQuery,
+    FieldPresenceQuery, FullTextQuery, PhrasePrefixQuery, QueryAst, QueryAstVisitor, RangeQuery,
+    TermSetQuery, WildcardQuery,
 };
+use quickwit_query::tokenizers::TokenizerManager;
 use quickwit_query::{find_field_or_hit_dynamic, InvalidQuery};
 use tantivy::query::Query;
 use tantivy::schema::{Field, Schema};
-use tantivy::tokenizer::TokenizerManager;
 use tantivy::Term;
 
 use crate::{QueryParserError, TermRange, WarmupInfo};
@@ -215,16 +215,13 @@ impl<'a, 'b: 'a> QueryAstVisitor<'a> for ExtractPrefixTermRanges<'b> {
     type Err = InvalidQuery;
 
     fn visit_full_text(&mut self, full_text_query: &'a FullTextQuery) -> Result<(), Self::Err> {
-        if let FullTextMode::BoolPrefix {
-            operator: _,
-            max_expansions,
-        } = &full_text_query.params.mode
+        if let Some(prefix_term) =
+            full_text_query.get_prefix_term(self.schema, self.tokenizer_manager)
         {
-            if let Some(prefix_term) =
-                full_text_query.get_last_term(self.schema, self.tokenizer_manager)
-            {
-                self.add_prefix_term(prefix_term, *max_expansions, false);
-            }
+            // the max_expansion expansion of a bool prefix query is used for the fuzzy part of the
+            // query, not for the expension to a range request.
+            // see https://github.com/elastic/elasticsearch/blob/6ad48306d029e6e527c0481e2e9880bd2f06b239/docs/reference/query-dsl/match-bool-prefix-query.asciidoc#parameters
+            self.add_prefix_term(prefix_term, u32::MAX, false);
         }
         Ok(())
     }
@@ -235,10 +232,16 @@ impl<'a, 'b: 'a> QueryAstVisitor<'a> for ExtractPrefixTermRanges<'b> {
     ) -> Result<(), Self::Err> {
         let (_, terms) = phrase_prefix.get_terms(self.schema, self.tokenizer_manager)?;
         if let Some((_, term)) = terms.last() {
-            self.add_prefix_term(term.clone(), phrase_prefix.max_expansions, true);
+            self.add_prefix_term(term.clone(), phrase_prefix.max_expansions, terms.len() > 1);
         }
         Ok(())
     }
+
+    fn visit_wildcard(&mut self, wildcard_query: &'a WildcardQuery) -> Result<(), Self::Err> {
+        let (_, term) = wildcard_query.extract_prefix_term(self.schema, self.tokenizer_manager)?;
+        self.add_prefix_term(term, u32::MAX, false);
+        Ok(())
+    }
 }
 
 fn extract_prefix_term_ranges(
@@ -253,16 +256,19 @@ fn extract_prefix_term_ranges(
 
 #[cfg(test)]
 mod test {
+    use quickwit_datetime::{parse_date_time_str, DateTimeInputFormat};
     use quickwit_query::create_default_quickwit_tokenizer_manager;
     use quickwit_query::query_ast::query_ast_from_user_text;
+    use tantivy::columnar::MonotonicallyMappableToU64;
     use tantivy::schema::{Schema, FAST, INDEXED, STORED, TEXT};
+    use tantivy::{DateOptions, DateTime, DateTimePrecision};
 
     use super::build_query;
     use crate::{DYNAMIC_FIELD_NAME, SOURCE_FIELD_NAME};
 
-    enum TestExpectation {
-        Err(&'static str),
-        Ok(&'static str),
+    enum TestExpectation<'a> {
+        Err(&'a str),
+        Ok(&'a str),
     }
 
     fn make_schema(dynamic_mode: bool) -> Schema {
@@ -276,7 +282,10 @@ mod test {
         schema_builder.add_ip_addr_field("ip", FAST | STORED);
         schema_builder.add_ip_addr_field("ips", FAST);
         schema_builder.add_ip_addr_field("ip_notff", STORED);
-        schema_builder.add_date_field("dt", FAST);
+        let date_options = DateOptions::default()
+            .set_fast()
+            .set_precision(tantivy::DateTimePrecision::Milliseconds);
+        schema_builder.add_date_field("dt", date_options);
         schema_builder.add_u64_field("u64_fast", FAST | STORED);
         schema_builder.add_i64_field("i64_fast", FAST | STORED);
         schema_builder.add_f64_field("f64_fast", FAST | STORED);
@@ -474,16 +483,56 @@ mod test {
 
     #[test]
     fn test_datetime_range_query() {
-        check_build_query_static_mode(
-            "dt:[2023-01-10T15:13:35Z TO 2023-01-10T15:13:40Z]",
-            Vec::new(),
-            TestExpectation::Ok("RangeQuery { field: \"dt\", value_type: Date"),
-        );
-        check_build_query_static_mode(
-            "dt:<2023-01-10T15:13:35Z",
-            Vec::new(),
-            TestExpectation::Ok("RangeQuery { field: \"dt\", value_type: Date"),
-        );
+        let input_formats = [DateTimeInputFormat::Rfc3339];
+        {
+            // Check range on datetime in millisecond, precision has no impact as it is in
+            // milliseconds.
+            let start_date_time_str = "2023-01-10T08:38:51.150Z";
+            let start_date_time = parse_date_time_str(start_date_time_str, &input_formats).unwrap();
+            let start_date_time_u64 = start_date_time.to_u64();
+            let end_date_time_str = "2023-01-10T08:38:51.160Z";
+            let end_date_time: DateTime =
+                parse_date_time_str(end_date_time_str, &input_formats).unwrap();
+            let end_date_time_u64 = end_date_time.to_u64();
+            let expectation_with_lower_and_upper_bounds = format!(
+                r#"FastFieldRangeWeight {{ field: "dt", lower_bound: Included({start_date_time_u64}), upper_bound: Included({end_date_time_u64}), column_type_opt: Some(DateTime) }}"#,
+            );
+            check_build_query_static_mode(
+                &format!("dt:[{start_date_time_str} TO {end_date_time_str}]"),
+                Vec::new(),
+                TestExpectation::Ok(&expectation_with_lower_and_upper_bounds),
+            );
+            let expectation_with_upper_bound = format!(
+                r#"FastFieldRangeWeight {{ field: "dt", lower_bound: Unbounded, upper_bound: Excluded({end_date_time_u64}), column_type_opt: Some(DateTime) }}"#,
+            );
+            check_build_query_static_mode(
+                &format!("dt:<{end_date_time_str}"),
+                Vec::new(),
+                TestExpectation::Ok(&expectation_with_upper_bound),
+            );
+        }
+
+        // Check range on datetime in microseconds and truncation to milliseconds.
+        {
+            let start_date_time_str = "2023-01-10T08:38:51.000150Z";
+            let start_date_time = parse_date_time_str(start_date_time_str, &input_formats)
+                .unwrap()
+                .truncate(DateTimePrecision::Milliseconds);
+            let start_date_time_u64 = start_date_time.to_u64();
+            let end_date_time_str = "2023-01-10T08:38:51.000151Z";
+            let end_date_time: DateTime = parse_date_time_str(end_date_time_str, &input_formats)
+                .unwrap()
+                .truncate(DateTimePrecision::Milliseconds);
+            let end_date_time_u64 = end_date_time.to_u64();
+            let expectation_with_lower_and_upper_bounds = format!(
+                r#"FastFieldRangeWeight {{ field: "dt", lower_bound: Included({start_date_time_u64}), upper_bound: Included({end_date_time_u64}), column_type_opt: Some(DateTime) }}"#,
+            );
+            check_build_query_static_mode(
+                &format!("dt:[{start_date_time_str} TO {end_date_time_str}]"),
+                Vec::new(),
+                TestExpectation::Ok(&expectation_with_lower_and_upper_bounds),
+            );
+        }
     }
 
     #[test]
diff --git a/quickwit/quickwit-doc-mapper/src/tag_pruning.rs b/quickwit/quickwit-doc-mapper/src/tag_pruning.rs
index 78184a0194d..a41ecfd4795 100644
--- a/quickwit/quickwit-doc-mapper/src/tag_pruning.rs
+++ b/quickwit/quickwit-doc-mapper/src/tag_pruning.rs
@@ -98,13 +98,21 @@ fn extract_unsimplified_tags_filter_ast(query_ast: QueryAst) -> UnsimplifiedTagF
             }
         }
         QueryAst::PhrasePrefix(phrase_prefix_query) => {
-            // TODO same as Phrase above.
+            // TODO same as FullText above.
             UnsimplifiedTagFilterAst::Tag {
                 is_present: true,
                 field: phrase_prefix_query.field,
                 value: phrase_prefix_query.phrase,
             }
         }
+        QueryAst::Wildcard(wildcard_query) => {
+            // TODO same as FullText above.
+            UnsimplifiedTagFilterAst::Tag {
+                is_present: true,
+                field: wildcard_query.field,
+                value: wildcard_query.value,
+            }
+        }
         QueryAst::Boost { underlying, .. } => extract_unsimplified_tags_filter_ast(*underlying),
         QueryAst::UserInput(_user_text_query) => {
             panic!("Extract unsimplified should only be called on AST without UserInputQuery.");
diff --git a/quickwit/quickwit-index-management/Cargo.toml b/quickwit/quickwit-index-management/Cargo.toml
index cec81a19dcb..69c4434b347 100644
--- a/quickwit/quickwit-index-management/Cargo.toml
+++ b/quickwit/quickwit-index-management/Cargo.toml
@@ -12,7 +12,6 @@ documentation = "https://quickwit.io/docs/"
 [dependencies]
 anyhow = { workspace = true }
 async-trait = { workspace = true }
-byte-unit = { workspace = true }
 futures = { workspace = true }
 futures-util = { workspace = true }
 itertools = { workspace = true }
diff --git a/quickwit/quickwit-indexing/Cargo.toml b/quickwit/quickwit-indexing/Cargo.toml
index d10b3188fc7..2752b22bad5 100644
--- a/quickwit/quickwit-indexing/Cargo.toml
+++ b/quickwit/quickwit-indexing/Cargo.toml
@@ -18,8 +18,8 @@ anyhow = { workspace = true }
 arc-swap = { workspace = true }
 async-trait = { workspace = true }
 backoff = { workspace = true, optional = true }
-byte-unit = { workspace = true }
 bytes = { workspace = true }
+bytesize = { workspace = true }
 chitchat = { workspace = true }
 fail = { workspace = true }
 flume = { workspace = true }
@@ -50,7 +50,6 @@ tracing = { workspace = true }
 ulid = { workspace = true }
 utoipa = { workspace = true }
 vrl = { workspace = true, optional = true }
-vrl-stdlib = { workspace = true, optional = true }
 
 quickwit-actors = { workspace = true }
 quickwit-aws = { workspace = true }
@@ -70,18 +69,18 @@ gcp-pubsub = ["dep:google-cloud-pubsub", "dep:google-cloud-default", "dep:google
 gcp-pubsub-emulator-tests = []
 kafka = ["rdkafka", "backoff"]
 kafka-broker-tests = []
-vendored-kafka = ["kafka", "libz-sys/static", "openssl/vendored", "rdkafka/gssapi-vendored"]
-vendored-kafka-macos = ["kafka", "libz-sys/static", "openssl/vendored"]
 kinesis = ["aws-config", "aws-smithy-client", "aws-sdk-kinesis", "quickwit-aws/kinesis"]
 kinesis-localstack-tests = []
-vrl = ["dep:vrl", "vrl-stdlib", "quickwit-config/vrl"]
 pulsar = ["dep:pulsar"]
 pulsar-broker-tests = []
+vendored-kafka = ["kafka", "libz-sys/static", "openssl/vendored", "rdkafka/gssapi-vendored"]
+vendored-kafka-macos = ["kafka", "libz-sys/static", "openssl/vendored"]
 testsuite = [
   "quickwit-actors/testsuite",
   "quickwit-cluster/testsuite",
   "quickwit-common/testsuite",
 ]
+vrl = ["dep:vrl", "quickwit-config/vrl"]
 
 [dev-dependencies]
 bytes = { workspace = true }
diff --git a/quickwit/quickwit-indexing/src/actors/indexer.rs b/quickwit/quickwit-indexing/src/actors/indexer.rs
index 6c6c3b8cbc9..b13823fbf01 100644
--- a/quickwit/quickwit-indexing/src/actors/indexer.rs
+++ b/quickwit/quickwit-indexing/src/actors/indexer.rs
@@ -25,7 +25,7 @@ use std::time::{Duration, Instant};
 
 use anyhow::Context;
 use async_trait::async_trait;
-use byte_unit::Byte;
+use bytesize::ByteSize;
 use fail::fail_point;
 use fnv::FnvHashMap;
 use itertools::Itertools;
@@ -38,7 +38,9 @@ use quickwit_common::temp_dir::TempDirectory;
 use quickwit_config::IndexingSettings;
 use quickwit_doc_mapper::DocMapper;
 use quickwit_metastore::checkpoint::{IndexCheckpointDelta, SourceCheckpointDelta};
-use quickwit_proto::indexing::{IndexingPipelineId, PipelineMetrics};
+use quickwit_proto::indexing::{
+    CpuCapacity, IndexingPipelineId, PipelineMetrics, PIPELINE_FULL_CAPACITY,
+};
 use quickwit_proto::metastore::{
     LastDeleteOpstampRequest, MetastoreService, MetastoreServiceClient,
 };
@@ -110,7 +112,11 @@ impl IndexerState {
             .settings(self.index_settings.clone())
             .schema(self.schema.clone())
             .tokenizers(self.tokenizer_manager.clone())
-            .fast_field_tokenizers(get_quickwit_fastfield_normalizer_manager().clone());
+            .fast_field_tokenizers(
+                get_quickwit_fastfield_normalizer_manager()
+                    .tantivy_manager()
+                    .clone(),
+            );
 
         let io_controls = IoControls::default()
             .set_progress(ctx.progress().clone())
@@ -226,7 +232,7 @@ impl IndexerState {
             publish_lock,
             publish_token_opt,
             last_delete_opstamp,
-            memory_usage: Byte::from_bytes(0),
+            memory_usage: ByteSize(0),
         };
         Ok(workbench)
     }
@@ -317,7 +323,7 @@ impl IndexerState {
             memory_usage_delta += mem_usage_after - mem_usage_before;
             ctx.record_progress();
         }
-        *memory_usage = Byte::from_bytes(memory_usage.get_bytes() + memory_usage_delta);
+        *memory_usage = ByteSize(memory_usage.as_u64() + memory_usage_delta);
         Ok(())
     }
 }
@@ -343,7 +349,7 @@ struct IndexingWorkbench {
     // We use this value to set the `delete_opstamp` of the workbench splits.
     last_delete_opstamp: u64,
     // Number of bytes declared as used by tantivy.
-    memory_usage: Byte,
+    memory_usage: ByteSize,
 }
 
 pub struct Indexer {
@@ -534,7 +540,7 @@ impl Indexer {
                 publish_lock: PublishLock::default(),
                 publish_token_opt: None,
                 schema,
-                tokenizer_manager,
+                tokenizer_manager: tokenizer_manager.tantivy_manager().clone(),
                 index_settings,
                 max_num_partitions: doc_mapper.max_num_partitions(),
                 cooperative_indexing_permits,
@@ -547,11 +553,9 @@ impl Indexer {
 
     fn update_pipeline_metrics(&mut self, elapsed: Duration, uncompressed_num_bytes: u64) {
         let commit_timeout = self.indexer_state.indexing_settings.commit_timeout();
-        let cpu_millis: u16 = if elapsed >= commit_timeout {
-            1_000
-        } else {
-            (elapsed.as_micros() * 1_000 / commit_timeout.as_micros()) as u16
-        };
+        let pipeline_throughput_fraction =
+            (elapsed.as_micros() as f32 / commit_timeout.as_micros() as f32).min(1.0f32);
+        let cpu_millis: CpuCapacity = PIPELINE_FULL_CAPACITY * pipeline_throughput_fraction;
         self.counters.pipeline_metrics_opt = Some(PipelineMetrics {
             cpu_millis,
             throughput_mb_per_sec: (uncompressed_num_bytes / (1u64 + elapsed.as_micros() as u64))
@@ -559,11 +563,11 @@ impl Indexer {
         });
     }
 
-    fn memory_usage(&self) -> Byte {
+    fn memory_usage(&self) -> ByteSize {
         if let Some(workbench) = &self.indexing_workbench_opt {
             workbench.memory_usage
         } else {
-            Byte::from_bytes(0)
+            ByteSize(0)
         }
     }
 
@@ -865,7 +869,7 @@ mod tests {
         let body_field = schema.get_field("body").unwrap();
         let indexing_directory = TempDirectory::for_test();
         let mut indexing_settings = IndexingSettings::for_test();
-        indexing_settings.resources.heap_size = Byte::from_bytes(5_000_000);
+        indexing_settings.resources.heap_size = ByteSize::mb(5);
         let (index_serializer_mailbox, index_serializer_inbox) = universe.create_test_mailbox();
         let mut metastore = MetastoreServiceClient::mock();
         metastore.expect_publish_splits().never();
diff --git a/quickwit/quickwit-indexing/src/actors/merge_executor.rs b/quickwit/quickwit-indexing/src/actors/merge_executor.rs
index ec616fe8192..9444a05c22c 100644
--- a/quickwit/quickwit-indexing/src/actors/merge_executor.rs
+++ b/quickwit/quickwit-indexing/src/actors/merge_executor.rs
@@ -41,11 +41,9 @@ use quickwit_proto::metastore::{
 };
 use quickwit_query::get_quickwit_fastfield_normalizer_manager;
 use quickwit_query::query_ast::QueryAst;
-use tantivy::directory::{DirectoryClone, MmapDirectory, RamDirectory};
+use tantivy::directory::{Advice, DirectoryClone, MmapDirectory, RamDirectory};
 use tantivy::tokenizer::TokenizerManager;
-use tantivy::{
-    Advice, DateTime, Directory, Index, IndexMeta, IndexWriter, SegmentId, SegmentReader,
-};
+use tantivy::{DateTime, Directory, Index, IndexMeta, IndexWriter, SegmentId, SegmentReader};
 use tokio::runtime::Handle;
 use tracing::{debug, info, instrument, warn};
 
@@ -296,8 +294,10 @@ impl MergeExecutor {
         merge_scratch_directory: TempDirectory,
         ctx: &ActorContext<Self>,
     ) -> anyhow::Result<IndexedSplit> {
-        let (union_index_meta, split_directories) =
-            open_split_directories(&tantivy_dirs, self.doc_mapper.tokenizer_manager())?;
+        let (union_index_meta, split_directories) = open_split_directories(
+            &tantivy_dirs,
+            self.doc_mapper.tokenizer_manager().tantivy_manager(),
+        )?;
         // TODO it would be nice if tantivy could let us run the merge in the current thread.
         fail_point!("before-merge-split");
         let controlled_directory = self
@@ -316,7 +316,7 @@ impl MergeExecutor {
         // splits.
         let merged_index = open_index(
             controlled_directory.clone(),
-            self.doc_mapper.tokenizer_manager(),
+            self.doc_mapper.tokenizer_manager().tantivy_manager(),
         )?;
         ctx.record_progress();
 
@@ -362,8 +362,10 @@ impl MergeExecutor {
             num_delete_tasks = delete_tasks.len()
         );
 
-        let (union_index_meta, split_directories) =
-            open_split_directories(&tantivy_dirs, self.doc_mapper.tokenizer_manager())?;
+        let (union_index_meta, split_directories) = open_split_directories(
+            &tantivy_dirs,
+            self.doc_mapper.tokenizer_manager().tantivy_manager(),
+        )?;
         let controlled_directory = self
             .merge_split_directories(
                 union_index_meta,
@@ -378,8 +380,17 @@ impl MergeExecutor {
         // This will have the side effect of deleting the directory containing the downloaded split.
         let mut merged_index = Index::open(controlled_directory.clone())?;
         ctx.record_progress();
-        merged_index.set_tokenizers(self.doc_mapper.tokenizer_manager().clone());
-        merged_index.set_fast_field_tokenizers(get_quickwit_fastfield_normalizer_manager().clone());
+        merged_index.set_tokenizers(
+            self.doc_mapper
+                .tokenizer_manager()
+                .tantivy_manager()
+                .clone(),
+        );
+        merged_index.set_fast_field_tokenizers(
+            get_quickwit_fastfield_normalizer_manager()
+                .tantivy_manager()
+                .clone(),
+        );
 
         ctx.record_progress();
 
@@ -470,7 +481,10 @@ impl MergeExecutor {
         ];
         directory_stack.extend(split_directories.into_iter());
         let union_directory = UnionDirectory::union_of(directory_stack);
-        let union_index = open_index(union_directory, self.doc_mapper.tokenizer_manager())?;
+        let union_index = open_index(
+            union_directory,
+            self.doc_mapper.tokenizer_manager().tantivy_manager(),
+        )?;
 
         ctx.record_progress();
         let _protect_guard = ctx.protect_zone();
@@ -532,7 +546,11 @@ fn open_index<T: Into<Box<dyn Directory>>>(
 ) -> tantivy::Result<Index> {
     let mut index = Index::open(directory)?;
     index.set_tokenizers(tokenizer_manager.clone());
-    index.set_fast_field_tokenizers(get_quickwit_fastfield_normalizer_manager().clone());
+    index.set_fast_field_tokenizers(
+        get_quickwit_fastfield_normalizer_manager()
+            .tantivy_manager()
+            .clone(),
+    );
     Ok(index)
 }
 
diff --git a/quickwit/quickwit-indexing/src/actors/merge_pipeline.rs b/quickwit/quickwit-indexing/src/actors/merge_pipeline.rs
index 1a094c579cb..0ca0b00de78 100644
--- a/quickwit/quickwit-indexing/src/actors/merge_pipeline.rs
+++ b/quickwit/quickwit-indexing/src/actors/merge_pipeline.rs
@@ -21,7 +21,7 @@ use std::sync::Arc;
 use std::time::{Duration, Instant};
 
 use async_trait::async_trait;
-use byte_unit::Byte;
+use bytesize::ByteSize;
 use quickwit_actors::{
     Actor, ActorContext, ActorExitStatus, ActorHandle, Handler, Health, Inbox, Mailbox,
     SpawnContext, Supervisable, HEARTBEAT,
@@ -272,7 +272,7 @@ impl MergePipeline {
             .params
             .merge_max_io_num_bytes_per_sec
             .as_ref()
-            .map(|bytes_per_sec| bytes_per_sec.get_bytes() as f64)
+            .map(|bytes_per_sec| bytes_per_sec.as_u64() as f64)
             .unwrap_or(f64::INFINITY);
 
         let split_downloader_io_controls = IoControls::default()
@@ -474,7 +474,7 @@ pub struct MergePipelineParams {
     pub split_store: IndexingSplitStore,
     pub merge_policy: Arc<dyn MergePolicy>,
     pub max_concurrent_split_uploads: usize, //< TODO share with the indexing pipeline.
-    pub merge_max_io_num_bytes_per_sec: Option<Byte>,
+    pub merge_max_io_num_bytes_per_sec: Option<ByteSize>,
     pub event_broker: EventBroker,
 }
 
diff --git a/quickwit/quickwit-indexing/src/actors/packager.rs b/quickwit/quickwit-indexing/src/actors/packager.rs
index 7ba54b82b08..26639e2d6bf 100644
--- a/quickwit/quickwit-indexing/src/actors/packager.rs
+++ b/quickwit/quickwit-indexing/src/actors/packager.rs
@@ -368,9 +368,15 @@ mod tests {
         let index_builder = IndexBuilder::new()
             .settings(IndexSettings::default())
             .schema(schema)
-            .tokenizers(quickwit_query::create_default_quickwit_tokenizer_manager())
+            .tokenizers(
+                quickwit_query::create_default_quickwit_tokenizer_manager()
+                    .tantivy_manager()
+                    .clone(),
+            )
             .fast_field_tokenizers(
-                quickwit_query::get_quickwit_fastfield_normalizer_manager().clone(),
+                quickwit_query::get_quickwit_fastfield_normalizer_manager()
+                    .tantivy_manager()
+                    .clone(),
             );
         let index_directory = MmapDirectory::open(split_scratch_directory.path())?;
         let mut index_writer =
diff --git a/quickwit/quickwit-indexing/src/lib.rs b/quickwit/quickwit-indexing/src/lib.rs
index 61c0bc655b5..ba0169b3d5e 100644
--- a/quickwit/quickwit-indexing/src/lib.rs
+++ b/quickwit/quickwit-indexing/src/lib.rs
@@ -47,6 +47,7 @@ mod split_store;
 #[cfg(any(test, feature = "testsuite"))]
 mod test_utils;
 
+use quickwit_proto::indexing::CpuCapacity;
 #[cfg(any(test, feature = "testsuite"))]
 pub use test_utils::{mock_split, mock_split_meta, MockSplitBuilder, TestSandbox};
 
@@ -54,7 +55,7 @@ use self::merge_policy::MergePolicy;
 pub use self::source::check_source_connectivity;
 
 #[derive(utoipa::OpenApi)]
-#[openapi(components(schemas(IndexingStatistics, PipelineMetrics)))]
+#[openapi(components(schemas(IndexingStatistics, PipelineMetrics, CpuCapacity)))]
 /// Schema used for the OpenAPI generation which are apart of this crate.
 pub struct IndexingApiSchemas;
 
diff --git a/quickwit/quickwit-indexing/src/source/gcp_pubsub_source.rs b/quickwit/quickwit-indexing/src/source/gcp_pubsub_source.rs
index 389ec96d366..a22e5e2c454 100644
--- a/quickwit/quickwit-indexing/src/source/gcp_pubsub_source.rs
+++ b/quickwit/quickwit-indexing/src/source/gcp_pubsub_source.rs
@@ -37,11 +37,10 @@ use serde_json::{json, Value as JsonValue};
 use tokio::time;
 use tracing::{debug, info, warn};
 
-use super::SourceActor;
+use super::{SourceActor, BATCH_NUM_BYTES_LIMIT, EMIT_BATCHES_TIMEOUT};
 use crate::actors::DocProcessor;
 use crate::source::{BatchBuilder, Source, SourceContext, SourceRuntimeArgs, TypedSourceFactory};
 
-const BATCH_NUM_BYTES_LIMIT: u64 = 5_000_000;
 const DEFAULT_MAX_MESSAGES_PER_PULL: i32 = 1_000;
 
 pub struct GcpPubSubSourceFactory;
@@ -168,7 +167,7 @@ impl Source for GcpPubSubSource {
     ) -> Result<Duration, ActorExitStatus> {
         let now = Instant::now();
         let mut batch: BatchBuilder = BatchBuilder::default();
-        let deadline = time::sleep(*quickwit_actors::HEARTBEAT / 2);
+        let deadline = time::sleep(EMIT_BATCHES_TIMEOUT);
         tokio::pin!(deadline);
         // TODO: ensure we ACK the message after being commit: at least once
         // TODO: ensure we increase_ack_deadline for the items
diff --git a/quickwit/quickwit-indexing/src/source/ingest/mod.rs b/quickwit/quickwit-indexing/src/source/ingest/mod.rs
index 1d7aee86ed1..53aa50a4b1e 100644
--- a/quickwit/quickwit-indexing/src/source/ingest/mod.rs
+++ b/quickwit/quickwit-indexing/src/source/ingest/mod.rs
@@ -45,12 +45,11 @@ use ulid::Ulid;
 
 use super::{
     Assignment, BatchBuilder, Source, SourceContext, SourceRuntimeArgs, TypedSourceFactory,
+    BATCH_NUM_BYTES_LIMIT, EMIT_BATCHES_TIMEOUT,
 };
 use crate::actors::DocProcessor;
 use crate::models::{NewPublishLock, NewPublishToken, PublishLock};
 
-const EMIT_BATCHES_TIMEOUT: Duration = Duration::from_millis(if cfg!(test) { 100 } else { 1_000 });
-
 pub struct IngestSourceFactory;
 
 #[async_trait]
@@ -309,7 +308,7 @@ impl Source for IngestSource {
                 Ok(Ok(fetch_payload)) => {
                     self.process_fetch_response(&mut batch_builder, fetch_payload)?;
 
-                    if batch_builder.num_bytes >= 5 * 1024 * 1024 {
+                    if batch_builder.num_bytes >= BATCH_NUM_BYTES_LIMIT {
                         break;
                     }
                 }
diff --git a/quickwit/quickwit-indexing/src/source/kafka_source.rs b/quickwit/quickwit-indexing/src/source/kafka_source.rs
index d2db3477425..52c97722a47 100644
--- a/quickwit/quickwit-indexing/src/source/kafka_source.rs
+++ b/quickwit/quickwit-indexing/src/source/kafka_source.rs
@@ -49,20 +49,10 @@ use tracing::{debug, info, warn};
 
 use crate::actors::DocProcessor;
 use crate::models::{NewPublishLock, PublishLock};
-use crate::source::{BatchBuilder, Source, SourceContext, SourceRuntimeArgs, TypedSourceFactory};
-
-/// Number of bytes after which we cut a new batch.
-///
-/// We try to emit chewable batches for the indexer.
-/// One batch = one message to the indexer actor.
-///
-/// If batches are too large:
-/// - we might not be able to observe the state of the indexer for 5 seconds.
-/// - we will be needlessly occupying resident memory in the mailbox.
-/// - we will not have a precise control of the timeout before commit.
-///
-/// 5MB seems like a good one size fits all value.
-const BATCH_NUM_BYTES_LIMIT: u64 = 5_000_000;
+use crate::source::{
+    BatchBuilder, Source, SourceContext, SourceRuntimeArgs, TypedSourceFactory,
+    BATCH_NUM_BYTES_LIMIT, EMIT_BATCHES_TIMEOUT,
+};
 
 type GroupId = String;
 
@@ -128,7 +118,7 @@ macro_rules! return_if_err {
         match $expression {
             Ok(v) => v,
             Err(_) => {
-                debug!(concat!($lit, "The source was dropped."));
+                debug!(concat!($lit, "the source was dropped"));
                 return;
             }
         }
@@ -148,19 +138,19 @@ impl ConsumerContext for RdKafkaContext {
     fn pre_rebalance(&self, rebalance: &Rebalance) {
         if let Rebalance::Revoke(tpl) = rebalance {
             let partitions = collect_partitions(tpl, &self.topic);
-            debug!(partitions=?partitions, "Revoke partitions.");
+            debug!(partitions=?partitions, "revoke partitions");
 
             let (ack_tx, ack_rx) = oneshot::channel();
             return_if_err!(
                 self.events_tx
                     .blocking_send(KafkaEvent::RevokePartitions { ack_tx }),
-                "Failed to send revoke message to source."
+                "failed to send revoke message to source"
             );
-            return_if_err!(ack_rx.recv(), "Failed to receive revoke ack from source");
+            return_if_err!(ack_rx.recv(), "failed to receive revoke ack from source");
         }
         if let Rebalance::Assign(tpl) = rebalance {
             let partitions = collect_partitions(tpl, &self.topic);
-            debug!(partitions=?partitions, "Assign partitions.");
+            debug!(partitions=?partitions, "assign partitions");
 
             let (assignment_tx, assignment_rx) = oneshot::channel();
             return_if_err!(
@@ -168,19 +158,23 @@ impl ConsumerContext for RdKafkaContext {
                     partitions,
                     assignment_tx,
                 }),
-                "Failed to send assign message to source."
+                "failed to send assign message to source"
             );
             let assignment = return_if_err!(
                 assignment_rx.recv(),
-                "Failed to receive assignment from source."
+                "failed to receive assignment from source"
             );
-            for (partition, offset) in assignment {
-                let mut partition = tpl
-                    .find_partition(&self.topic, partition)
-                    .expect("Failed to find partition in assignment. This should never happen! Please, report on https://github.com/quickwit-oss/quickwit/issues.");
-                partition
-                    .set_offset(offset)
-                    .expect("Failed to convert `Offset` to `i64`. This should never happen! Please, report on https://github.com/quickwit-oss/quickwit/issues.");
+            for (partition_id, offset) in assignment {
+                let Some(mut partition) = tpl.find_partition(&self.topic, partition_id) else {
+                    warn!("partition `{partition_id}` not found in assignment");
+                    continue;
+                };
+                if let Err(error) = partition.set_offset(offset) {
+                    warn!(
+                        "failed to set offset to `{offset:?}` for partition `{partition_id}`: \
+                         {error}"
+                    );
+                }
             }
         }
     }
@@ -373,10 +367,15 @@ impl KafkaSource {
 
         for &partition in partitions {
             let partition_id = PartitionId::from(partition as i64);
-            let current_position = checkpoint
-                .position_for_partition(&partition_id)
-                .cloned()
-                .unwrap_or(Position::Beginning);
+
+            self.state
+                .assigned_partitions
+                .insert(partition, partition_id.clone());
+
+            let Some(current_position) = checkpoint.position_for_partition(&partition_id).cloned()
+            else {
+                continue;
+            };
             let next_offset = match &current_position {
                 Position::Beginning => Offset::Beginning,
                 Position::Offset(offset) => {
@@ -389,9 +388,6 @@ impl KafkaSource {
                     panic!("position of a Kafka partition should never be EOF")
                 }
             };
-            self.state
-                .assigned_partitions
-                .insert(partition, partition_id);
             self.state
                 .current_positions
                 .insert(partition, current_position);
@@ -441,7 +437,7 @@ impl KafkaSource {
             topic=%self.topic,
             partition=%partition,
             num_inactive_partitions=?self.state.num_inactive_partitions,
-            "Reached end of partition."
+            "reached end of partition"
         );
     }
 
@@ -480,7 +476,7 @@ impl Source for KafkaSource {
     ) -> Result<Duration, ActorExitStatus> {
         let now = Instant::now();
         let mut batch = BatchBuilder::default();
-        let deadline = time::sleep(*quickwit_actors::HEARTBEAT / 2);
+        let deadline = time::sleep(EMIT_BATCHES_TIMEOUT);
         tokio::pin!(deadline);
 
         loop {
@@ -901,7 +897,7 @@ mod kafka_broker_tests {
         format!("Key {id}")
     }
 
-    fn get_source_config(topic: &str) -> (String, SourceConfig) {
+    fn get_source_config(topic: &str, auto_offset_reset: &str) -> (String, SourceConfig) {
         let source_id = append_random_suffix("test-kafka-source--source");
         let source_config = SourceConfig {
             source_id: source_id.clone(),
@@ -912,6 +908,7 @@ mod kafka_broker_tests {
                 topic: topic.to_string(),
                 client_log_level: None,
                 client_params: json!({
+                    "auto.offset.reset": auto_offset_reset,
                     "bootstrap.servers": "localhost:9092",
                 }),
                 enable_backfill_mode: true,
@@ -1003,7 +1000,7 @@ mod kafka_broker_tests {
         let metastore = metastore_for_test();
         let index_id = append_random_suffix("test-kafka-source--process-message--index");
         let index_uid = IndexUid::new_with_random_ulid(&index_id);
-        let (_source_id, source_config) = get_source_config(&topic);
+        let (_source_id, source_config) = get_source_config(&topic, "earliest");
         let SourceParams::Kafka(params) = source_config.clone().source_params else {
             panic!(
                 "Expected Kafka source params, got {:?}.",
@@ -1127,7 +1124,7 @@ mod kafka_broker_tests {
 
         let metastore = metastore_for_test();
         let index_id = append_random_suffix("test-kafka-source--process-assign-partitions--index");
-        let (source_id, source_config) = get_source_config(&topic);
+        let (source_id, source_config) = get_source_config(&topic, "earliest");
 
         let index_uid = setup_index(metastore.clone(), &index_id, &source_id, &[(2, -1, 42)]).await;
 
@@ -1169,18 +1166,14 @@ mod kafka_broker_tests {
             kafka_source.state.assigned_partitions,
             expected_assigned_partitions
         );
-        let expected_current_positions =
-            HashMap::from_iter([(1, Position::Beginning), (2, Position::from(42u64))]);
+        let expected_current_positions = HashMap::from_iter([(2, Position::from(42u64))]);
         assert_eq!(
             kafka_source.state.current_positions,
             expected_current_positions
         );
 
         let assignment = assignment_rx.await.unwrap();
-        assert_eq!(
-            assignment,
-            &[(1, Offset::Beginning), (2, Offset::Offset(43))]
-        )
+        assert_eq!(assignment, &[(2, Offset::Offset(43))])
     }
 
     #[tokio::test]
@@ -1192,7 +1185,7 @@ mod kafka_broker_tests {
         let metastore = metastore_for_test();
         let index_id = append_random_suffix("test-kafka-source--process-revoke--partitions--index");
         let index_uid = IndexUid::new_with_random_ulid(&index_id);
-        let (_source_id, source_config) = get_source_config(&topic);
+        let (_source_id, source_config) = get_source_config(&topic, "earliest");
         let SourceParams::Kafka(params) = source_config.clone().source_params else {
             panic!(
                 "Expected Kafka source params, got {:?}.",
@@ -1250,7 +1243,7 @@ mod kafka_broker_tests {
         let metastore = metastore_for_test();
         let index_id = append_random_suffix("test-kafka-source--process-partition-eof--index");
         let index_uid = IndexUid::new_with_random_ulid(&index_id);
-        let (_source_id, source_config) = get_source_config(&topic);
+        let (_source_id, source_config) = get_source_config(&topic, "earliest");
         let SourceParams::Kafka(params) = source_config.clone().source_params else {
             panic!(
                 "Expected Kafka source params, got {:?}.",
@@ -1288,7 +1281,7 @@ mod kafka_broker_tests {
 
         let metastore = metastore_for_test();
         let index_id = append_random_suffix("test-kafka-source--suggest-truncate--index");
-        let (source_id, source_config) = get_source_config(&topic);
+        let (source_id, source_config) = get_source_config(&topic, "earliest");
 
         let index_uid = setup_index(metastore.clone(), &index_id, &source_id, &[(2, -1, 42)]).await;
 
@@ -1369,9 +1362,10 @@ mod kafka_broker_tests {
 
         let source_loader = quickwit_supported_sources();
         {
+            // Test Kafka source with empty topic.
             let metastore = metastore_for_test();
             let index_id = append_random_suffix("test-kafka-source--index");
-            let (source_id, source_config) = get_source_config(&topic);
+            let (source_id, source_config) = get_source_config(&topic, "earliest");
             let index_uid = setup_index(metastore.clone(), &index_id, &source_id, &[]).await;
             let source = source_loader
                 .load_source(
@@ -1401,7 +1395,7 @@ mod kafka_broker_tests {
                 "source_id": source_id,
                 "topic":  topic,
                 "assigned_partitions": vec![0, 1, 2],
-                "current_positions": vec![(0, ""), (1, ""), (2, "")],
+                "current_positions": json!([]),
                 "num_inactive_partitions": 3,
                 "num_bytes_processed": 0,
                 "num_messages_processed": 0,
@@ -1430,7 +1424,7 @@ mod kafka_broker_tests {
         {
             let metastore = metastore_for_test();
             let index_id = append_random_suffix("test-kafka-source--index");
-            let (source_id, source_config) = get_source_config(&topic);
+            let (source_id, source_config) = get_source_config(&topic, "earliest");
             let index_uid = setup_index(metastore.clone(), &index_id, &source_id, &[]).await;
             let source = source_loader
                 .load_source(
@@ -1491,9 +1485,10 @@ mod kafka_broker_tests {
             assert_eq!(exit_state, expected_state);
         }
         {
+            // Test Kafka source with `earliest` offset reset.
             let metastore = metastore_for_test();
             let index_id = append_random_suffix("test-kafka-source--index");
-            let (source_id, source_config) = get_source_config(&topic);
+            let (source_id, source_config) = get_source_config(&topic, "earliest");
             let index_uid = setup_index(
                 metastore.clone(),
                 &index_id,
@@ -1556,6 +1551,49 @@ mod kafka_broker_tests {
             });
             assert_eq!(exit_state, expected_exit_state);
         }
+        {
+            // Test Kafka source with `latest` offset reset.
+            let metastore = metastore_for_test();
+            let index_id = append_random_suffix("test-kafka-source--index");
+            let (source_id, source_config) = get_source_config(&topic, "latest");
+            let index_uid = setup_index(metastore.clone(), &index_id, &source_id, &[]).await;
+            let source = source_loader
+                .load_source(
+                    SourceRuntimeArgs::for_test(
+                        index_uid,
+                        source_config,
+                        metastore,
+                        PathBuf::from("./queues"),
+                    ),
+                    SourceCheckpoint::default(),
+                )
+                .await?;
+            let (doc_processor_mailbox, doc_processor_inbox) = universe.create_test_mailbox();
+            let source_actor = SourceActor {
+                source,
+                doc_processor_mailbox: doc_processor_mailbox.clone(),
+            };
+            let (_source_mailbox, source_handle) = universe.spawn_builder().spawn(source_actor);
+            let (exit_status, exit_state) = source_handle.join().await;
+            assert!(exit_status.is_success());
+
+            let messages: Vec<RawDocBatch> = doc_processor_inbox.drain_for_test_typed();
+            assert!(messages.is_empty());
+
+            let expected_state = json!({
+                "index_id": index_id,
+                "source_id": source_id,
+                "topic":  topic,
+                "assigned_partitions": vec![0, 1, 2],
+                "current_positions": json!([]),
+                "num_inactive_partitions": 3,
+                "num_bytes_processed": 0,
+                "num_messages_processed": 0,
+                "num_invalid_messages": 0,
+                "num_rebalances": 0,
+            });
+            assert_eq!(exit_state, expected_state);
+        }
         Ok(())
     }
 
diff --git a/quickwit/quickwit-indexing/src/source/kinesis/kinesis_source.rs b/quickwit/quickwit-indexing/src/source/kinesis/kinesis_source.rs
index 7b3ba961781..5df7892a39f 100644
--- a/quickwit/quickwit-indexing/src/source/kinesis/kinesis_source.rs
+++ b/quickwit/quickwit-indexing/src/source/kinesis/kinesis_source.rs
@@ -43,9 +43,10 @@ use super::shard_consumer::{ShardConsumer, ShardConsumerHandle, ShardConsumerMes
 use crate::actors::DocProcessor;
 use crate::models::RawDocBatch;
 use crate::source::kinesis::helpers::get_kinesis_client;
-use crate::source::{Source, SourceContext, SourceRuntimeArgs, TypedSourceFactory};
-
-const TARGET_BATCH_NUM_BYTES: u64 = 5_000_000;
+use crate::source::{
+    Source, SourceContext, SourceRuntimeArgs, TypedSourceFactory, BATCH_NUM_BYTES_LIMIT,
+    EMIT_BATCHES_TIMEOUT,
+};
 
 type ShardId = String;
 
@@ -215,7 +216,7 @@ impl Source for KinesisSource {
         let mut docs = Vec::new();
         let mut checkpoint_delta = SourceCheckpointDelta::default();
 
-        let deadline = time::sleep(*quickwit_actors::HEARTBEAT / 2);
+        let deadline = time::sleep(EMIT_BATCHES_TIMEOUT);
         tokio::pin!(deadline);
 
         loop {
@@ -278,7 +279,7 @@ impl Source for KinesisSource {
                                     ).context("failed to record partition delta")?;
                                 }
                             }
-                            if batch_num_bytes >= TARGET_BATCH_NUM_BYTES {
+                            if batch_num_bytes >= BATCH_NUM_BYTES_LIMIT {
                                 break;
                             }
                         }
diff --git a/quickwit/quickwit-indexing/src/source/mod.rs b/quickwit/quickwit-indexing/src/source/mod.rs
index 09796f7f751..d3325fc2195 100644
--- a/quickwit/quickwit-indexing/src/source/mod.rs
+++ b/quickwit/quickwit-indexing/src/source/mod.rs
@@ -75,10 +75,9 @@ mod void_source;
 use std::path::PathBuf;
 use std::time::Duration;
 
-#[cfg(not(any(feature = "kafka", feature = "kinesis", feature = "pulsar")))]
-use anyhow::bail;
 use async_trait::async_trait;
 use bytes::Bytes;
+use bytesize::ByteSize;
 pub use file_source::{FileSource, FileSourceFactory};
 #[cfg(feature = "gcp-pubsub")]
 pub use gcp_pubsub_source::{GcpPubSubSource, GcpPubSubSourceFactory};
@@ -111,6 +110,21 @@ use crate::models::RawDocBatch;
 use crate::source::ingest::IngestSourceFactory;
 use crate::source::ingest_api_source::IngestApiSourceFactory;
 
+/// Number of bytes after which we cut a new batch.
+///
+/// We try to emit chewable batches for the indexer.
+/// One batch = one message to the indexer actor.
+///
+/// If batches are too large:
+/// - we might not be able to observe the state of the indexer for 5 seconds.
+/// - we will be needlessly occupying resident memory in the mailbox.
+/// - we will not have a precise control of the timeout before commit.
+///
+/// 5MB seems like a good one size fits all value.
+const BATCH_NUM_BYTES_LIMIT: u64 = ByteSize::mib(5).as_u64();
+
+const EMIT_BATCHES_TIMEOUT: Duration = Duration::from_millis(if cfg!(test) { 100 } else { 1_000 });
+
 /// Runtime configuration used during execution of a source actor.
 pub struct SourceRuntimeArgs {
     pub pipeline_id: IndexingPipelineId,
@@ -398,7 +412,7 @@ pub async fn check_source_connectivity(
         #[allow(unused_variables)]
         SourceParams::Kafka(params) => {
             #[cfg(not(feature = "kafka"))]
-            bail!("Quickwit binary was not compiled with the `kafka` feature");
+            anyhow::bail!("Quickwit binary was not compiled with the `kafka` feature");
 
             #[cfg(feature = "kafka")]
             {
@@ -409,7 +423,7 @@ pub async fn check_source_connectivity(
         #[allow(unused_variables)]
         SourceParams::Kinesis(params) => {
             #[cfg(not(feature = "kinesis"))]
-            bail!("Quickwit binary was not compiled with the `kinesis` feature");
+            anyhow::bail!("Quickwit binary was not compiled with the `kinesis` feature");
 
             #[cfg(feature = "kinesis")]
             {
@@ -420,7 +434,7 @@ pub async fn check_source_connectivity(
         #[allow(unused_variables)]
         SourceParams::Pulsar(params) => {
             #[cfg(not(feature = "pulsar"))]
-            bail!("Quickwit binary was not compiled with the `pulsar` feature");
+            anyhow::bail!("Quickwit binary was not compiled with the `pulsar` feature");
 
             #[cfg(feature = "pulsar")]
             {
diff --git a/quickwit/quickwit-indexing/src/source/pulsar_source.rs b/quickwit/quickwit-indexing/src/source/pulsar_source.rs
index 691609464ce..23063c554ff 100644
--- a/quickwit/quickwit-indexing/src/source/pulsar_source.rs
+++ b/quickwit/quickwit-indexing/src/source/pulsar_source.rs
@@ -42,21 +42,9 @@ use tracing::{debug, info, warn};
 use crate::actors::DocProcessor;
 use crate::source::{
     BatchBuilder, Source, SourceActor, SourceContext, SourceRuntimeArgs, TypedSourceFactory,
+    BATCH_NUM_BYTES_LIMIT, EMIT_BATCHES_TIMEOUT,
 };
 
-/// Number of bytes after which we cut a new batch.
-///
-/// We try to emit chewable batches for the indexer.
-/// One batch = one message to the indexer actor.
-///
-/// If batches are too large:
-/// - we might not be able to observe the state of the indexer for 5 seconds.
-/// - we will be needlessly occupying resident memory in the mailbox.
-/// - we will not have a precise control of the timeout before commit.
-///
-/// 5MB seems like a good one size fits all value.
-const BATCH_NUM_BYTES_LIMIT: u64 = 5_000_000;
-
 type PulsarConsumer = Consumer<PulsarMessage, TokioExecutor>;
 
 pub struct PulsarSourceFactory;
@@ -225,7 +213,7 @@ impl Source for PulsarSource {
     ) -> Result<Duration, ActorExitStatus> {
         let now = Instant::now();
         let mut batch = BatchBuilder::default();
-        let deadline = time::sleep(*quickwit_actors::HEARTBEAT / 2);
+        let deadline = time::sleep(EMIT_BATCHES_TIMEOUT);
         tokio::pin!(deadline);
 
         loop {
@@ -339,7 +327,16 @@ async fn create_pulsar_consumer(
 fn msg_id_to_position(msg: &MessageIdData) -> Position {
     // The order of these fields are important as they affect the sorting
     // of the checkpoint positions.
-    // TODO: Confirm this layout is correct?
+    //
+    // The key parts of the ID used for ordering are:
+    // - The ledger ID which is a sequentially increasing ID.
+    // - The entry ID the unique ID of the message within the ledger.
+    // - The batch position for the current chunk of messages.
+    //
+    // The remaining keys are not required for sorting but are required
+    // in order to re-construct the message ID in order to send back to pulsar.
+    // The ledger_id, entry_id and the batch_index form a unique composite key which will
+    // prevent the remaining parts of the ID from interfering with the sorting.
     let id_str = format!(
         "{:0>20},{:0>20},{},{},{}",
         msg.ledger_id,
diff --git a/quickwit/quickwit-indexing/src/split_store/indexing_split_store.rs b/quickwit/quickwit-indexing/src/split_store/indexing_split_store.rs
index 50a849c9dac..6f5e2a231fe 100644
--- a/quickwit/quickwit-indexing/src/split_store/indexing_split_store.rs
+++ b/quickwit/quickwit-indexing/src/split_store/indexing_split_store.rs
@@ -25,13 +25,13 @@ use std::time::Instant;
 
 use anyhow::Context;
 #[cfg(any(test, feature = "testsuite"))]
-use byte_unit::Byte;
+use bytesize::ByteSize;
 use quickwit_common::io::{IoControls, IoControlsAccess};
 use quickwit_common::uri::Uri;
 use quickwit_metastore::SplitMetadata;
 use quickwit_storage::{PutPayload, Storage, StorageResult};
-use tantivy::directory::MmapDirectory;
-use tantivy::{Advice, Directory};
+use tantivy::directory::{Advice, MmapDirectory};
+use tantivy::Directory;
 use time::OffsetDateTime;
 use tracing::{info, info_span, instrument, Instrument};
 
@@ -234,7 +234,7 @@ impl IndexingSplitStore {
 
     /// Takes a snapshot of the cache view (only used for testing).
     #[cfg(any(test, feature = "testsuite"))]
-    pub async fn inspect_local_store(&self) -> HashMap<String, Byte> {
+    pub async fn inspect_local_store(&self) -> HashMap<String, ByteSize> {
         self.inner.local_split_store.inspect().await
     }
 }
@@ -244,7 +244,7 @@ mod tests {
     use std::sync::Arc;
     use std::time::Duration;
 
-    use byte_unit::Byte;
+    use bytesize::ByteSize;
     use quickwit_common::io::IoControls;
     use quickwit_metastore::{SplitMaturity, SplitMetadata};
     use quickwit_storage::{RamStorage, SplitPayloadBuilder};
@@ -300,7 +300,7 @@ mod tests {
             assert_eq!(local_store_stats.len(), 1);
             assert_eq!(
                 local_store_stats.get(&split_id1).cloned(),
-                Some(Byte::from_bytes(4))
+                Some(ByteSize(4))
             );
         }
         {
@@ -322,11 +322,11 @@ mod tests {
         assert_eq!(local_store_stats.len(), 2);
         assert_eq!(
             local_store_stats.get(&split_id1).cloned(),
-            Some(Byte::from_bytes(4))
+            Some(ByteSize(4))
         );
         assert_eq!(
             local_store_stats.get(&split_id2).cloned(),
-            Some(Byte::from_bytes(3))
+            Some(ByteSize(3))
         );
 
         Ok(())
@@ -339,7 +339,7 @@ mod tests {
         let split_cache_dir = tempdir()?;
         let local_split_store = LocalSplitStore::open(
             split_cache_dir.path().to_path_buf(),
-            SplitStoreQuota::new(1, Byte::from_bytes(1_000_000u64)),
+            SplitStoreQuota::new(1, ByteSize::mb(1)),
         )
         .await?;
 
@@ -370,7 +370,7 @@ mod tests {
             assert_eq!(local_store_stats.len(), 1);
             assert_eq!(
                 local_store_stats.get(&split_id1).cloned(),
-                Some(Byte::from_bytes(11))
+                Some(ByteSize(11))
             );
         }
         {
@@ -395,7 +395,7 @@ mod tests {
             assert_eq!(local_store_stats.len(), 1);
             assert_eq!(
                 local_store_stats.get(&split_id2).cloned(),
-                Some(Byte::from_bytes(12))
+                Some(ByteSize(12))
             );
         }
         {
diff --git a/quickwit/quickwit-indexing/src/split_store/local_split_store.rs b/quickwit/quickwit-indexing/src/split_store/local_split_store.rs
index e436023d60a..c8911ad8ece 100644
--- a/quickwit/quickwit-indexing/src/split_store/local_split_store.rs
+++ b/quickwit/quickwit-indexing/src/split_store/local_split_store.rs
@@ -57,7 +57,7 @@ use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
 use anyhow::Context;
-use byte_unit::Byte;
+use bytesize::ByteSize;
 use quickwit_common::split_file;
 use quickwit_directories::BundleDirectory;
 use quickwit_storage::StorageResult;
@@ -86,7 +86,7 @@ pub fn get_tantivy_directory_from_split_bundle(
 }
 
 /// Returns the number of bytes held in a given directory.
-async fn num_bytes_in_folder(directory_path: &Path) -> io::Result<Byte> {
+async fn num_bytes_in_folder(directory_path: &Path) -> io::Result<ByteSize> {
     let mut total_bytes = 0;
     let mut read_dir = tokio::fs::read_dir(directory_path).await?;
     while let Some(dir_entry) = read_dir.next_entry().await? {
@@ -100,7 +100,7 @@ async fn num_bytes_in_folder(directory_path: &Path) -> io::Result<Byte> {
             );
         }
     }
-    Ok(Byte::from_bytes(total_bytes))
+    Ok(ByteSize(total_bytes))
 }
 
 /// The local split store is a cache for freshly indexed splits.
@@ -109,7 +109,7 @@ async fn num_bytes_in_folder(directory_path: &Path) -> io::Result<Byte> {
 /// of a directory and the splits are built on the file upon upload.
 struct SplitFolder {
     split_id: Ulid,
-    num_bytes: Byte,
+    num_bytes: ByteSize,
 }
 
 impl SplitFolder {
@@ -353,7 +353,7 @@ impl LocalSplitStore {
     }
 
     #[cfg(any(test, feature = "testsuite"))]
-    pub async fn inspect(&self) -> HashMap<String, Byte> {
+    pub async fn inspect(&self) -> HashMap<String, ByteSize> {
         self.inner
             .lock()
             .await
@@ -420,7 +420,7 @@ mod tests {
     use std::io::Write;
     use std::path::Path;
 
-    use byte_unit::Byte;
+    use bytesize::ByteSize;
     use quickwit_directories::BundleDirectory;
     use quickwit_storage::{PutPayload, SplitPayloadBuilder};
     use tantivy::directory::FileSlice;
@@ -454,14 +454,8 @@ mod tests {
             LocalSplitStore::open(temp_dir.path().to_path_buf(), split_store_space_quota).await?;
         let cache_content = split_store.inspect().await;
         assert_eq!(cache_content.len(), 2);
-        assert_eq!(
-            cache_content.get(split_id1).cloned(),
-            Some(Byte::from_bytes(15))
-        );
-        assert_eq!(
-            cache_content.get(split_id2).cloned(),
-            Some(Byte::from_bytes(13))
-        );
+        assert_eq!(cache_content.get(split_id1).cloned(), Some(ByteSize(15)));
+        assert_eq!(cache_content.get(split_id2).cloned(), Some(ByteSize(13)));
         Ok(())
     }
 
@@ -480,7 +474,7 @@ mod tests {
         create_fake_split(dir.path(), "01GF521CZC1260V8QPA81T46X7", 45)
             .await
             .unwrap(); // 2
-        let split_store_space_quota = SplitStoreQuota::new(2, Byte::from_bytes(1_000));
+        let split_store_space_quota = SplitStoreQuota::new(2, ByteSize::kb(1));
         let local_split_store =
             LocalSplitStore::open(dir.path().to_path_buf(), split_store_space_quota)
                 .await
@@ -503,14 +497,14 @@ mod tests {
         create_fake_split(dir.path(), "01GF521CZC1260V8QPA81T46X7", 45)
             .await
             .unwrap(); // 2
-        let split_store_space_quota = SplitStoreQuota::new(6, Byte::from_bytes(61));
+        let split_store_space_quota = SplitStoreQuota::new(6, ByteSize(61));
         let local_split_store =
             LocalSplitStore::open(dir.path().to_path_buf(), split_store_space_quota)
                 .await
                 .unwrap();
         let cache_content = local_split_store.inspect().await;
         assert_eq!(cache_content.len(), 2);
-        assert_eq!(cache_content.values().map(Byte::get_bytes).sum::<u64>(), 50);
+        assert_eq!(cache_content.values().map(|v| v.as_u64()).sum::<u64>(), 50);
     }
 
     #[tokio::test]
@@ -532,7 +526,7 @@ mod tests {
         create_fake_split(dir.path(), "01GF1ZJBMBMEPMAQSFD09VTST2", 1)
             .await
             .unwrap();
-        let split_store_space_quota = SplitStoreQuota::new(6, Byte::from_bytes(100));
+        let split_store_space_quota = SplitStoreQuota::new(6, ByteSize(100));
         let local_split_store =
             LocalSplitStore::open(dir.path().to_path_buf(), split_store_space_quota)
                 .await
diff --git a/quickwit/quickwit-indexing/src/split_store/split_store_quota.rs b/quickwit/quickwit-indexing/src/split_store/split_store_quota.rs
index 9d60b1ec15a..a5c6062c488 100644
--- a/quickwit/quickwit-indexing/src/split_store/split_store_quota.rs
+++ b/quickwit/quickwit-indexing/src/split_store/split_store_quota.rs
@@ -17,7 +17,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 
-use byte_unit::Byte;
+use bytesize::ByteSize;
 use quickwit_config::IndexerConfig;
 
 /// A struct for keeping in check multiple SplitStore.
@@ -26,18 +26,18 @@ pub struct SplitStoreQuota {
     /// Current number of splits in the cache.
     num_splits_in_cache: usize,
     /// Current size in bytes of splits in the cache.
-    size_in_bytes_in_cache: Byte,
+    size_in_bytes_in_cache: ByteSize,
     /// Maximum number of files allowed in the cache.
     max_num_splits: usize,
     /// Maximum size in bytes allowed in the cache.
-    max_num_bytes: Byte,
+    max_num_bytes: ByteSize,
 }
 
 impl Default for SplitStoreQuota {
     fn default() -> Self {
         Self {
             num_splits_in_cache: 0,
-            size_in_bytes_in_cache: Byte::default(),
+            size_in_bytes_in_cache: ByteSize::default(),
             max_num_bytes: IndexerConfig::default_split_store_max_num_bytes(),
             max_num_splits: IndexerConfig::default_split_store_max_num_splits(),
         }
@@ -45,7 +45,7 @@ impl Default for SplitStoreQuota {
 }
 
 impl SplitStoreQuota {
-    pub fn new(max_num_splits: usize, max_num_bytes: Byte) -> Self {
+    pub fn new(max_num_splits: usize, max_num_bytes: ByteSize) -> Self {
         Self {
             max_num_splits,
             max_num_bytes,
@@ -55,70 +55,68 @@ impl SplitStoreQuota {
 
     /// Space quota that prevents any caching.
     pub fn no_caching() -> Self {
-        Self::new(0, Byte::default())
+        Self::new(0, ByteSize::default())
     }
 
-    pub fn can_fit_split(&self, split_size_in_bytes: Byte) -> bool {
+    pub fn can_fit_split(&self, split_size_in_bytes: ByteSize) -> bool {
         if self.num_splits_in_cache >= self.max_num_splits {
             return false;
         }
-        if self.size_in_bytes_in_cache.get_bytes() + split_size_in_bytes.get_bytes()
-            > self.max_num_bytes.get_bytes()
+        if self.size_in_bytes_in_cache.as_u64() + split_size_in_bytes.as_u64()
+            > self.max_num_bytes.as_u64()
         {
             return false;
         }
         true
     }
 
-    pub fn add_split(&mut self, split_size_in_bytes: Byte) {
+    pub fn add_split(&mut self, split_size_in_bytes: ByteSize) {
         self.num_splits_in_cache += 1;
-        self.size_in_bytes_in_cache = Byte::from_bytes(
-            self.size_in_bytes_in_cache.get_bytes() + split_size_in_bytes.get_bytes(),
-        );
+        self.size_in_bytes_in_cache =
+            ByteSize(self.size_in_bytes_in_cache.as_u64() + split_size_in_bytes.as_u64());
     }
 
-    pub fn remove_split(&mut self, split_size_in_bytes: Byte) {
-        self.size_in_bytes_in_cache = Byte::from_bytes(
-            self.size_in_bytes_in_cache.get_bytes() - split_size_in_bytes.get_bytes(),
-        );
+    pub fn remove_split(&mut self, split_size_in_bytes: ByteSize) {
+        self.size_in_bytes_in_cache =
+            ByteSize(self.size_in_bytes_in_cache.as_u64() - split_size_in_bytes.as_u64());
         self.num_splits_in_cache -= 1;
     }
 
-    pub fn max_num_bytes(&self) -> Byte {
+    pub fn max_num_bytes(&self) -> ByteSize {
         self.max_num_bytes
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use byte_unit::Byte;
+    use bytesize::ByteSize;
 
     use crate::split_store::SplitStoreQuota;
 
     #[test]
     fn test_split_store_quota_max_bytes_accepted() {
-        let split_store_quota = SplitStoreQuota::new(3, Byte::from_bytes(100));
-        assert!(split_store_quota.can_fit_split(Byte::from_bytes(100)));
+        let split_store_quota = SplitStoreQuota::new(3, ByteSize(100));
+        assert!(split_store_quota.can_fit_split(ByteSize(100)));
     }
 
     #[test]
     fn test_split_store_quota_exceeding_bytes() {
-        let split_store_quota = SplitStoreQuota::new(3, Byte::from_bytes(100));
-        assert!(!split_store_quota.can_fit_split(Byte::from_bytes(101)));
+        let split_store_quota = SplitStoreQuota::new(3, ByteSize(100));
+        assert!(!split_store_quota.can_fit_split(ByteSize(101)));
     }
 
     #[test]
     fn test_split_store_quota_max_num_files_accepted() {
-        let mut split_store_quota = SplitStoreQuota::new(2, Byte::from_bytes(100));
-        split_store_quota.add_split(Byte::from_bytes(1));
-        assert!(split_store_quota.can_fit_split(Byte::from_bytes(1)));
+        let mut split_store_quota = SplitStoreQuota::new(2, ByteSize(100));
+        split_store_quota.add_split(ByteSize(1));
+        assert!(split_store_quota.can_fit_split(ByteSize(1)));
     }
 
     #[test]
     fn test_split_store_quota_exceeding_max_num_files() {
-        let mut split_store_quota = SplitStoreQuota::new(2, Byte::from_bytes(100));
-        split_store_quota.add_split(Byte::from_bytes(1));
-        split_store_quota.add_split(Byte::from_bytes(1));
-        assert!(!split_store_quota.can_fit_split(Byte::from_bytes(1)));
+        let mut split_store_quota = SplitStoreQuota::new(2, ByteSize(100));
+        split_store_quota.add_split(ByteSize(1));
+        split_store_quota.add_split(ByteSize(1));
+        assert!(!split_store_quota.can_fit_split(ByteSize(1)));
     }
 }
diff --git a/quickwit/quickwit-ingest/Cargo.toml b/quickwit/quickwit-ingest/Cargo.toml
index 5555b016b0e..86f6bb11973 100644
--- a/quickwit/quickwit-ingest/Cargo.toml
+++ b/quickwit/quickwit-ingest/Cargo.toml
@@ -8,8 +8,8 @@ description = "Quickwit is a cost-efficient search engine."
 [dependencies]
 anyhow = { workspace = true }
 async-trait = { workspace = true }
-byte-unit = { workspace = true }
 bytes = { workspace = true }
+bytesize = { workspace = true }
 dyn-clone = { workspace = true }
 flume = { workspace = true }
 futures = { workspace = true }
diff --git a/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs b/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs
index cdf00f0202f..a26c2452d48 100644
--- a/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs
+++ b/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs
@@ -136,13 +136,7 @@ impl FetchTask {
                 .mrecordlog
                 .range(&self.queue_id, self.fetch_range)
             else {
-                warn!(
-                    client_id=%self.client_id,
-                    index_uid=%self.index_uid,
-                    source_id=%self.source_id,
-                    shard_id=%self.shard_id,
-                    "failed to read from record log because it was dropped."
-                );
+                // The queue was dropped.
                 break;
             };
             for (_position, mrecord) in mrecords {
@@ -200,13 +194,21 @@ impl FetchTask {
                 break;
             }
         }
-        debug!(
-            client_id=%self.client_id,
-            index_uid=%self.index_uid,
-            source_id=%self.source_id,
-            shard_id=%self.shard_id,
-            "fetch task completed"
-        );
+        if !has_reached_eof || !self.fetch_range.is_empty() {
+            error!(
+                client_id=%self.client_id,
+                index_uid=%self.index_uid,
+                source_id=%self.source_id,
+                shard_id=%self.shard_id,
+                "fetch stream ended unexpectedly"
+            );
+            let _ = self
+                .fetch_response_tx
+                .send(Err(IngestV2Error::Internal(
+                    "fetch stream ended unexpectedly".to_string(),
+                )))
+                .await;
+        }
         (
             num_records_total,
             self.fetch_range.from_position_exclusive(),
@@ -338,6 +340,12 @@ impl MultiFetchStream {
     }
 }
 
+impl Drop for MultiFetchStream {
+    fn drop(&mut self) {
+        self.reset();
+    }
+}
+
 /// Chooses the ingester to stream records from, preferring "local" ingesters.
 fn select_preferred_and_failover_ingesters(
     self_node_id: &NodeId,
@@ -505,13 +513,13 @@ async fn fault_tolerant_fetch_task(
 }
 
 #[derive(Debug, Clone, Copy)]
-struct FetchRange {
+pub(super) struct FetchRange {
     from_position_exclusive_opt: Option<u64>,
     to_position_inclusive_opt: Option<u64>,
 }
 
 impl FetchRange {
-    fn new(from_position_exclusive: Position, to_position_inclusive: Position) -> Self {
+    pub(super) fn new(from_position_exclusive: Position, to_position_inclusive: Position) -> Self {
         Self {
             from_position_exclusive_opt: from_position_exclusive.as_u64(),
             to_position_inclusive_opt: to_position_inclusive.as_u64(),
@@ -620,6 +628,7 @@ mod tests {
         let state = Arc::new(RwLock::new(IngesterState {
             mrecordlog,
             shards: HashMap::new(),
+            rate_limiters: HashMap::new(),
             replication_streams: HashMap::new(),
             replication_tasks: HashMap::new(),
         }));
@@ -770,6 +779,47 @@ mod tests {
         assert_eq!(last_position, Position::from(4u64));
     }
 
+    #[tokio::test]
+    async fn test_fetch_task_error() {
+        let tempdir = tempfile::tempdir().unwrap();
+        let mrecordlog = MultiRecordLog::open(tempdir.path()).await.unwrap();
+        let client_id = "test-client".to_string();
+        let index_uid = "test-index:0".to_string();
+        let source_id = "test-source".to_string();
+        let open_fetch_stream_request = OpenFetchStreamRequest {
+            client_id: client_id.clone(),
+            index_uid: index_uid.clone(),
+            source_id: source_id.clone(),
+            shard_id: 1,
+            from_position_exclusive: None,
+            to_position_inclusive: None,
+        };
+        let (_new_records_tx, new_records_rx) = watch::channel(());
+        let state = Arc::new(RwLock::new(IngesterState {
+            mrecordlog,
+            shards: HashMap::new(),
+            rate_limiters: HashMap::new(),
+            replication_streams: HashMap::new(),
+            replication_tasks: HashMap::new(),
+        }));
+        let (mut fetch_stream, fetch_task_handle) = FetchTask::spawn(
+            open_fetch_stream_request,
+            state.clone(),
+            new_records_rx,
+            1024,
+        );
+        let ingest_error = timeout(Duration::from_millis(50), fetch_stream.next())
+            .await
+            .unwrap()
+            .unwrap()
+            .unwrap_err();
+        assert!(matches!(ingest_error, IngestV2Error::Internal(_)));
+
+        let (num_records, last_position) = fetch_task_handle.await.unwrap();
+        assert_eq!(num_records, 0);
+        assert_eq!(last_position, Position::Beginning);
+    }
+
     #[tokio::test]
     async fn test_fetch_task_up_to_position() {
         let tempdir = tempfile::tempdir().unwrap();
@@ -788,6 +838,7 @@ mod tests {
         let state = Arc::new(RwLock::new(IngesterState {
             mrecordlog,
             shards: HashMap::new(),
+            rate_limiters: HashMap::new(),
             replication_streams: HashMap::new(),
             replication_tasks: HashMap::new(),
         }));
@@ -854,6 +905,7 @@ mod tests {
         let state = Arc::new(RwLock::new(IngesterState {
             mrecordlog,
             shards: HashMap::new(),
+            rate_limiters: HashMap::new(),
             replication_streams: HashMap::new(),
             replication_tasks: HashMap::new(),
         }));
diff --git a/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs b/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs
index 608e2f626d1..0b372e8ebbc 100644
--- a/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs
+++ b/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs
@@ -26,27 +26,30 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use async_trait::async_trait;
+use bytesize::ByteSize;
 use futures::stream::FuturesUnordered;
 use futures::StreamExt;
-use mrecordlog::error::{CreateQueueError, DeleteQueueError, TruncateError};
+use mrecordlog::error::{CreateQueueError, TruncateError};
 use mrecordlog::MultiRecordLog;
 use quickwit_common::tower::Pool;
 use quickwit_common::ServiceStream;
 use quickwit_proto::ingest::ingester::{
-    AckReplicationMessage, FetchResponseV2, IngesterService, IngesterServiceClient,
-    IngesterServiceStream, OpenFetchStreamRequest, OpenReplicationStreamRequest,
-    OpenReplicationStreamResponse, PersistFailure, PersistFailureReason, PersistRequest,
-    PersistResponse, PersistSuccess, PingRequest, PingResponse, ReplicateRequest,
-    ReplicateSubrequest, SynReplicationMessage, TruncateRequest, TruncateResponse,
+    AckReplicationMessage, CloseShardsRequest, CloseShardsResponse, FetchResponseV2,
+    IngesterService, IngesterServiceClient, IngesterServiceStream, OpenFetchStreamRequest,
+    OpenReplicationStreamRequest, OpenReplicationStreamResponse, PersistFailure,
+    PersistFailureReason, PersistRequest, PersistResponse, PersistSuccess, PingRequest,
+    PingResponse, ReplicateFailureReason, ReplicateRequest, ReplicateSubrequest,
+    SynReplicationMessage, TruncateRequest, TruncateResponse,
 };
 use quickwit_proto::ingest::{CommitTypeV2, IngestV2Error, IngestV2Result, ShardState};
 use quickwit_proto::types::{NodeId, Position, QueueId};
 use tokio::sync::RwLock;
-use tracing::{error, info};
+use tracing::{error, info, warn};
 
 use super::fetch::FetchTask;
 use super::models::{IngesterShard, PrimaryShard};
-use super::mrecord::{is_eof_mrecord, MRecord};
+use super::mrecordlog_utils::{append_eof_record_if_necessary, check_enough_capacity};
+use super::rate_limiter::{RateLimiter, RateLimiterSettings};
 use super::replication::{
     ReplicationStreamTask, ReplicationStreamTaskHandle, ReplicationTask, ReplicationTaskHandle,
     SYN_REPLICATION_STREAM_CAPACITY,
@@ -54,7 +57,7 @@ use super::replication::{
 use super::IngesterPool;
 use crate::ingest_v2::models::SoloShard;
 use crate::metrics::INGEST_METRICS;
-use crate::{FollowerId, LeaderId};
+use crate::{estimate_size, FollowerId, LeaderId, MRecord};
 
 /// Duration after which persist requests time out with
 /// [`quickwit_proto::ingest::IngestV2Error::Timeout`].
@@ -69,6 +72,9 @@ pub struct Ingester {
     self_node_id: NodeId,
     ingester_pool: IngesterPool,
     state: Arc<RwLock<IngesterState>>,
+    disk_capacity: ByteSize,
+    memory_capacity: ByteSize,
+    rate_limiter_settings: RateLimiterSettings,
     replication_factor: usize,
 }
 
@@ -83,6 +89,7 @@ impl fmt::Debug for Ingester {
 pub(super) struct IngesterState {
     pub mrecordlog: MultiRecordLog,
     pub shards: HashMap<QueueId, IngesterShard>,
+    pub rate_limiters: HashMap<QueueId, RateLimiter>,
     // Replication stream opened with followers.
     pub replication_streams: HashMap<FollowerId, ReplicationStreamTaskHandle>,
     // Replication tasks running for each replication stream opened with leaders.
@@ -94,6 +101,9 @@ impl Ingester {
         self_node_id: NodeId,
         ingester_pool: Pool<NodeId, IngesterServiceClient>,
         wal_dir_path: &Path,
+        disk_capacity: ByteSize,
+        memory_capacity: ByteSize,
+        rate_limiter_settings: RateLimiterSettings,
         replication_factor: usize,
     ) -> IngestV2Result<Self> {
         let mrecordlog = MultiRecordLog::open_with_prefs(
@@ -106,6 +116,7 @@ impl Ingester {
         let inner = IngesterState {
             mrecordlog,
             shards: HashMap::new(),
+            rate_limiters: HashMap::new(),
             replication_streams: HashMap::new(),
             replication_tasks: HashMap::new(),
         };
@@ -113,6 +124,9 @@ impl Ingester {
             self_node_id,
             ingester_pool,
             state: Arc::new(RwLock::new(inner)),
+            disk_capacity,
+            memory_capacity,
+            rate_limiter_settings,
             replication_factor,
         };
         info!(
@@ -134,12 +148,17 @@ impl Ingester {
             .map(|queue_id| queue_id.to_string())
             .collect();
 
+        if queue_ids.is_empty() {
+            return Ok(());
+        }
+        info!("closing {} shard(s)", queue_ids.len());
+
         for queue_id in queue_ids {
             append_eof_record_if_necessary(&mut state_guard.mrecordlog, &queue_id).await;
 
             let solo_shard = SoloShard::new(ShardState::Closed, Position::Eof);
             let shard = IngesterShard::Solo(solo_shard);
-            state_guard.shards.insert(queue_id, shard);
+            state_guard.shards.insert(queue_id.clone(), shard);
         }
         Ok(())
     }
@@ -165,13 +184,17 @@ impl Ingester {
                 });
             }
         };
+        let rate_limiter = RateLimiter::from_settings(self.rate_limiter_settings);
+        state.rate_limiters.insert(queue_id.clone(), rate_limiter);
+
         let shard = if let Some(follower_id) = follower_id_opt {
             self.init_replication_stream(state, leader_id, follower_id)
                 .await?;
             let primary_shard = PrimaryShard::new(follower_id.clone());
             IngesterShard::Primary(primary_shard)
         } else {
-            IngesterShard::Solo(SoloShard::default())
+            let solo_shard = SoloShard::new(ShardState::Open, Position::Beginning);
+            IngesterShard::Solo(solo_shard)
         };
         let entry = state.shards.entry(queue_id.clone());
         Ok(entry.or_insert(shard))
@@ -238,8 +261,6 @@ impl IngesterService for Ingester {
                 self.self_node_id, persist_request.leader_id,
             )));
         }
-        let mut state_guard = self.state.write().await;
-
         let mut persist_successes = Vec::with_capacity(persist_request.subrequests.len());
         let mut persist_failures = Vec::new();
         let mut replicate_subrequests: HashMap<NodeId, Vec<ReplicateSubrequest>> = HashMap::new();
@@ -248,6 +269,8 @@ impl IngesterService for Ingester {
         let force_commit = commit_type == CommitTypeV2::Force;
         let leader_id: NodeId = persist_request.leader_id.into();
 
+        let mut state_guard = self.state.write().await;
+
         for subrequest in persist_request.subrequests {
             let queue_id = subrequest.queue_id();
             let follower_id_opt: Option<NodeId> = subrequest.follower_id.map(Into::into);
@@ -276,10 +299,64 @@ impl IngesterService for Ingester {
                 persist_failures.push(persist_failure);
                 continue;
             }
-            let doc_batch = subrequest
-                .doc_batch
-                .expect("router should not send empty persist subrequests");
+            let doc_batch = match subrequest.doc_batch {
+                Some(doc_batch) if !doc_batch.is_empty() => doc_batch,
+                _ => {
+                    warn!("received empty persist request");
+
+                    let persist_success = PersistSuccess {
+                        subrequest_id: subrequest.subrequest_id,
+                        index_uid: subrequest.index_uid,
+                        source_id: subrequest.source_id,
+                        shard_id: subrequest.shard_id,
+                        replication_position_inclusive: Some(
+                            shard.replication_position_inclusive(),
+                        ),
+                    };
+                    persist_successes.push(persist_success);
+                    continue;
+                }
+            };
+            let requested_capacity = estimate_size(&doc_batch);
+
+            if let Err(error) = check_enough_capacity(
+                &state_guard.mrecordlog,
+                self.disk_capacity,
+                self.memory_capacity,
+                requested_capacity,
+            ) {
+                warn!(
+                    "failed to persist records to ingester `{}`: {error}",
+                    self.self_node_id
+                );
+                let persist_failure = PersistFailure {
+                    subrequest_id: subrequest.subrequest_id,
+                    index_uid: subrequest.index_uid,
+                    source_id: subrequest.source_id,
+                    shard_id: subrequest.shard_id,
+                    reason: PersistFailureReason::ResourceExhausted as i32,
+                };
+                persist_failures.push(persist_failure);
+                continue;
+            }
+            let rate_limiter = state_guard
+                .rate_limiters
+                .get_mut(&queue_id)
+                .expect("rate limiter should be initialized");
 
+            if !rate_limiter.acquire(requested_capacity) {
+                warn!("failed to persist records to shard `{queue_id}`: rate limited");
+
+                let persist_failure = PersistFailure {
+                    subrequest_id: subrequest.subrequest_id,
+                    index_uid: subrequest.index_uid,
+                    source_id: subrequest.source_id,
+                    shard_id: subrequest.shard_id,
+                    reason: PersistFailureReason::RateLimited as i32,
+                };
+                persist_failures.push(persist_failure);
+                continue;
+            }
             let current_position_inclusive: Position = if force_commit {
                 let encoded_mrecords = doc_batch
                     .docs()
@@ -389,17 +466,31 @@ impl IngesterService for Ingester {
                 };
                 persist_successes.push(persist_success);
             }
-        }
-        let _state_guard = self.state.write().await;
-
-        for persist_success in &persist_successes {
-            let _queue_id = persist_success.queue_id();
+            for replicate_failure in replicate_response.failures {
+                // TODO: If the replica shard is closed, close the primary shard if it is not
+                // already.
+                let persist_failure_reason = match replicate_failure.reason() {
+                    ReplicateFailureReason::Unspecified => PersistFailureReason::Unspecified,
+                    ReplicateFailureReason::ShardClosed => PersistFailureReason::ShardClosed,
+                    ReplicateFailureReason::ResourceExhausted => {
+                        PersistFailureReason::ResourceExhausted
+                    }
+                };
+                let persist_failure = PersistFailure {
+                    subrequest_id: replicate_failure.subrequest_id,
+                    index_uid: replicate_failure.index_uid,
+                    source_id: replicate_failure.source_id,
+                    shard_id: replicate_failure.shard_id,
+                    reason: persist_failure_reason as i32,
+                };
+                persist_failures.push(persist_failure);
+            }
         }
         let leader_id = self.self_node_id.to_string();
         let persist_response = PersistResponse {
             leader_id,
             successes: persist_successes,
-            failures: Vec::new(), // TODO
+            failures: persist_failures,
         };
         Ok(persist_response)
     }
@@ -444,6 +535,8 @@ impl IngesterService for Ingester {
             self.state.clone(),
             syn_replication_stream,
             ack_replication_stream_tx,
+            self.disk_capacity,
+            self.memory_capacity,
         );
         entry.insert(replication_task_handle);
         Ok(ack_replication_stream)
@@ -505,57 +598,54 @@ impl IngesterService for Ingester {
 
         for subrequest in truncate_request.subrequests {
             let queue_id = subrequest.queue_id();
-            let to_position_inclusive = subrequest.to_position_inclusive();
 
-            if let Some(to_offset_inclusive) = to_position_inclusive.as_u64() {
+            let truncate_position_opt = match subrequest.to_position_inclusive() {
+                Position::Beginning => None,
+                Position::Offset(offset) => offset.as_u64(),
+                Position::Eof => state_guard
+                    .mrecordlog
+                    .last_position(&queue_id)
+                    .ok()
+                    .flatten(),
+            };
+            if let Some(truncate_position) = truncate_position_opt {
                 match state_guard
                     .mrecordlog
-                    .truncate(&queue_id, to_offset_inclusive)
+                    .truncate(&queue_id, truncate_position)
                     .await
                 {
                     Ok(_) | Err(TruncateError::MissingQueue(_)) => {}
                     Err(error) => {
-                        error!("failed to truncate queue `{}`: {}", queue_id, error);
+                        error!("failed to truncate queue `{queue_id}`: {error}");
                     }
                 }
-            } else if to_position_inclusive == Position::Eof {
-                match state_guard.mrecordlog.delete_queue(&queue_id).await {
-                    Ok(_) | Err(DeleteQueueError::MissingQueue(_)) => {}
-                    Err(error) => {
-                        error!("failed to delete queue `{}`: {}", queue_id, error);
-                    }
-                }
-                state_guard.shards.remove(&queue_id);
-            };
+            }
         }
         let truncate_response = TruncateResponse {};
         Ok(truncate_response)
     }
-}
 
-/// Appends an EOF record to the queue if the it is empty or the last record is not an EOF
-/// record.
-///
-/// # Panics
-///
-/// Panics if the queue does not exist.
-async fn append_eof_record_if_necessary(mrecordlog: &mut MultiRecordLog, queue_id: &QueueId) {
-    let mut should_append_eof_record = true;
-
-    if let Some(current_position) = mrecordlog.current_position(queue_id).expect("TODO") {
-        let mrecords = mrecordlog
-            .range(queue_id, current_position..)
-            .expect("TODO");
-
-        if let Some((_, last_mecord)) = mrecords.last() {
-            should_append_eof_record = !is_eof_mrecord(&last_mecord);
+    async fn close_shards(
+        &mut self,
+        close_shards_request: CloseShardsRequest,
+    ) -> IngestV2Result<CloseShardsResponse> {
+        let mut state_guard = self.state.write().await;
+        for close_shard in close_shards_request.closed_shards {
+            for queue_id in close_shard.queue_ids() {
+                if !state_guard.mrecordlog.queue_exists(&queue_id) {
+                    continue;
+                }
+                append_eof_record_if_necessary(&mut state_guard.mrecordlog, &queue_id).await;
+                let shard = state_guard
+                    .shards
+                    .get_mut(&queue_id)
+                    .expect("shard must exist");
+                // Notify fetch task.
+                shard.notify_new_records();
+                shard.close();
+            }
         }
-    }
-    if should_append_eof_record {
-        mrecordlog
-            .append_record(queue_id, None, MRecord::Eof.encode())
-            .await
-            .expect("TODO");
+        Ok(CloseShardsResponse {})
     }
 }
 
@@ -564,15 +654,18 @@ mod tests {
     use std::net::SocketAddr;
 
     use bytes::Bytes;
+    use quickwit_common::tower::ConstantRate;
     use quickwit_proto::ingest::ingester::{
         IngesterServiceGrpcServer, IngesterServiceGrpcServerAdapter, PersistSubrequest,
         TruncateSubrequest,
     };
-    use quickwit_proto::ingest::DocBatchV2;
+    use quickwit_proto::ingest::{ClosedShards, DocBatchV2};
     use quickwit_proto::types::queue_id;
     use tonic::transport::{Endpoint, Server};
 
     use super::*;
+    use crate::ingest_v2::fetch::FetchRange;
+    use crate::ingest_v2::mrecord::is_eof_mrecord;
     use crate::ingest_v2::test_utils::{IngesterShardTestExt, MultiRecordLogTestExt};
 
     #[tokio::test]
@@ -581,11 +674,17 @@ mod tests {
         let self_node_id: NodeId = "test-ingester-0".into();
         let ingester_pool = IngesterPool::default();
         let wal_dir_path = tempdir.path();
+        let disk_capacity = ByteSize::mb(256);
+        let memory_capacity = ByteSize::mb(1);
+        let rate_limiter_settings = RateLimiterSettings::default();
         let replication_factor = 2;
         let mut ingester = Ingester::try_new(
             self_node_id.clone(),
             ingester_pool,
             wal_dir_path,
+            disk_capacity,
+            memory_capacity,
+            rate_limiter_settings,
             replication_factor,
         )
         .await
@@ -682,11 +781,17 @@ mod tests {
         let self_node_id: NodeId = "test-ingester-0".into();
         let ingester_pool = IngesterPool::default();
         let wal_dir_path = tempdir.path();
+        let disk_capacity = ByteSize::mb(256);
+        let memory_capacity = ByteSize::mb(1);
+        let rate_limiter_settings = RateLimiterSettings::default();
         let replication_factor = 1;
         let mut ingester = Ingester::try_new(
             self_node_id.clone(),
             ingester_pool,
             wal_dir_path,
+            disk_capacity,
+            memory_capacity,
+            rate_limiter_settings,
             replication_factor,
         )
         .await
@@ -714,7 +819,30 @@ mod tests {
                 },
             ],
         };
-        ingester.persist(persist_request).await.unwrap();
+        let persist_response = ingester.persist(persist_request).await.unwrap();
+        assert_eq!(persist_response.leader_id, "test-ingester-0");
+        assert_eq!(persist_response.successes.len(), 2);
+        assert_eq!(persist_response.failures.len(), 0);
+
+        let persist_success_0 = &persist_response.successes[0];
+        assert_eq!(persist_success_0.subrequest_id, 0);
+        assert_eq!(persist_success_0.index_uid, "test-index:0");
+        assert_eq!(persist_success_0.source_id, "test-source");
+        assert_eq!(persist_success_0.shard_id, 1);
+        assert_eq!(
+            persist_success_0.replication_position_inclusive,
+            Some(Position::from(1u64))
+        );
+
+        let persist_success_1 = &persist_response.successes[1];
+        assert_eq!(persist_success_1.subrequest_id, 1);
+        assert_eq!(persist_success_1.index_uid, "test-index:1");
+        assert_eq!(persist_success_1.source_id, "test-source");
+        assert_eq!(persist_success_1.shard_id, 1);
+        assert_eq!(
+            persist_success_1.replication_position_inclusive,
+            Some(Position::from(2u64))
+        );
 
         let state_guard = ingester.state.read().await;
         assert_eq!(state_guard.shards.len(), 2);
@@ -754,11 +882,17 @@ mod tests {
         let self_node_id: NodeId = "test-follower".into();
         let ingester_pool = IngesterPool::default();
         let wal_dir_path = tempdir.path();
+        let disk_capacity = ByteSize::mb(256);
+        let memory_capacity = ByteSize::mb(1);
+        let rate_limiter_settings = RateLimiterSettings::default();
         let replication_factor = 1;
         let mut ingester = Ingester::try_new(
             self_node_id.clone(),
             ingester_pool,
             wal_dir_path,
+            disk_capacity,
+            memory_capacity,
+            rate_limiter_settings,
             replication_factor,
         )
         .await
@@ -795,11 +929,19 @@ mod tests {
         let leader_id: NodeId = "test-leader".into();
         let ingester_pool = IngesterPool::default();
         let wal_dir_path = tempdir.path();
+
+        let disk_capacity = ByteSize::mb(256);
+        let memory_capacity = ByteSize::mb(1);
+        let rate_limiter_settings = RateLimiterSettings::default();
         let replication_factor = 2;
+
         let mut leader = Ingester::try_new(
             leader_id.clone(),
             ingester_pool.clone(),
             wal_dir_path,
+            disk_capacity,
+            memory_capacity,
+            rate_limiter_settings,
             replication_factor,
         )
         .await
@@ -808,11 +950,14 @@ mod tests {
         let tempdir = tempfile::tempdir().unwrap();
         let follower_id: NodeId = "test-follower".into();
         let wal_dir_path = tempdir.path();
-        let replication_factor = 2;
+
         let follower = Ingester::try_new(
             follower_id.clone(),
             ingester_pool.clone(),
             wal_dir_path,
+            disk_capacity,
+            memory_capacity,
+            rate_limiter_settings,
             replication_factor,
         )
         .await
@@ -850,6 +995,26 @@ mod tests {
         assert_eq!(persist_response.successes.len(), 2);
         assert_eq!(persist_response.failures.len(), 0);
 
+        let persist_success_0 = &persist_response.successes[0];
+        assert_eq!(persist_success_0.subrequest_id, 0);
+        assert_eq!(persist_success_0.index_uid, "test-index:0");
+        assert_eq!(persist_success_0.source_id, "test-source");
+        assert_eq!(persist_success_0.shard_id, 1);
+        assert_eq!(
+            persist_success_0.replication_position_inclusive,
+            Some(Position::from(1u64))
+        );
+
+        let persist_success_1 = &persist_response.successes[1];
+        assert_eq!(persist_success_1.subrequest_id, 1);
+        assert_eq!(persist_success_1.index_uid, "test-index:1");
+        assert_eq!(persist_success_1.source_id, "test-source");
+        assert_eq!(persist_success_1.shard_id, 1);
+        assert_eq!(
+            persist_success_1.replication_position_inclusive,
+            Some(Position::from(2u64))
+        );
+
         let leader_state_guard = leader.state.read().await;
         assert_eq!(leader_state_guard.shards.len(), 2);
 
@@ -917,11 +1082,17 @@ mod tests {
         let leader_id: NodeId = "test-leader".into();
         let ingester_pool = IngesterPool::default();
         let wal_dir_path = tempdir.path();
+        let disk_capacity = ByteSize::mb(256);
+        let memory_capacity = ByteSize::mb(1);
+        let rate_limiter_settings = RateLimiterSettings::default();
         let replication_factor = 2;
         let mut leader = Ingester::try_new(
             leader_id.clone(),
             ingester_pool.clone(),
             wal_dir_path,
+            disk_capacity,
+            memory_capacity,
+            rate_limiter_settings,
             replication_factor,
         )
         .await
@@ -944,11 +1115,17 @@ mod tests {
         let tempdir = tempfile::tempdir().unwrap();
         let follower_id: NodeId = "test-follower".into();
         let wal_dir_path = tempdir.path();
+        let disk_capacity = ByteSize::mb(256);
+        let memory_capacity = ByteSize::mb(1);
+        let rate_limiter_settings = RateLimiterSettings::default();
         let replication_factor = 2;
         let follower = Ingester::try_new(
             follower_id.clone(),
             ingester_pool.clone(),
             wal_dir_path,
+            disk_capacity,
+            memory_capacity,
+            rate_limiter_settings,
             replication_factor,
         )
         .await
@@ -1002,6 +1179,26 @@ mod tests {
         assert_eq!(persist_response.successes.len(), 2);
         assert_eq!(persist_response.failures.len(), 0);
 
+        let persist_success_0 = &persist_response.successes[0];
+        assert_eq!(persist_success_0.subrequest_id, 0);
+        assert_eq!(persist_success_0.index_uid, "test-index:0");
+        assert_eq!(persist_success_0.source_id, "test-source");
+        assert_eq!(persist_success_0.shard_id, 1);
+        assert_eq!(
+            persist_success_0.replication_position_inclusive,
+            Some(Position::from(0u64))
+        );
+
+        let persist_success_1 = &persist_response.successes[1];
+        assert_eq!(persist_success_1.subrequest_id, 1);
+        assert_eq!(persist_success_1.index_uid, "test-index:1");
+        assert_eq!(persist_success_1.source_id, "test-source");
+        assert_eq!(persist_success_1.shard_id, 1);
+        assert_eq!(
+            persist_success_1.replication_position_inclusive,
+            Some(Position::from(1u64))
+        );
+
         let leader_state_guard = leader.state.read().await;
         assert_eq!(leader_state_guard.shards.len(), 2);
 
@@ -1055,17 +1252,218 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_ingester_persist_shard_closed() {
+        let tempdir = tempfile::tempdir().unwrap();
+        let self_node_id: NodeId = "test-ingester-0".into();
+        let ingester_pool = IngesterPool::default();
+        let wal_dir_path = tempdir.path();
+        let disk_capacity = ByteSize::mib(256);
+        let memory_capacity = ByteSize::mib(1);
+        let rate_limiter_settings = RateLimiterSettings::default();
+        let replication_factor = 1;
+        let mut ingester = Ingester::try_new(
+            self_node_id.clone(),
+            ingester_pool,
+            wal_dir_path,
+            disk_capacity,
+            memory_capacity,
+            rate_limiter_settings,
+            replication_factor,
+        )
+        .await
+        .unwrap();
+
+        let queue_id_01 = queue_id("test-index:0", "test-source", 1);
+
+        let solo_shard = SoloShard::new(ShardState::Closed, Position::Beginning);
+        let shard = IngesterShard::Solo(solo_shard);
+
+        ingester
+            .state
+            .write()
+            .await
+            .shards
+            .insert(queue_id_01.clone(), shard);
+
+        let persist_request = PersistRequest {
+            leader_id: self_node_id.to_string(),
+            commit_type: CommitTypeV2::Auto as i32,
+            subrequests: vec![PersistSubrequest {
+                subrequest_id: 0,
+                index_uid: "test-index:0".to_string(),
+                source_id: "test-source".to_string(),
+                shard_id: 1,
+                follower_id: None,
+                doc_batch: Some(DocBatchV2::for_test(["test-doc-010"])),
+            }],
+        };
+        let persist_response = ingester.persist(persist_request).await.unwrap();
+        assert_eq!(persist_response.leader_id, "test-ingester-0");
+        assert_eq!(persist_response.successes.len(), 0);
+        assert_eq!(persist_response.failures.len(), 1);
+
+        let persist_failure = &persist_response.failures[0];
+        assert_eq!(persist_failure.subrequest_id, 0);
+        assert_eq!(persist_failure.index_uid, "test-index:0");
+        assert_eq!(persist_failure.source_id, "test-source");
+        assert_eq!(persist_failure.shard_id, 1);
+        assert_eq!(persist_failure.reason(), PersistFailureReason::ShardClosed);
+
+        let state_guard = ingester.state.read().await;
+        assert_eq!(state_guard.shards.len(), 1);
+
+        let solo_shard_01 = state_guard.shards.get(&queue_id_01).unwrap();
+        solo_shard_01.assert_is_solo();
+        solo_shard_01.assert_is_closed();
+        solo_shard_01.assert_replication_position(Position::Beginning);
+    }
+
+    #[tokio::test]
+    async fn test_ingester_persist_rate_limited() {
+        let tempdir = tempfile::tempdir().unwrap();
+        let self_node_id: NodeId = "test-ingester-0".into();
+        let ingester_pool = IngesterPool::default();
+        let wal_dir_path = tempdir.path();
+        let disk_capacity = ByteSize::mib(256);
+        let memory_capacity = ByteSize::mib(1);
+        let rate_limiter_settings = RateLimiterSettings {
+            burst_limit: ByteSize(0),
+            rate_limit: ConstantRate::bytes_per_sec(ByteSize(0)),
+            refill_period: Duration::from_millis(100),
+        };
+        let replication_factor = 1;
+        let mut ingester = Ingester::try_new(
+            self_node_id.clone(),
+            ingester_pool,
+            wal_dir_path,
+            disk_capacity,
+            memory_capacity,
+            rate_limiter_settings,
+            replication_factor,
+        )
+        .await
+        .unwrap();
+
+        let persist_request = PersistRequest {
+            leader_id: self_node_id.to_string(),
+            commit_type: CommitTypeV2::Auto as i32,
+            subrequests: vec![PersistSubrequest {
+                subrequest_id: 0,
+                index_uid: "test-index:0".to_string(),
+                source_id: "test-source".to_string(),
+                shard_id: 1,
+                follower_id: None,
+                doc_batch: Some(DocBatchV2::for_test(["test-doc-010"])),
+            }],
+        };
+        let persist_response = ingester.persist(persist_request).await.unwrap();
+        assert_eq!(persist_response.leader_id, "test-ingester-0");
+        assert_eq!(persist_response.successes.len(), 0);
+        assert_eq!(persist_response.failures.len(), 1);
+
+        let persist_failure = &persist_response.failures[0];
+        assert_eq!(persist_failure.subrequest_id, 0);
+        assert_eq!(persist_failure.index_uid, "test-index:0");
+        assert_eq!(persist_failure.source_id, "test-source");
+        assert_eq!(persist_failure.shard_id, 1);
+        assert_eq!(persist_failure.reason(), PersistFailureReason::RateLimited);
+
+        let state_guard = ingester.state.read().await;
+        assert_eq!(state_guard.shards.len(), 1);
+
+        let queue_id_01 = queue_id("test-index:0", "test-source", 1);
+
+        let solo_shard_01 = state_guard.shards.get(&queue_id_01).unwrap();
+        solo_shard_01.assert_is_solo();
+        solo_shard_01.assert_is_open();
+        solo_shard_01.assert_replication_position(Position::Beginning);
+
+        state_guard
+            .mrecordlog
+            .assert_records_eq(&queue_id_01, .., &[]);
+    }
+
+    #[tokio::test]
+    async fn test_ingester_persist_resource_exhausted() {
+        let tempdir = tempfile::tempdir().unwrap();
+        let self_node_id: NodeId = "test-ingester-0".into();
+        let ingester_pool = IngesterPool::default();
+        let wal_dir_path = tempdir.path();
+        let disk_capacity = ByteSize(0);
+        let memory_capacity = ByteSize(0);
+        let rate_limiter_settings = RateLimiterSettings::default();
+        let replication_factor = 1;
+        let mut ingester = Ingester::try_new(
+            self_node_id.clone(),
+            ingester_pool,
+            wal_dir_path,
+            disk_capacity,
+            memory_capacity,
+            rate_limiter_settings,
+            replication_factor,
+        )
+        .await
+        .unwrap();
+
+        let persist_request = PersistRequest {
+            leader_id: self_node_id.to_string(),
+            commit_type: CommitTypeV2::Auto as i32,
+            subrequests: vec![PersistSubrequest {
+                subrequest_id: 0,
+                index_uid: "test-index:0".to_string(),
+                source_id: "test-source".to_string(),
+                shard_id: 1,
+                follower_id: None,
+                doc_batch: Some(DocBatchV2::for_test(["test-doc-010"])),
+            }],
+        };
+        let persist_response = ingester.persist(persist_request).await.unwrap();
+        assert_eq!(persist_response.leader_id, "test-ingester-0");
+        assert_eq!(persist_response.successes.len(), 0);
+        assert_eq!(persist_response.failures.len(), 1);
+
+        let persist_failure = &persist_response.failures[0];
+        assert_eq!(persist_failure.subrequest_id, 0);
+        assert_eq!(persist_failure.index_uid, "test-index:0");
+        assert_eq!(persist_failure.source_id, "test-source");
+        assert_eq!(persist_failure.shard_id, 1);
+        assert_eq!(
+            persist_failure.reason(),
+            PersistFailureReason::ResourceExhausted
+        );
+
+        let state_guard = ingester.state.read().await;
+        assert_eq!(state_guard.shards.len(), 1);
+
+        let queue_id_01 = queue_id("test-index:0", "test-source", 1);
+        let solo_shard_01 = state_guard.shards.get(&queue_id_01).unwrap();
+        solo_shard_01.assert_is_solo();
+        solo_shard_01.assert_is_open();
+        solo_shard_01.assert_replication_position(Position::Beginning);
+
+        state_guard
+            .mrecordlog
+            .assert_records_eq(&queue_id_01, .., &[]);
+    }
+
     #[tokio::test]
     async fn test_ingester_open_fetch_stream() {
         let tempdir = tempfile::tempdir().unwrap();
         let self_node_id: NodeId = "test-ingester-0".into();
         let ingester_pool = IngesterPool::default();
         let wal_dir_path = tempdir.path();
+        let disk_capacity = ByteSize::mb(256);
+        let memory_capacity = ByteSize::mb(1);
+        let rate_limiter_settings = RateLimiterSettings::default();
         let replication_factor = 1;
         let mut ingester = Ingester::try_new(
             self_node_id.clone(),
             ingester_pool,
             wal_dir_path,
+            disk_capacity,
+            memory_capacity,
+            rate_limiter_settings,
             replication_factor,
         )
         .await
@@ -1159,11 +1557,17 @@ mod tests {
         let self_node_id: NodeId = "test-ingester-0".into();
         let ingester_pool = IngesterPool::default();
         let wal_dir_path = tempdir.path();
+        let disk_capacity = ByteSize::mb(256);
+        let memory_capacity = ByteSize::mb(1);
+        let rate_limiter_settings = RateLimiterSettings::default();
         let replication_factor = 1;
         let mut ingester = Ingester::try_new(
             self_node_id.clone(),
             ingester_pool,
             wal_dir_path,
+            disk_capacity,
+            memory_capacity,
+            rate_limiter_settings,
             replication_factor,
         )
         .await
@@ -1230,12 +1634,138 @@ mod tests {
         ingester.truncate(truncate_request).await.unwrap();
 
         let state_guard = ingester.state.read().await;
-        assert_eq!(state_guard.shards.len(), 1);
-        assert!(state_guard.shards.contains_key(&queue_id_01));
+        assert_eq!(state_guard.shards.len(), 2);
 
+        assert!(state_guard.shards.contains_key(&queue_id_01));
         state_guard
             .mrecordlog
             .assert_records_eq(&queue_id_01, .., &[(1, "\0\0test-doc-011")]);
-        assert!(!state_guard.shards.contains_key(&queue_id_02));
+
+        assert!(state_guard.shards.contains_key(&queue_id_02));
+        state_guard
+            .mrecordlog
+            .assert_records_eq(&queue_id_02, .., &[]);
+    }
+
+    #[tokio::test]
+    async fn test_ingester_close_shards() {
+        let tempdir = tempfile::tempdir().unwrap();
+        let self_node_id: NodeId = "test-ingester-0".into();
+        let ingester_pool = IngesterPool::default();
+        let wal_dir_path = tempdir.path();
+        let disk_capacity = ByteSize::mb(256);
+        let memory_capacity = ByteSize::mb(1);
+        let rate_limiter_settings = RateLimiterSettings::default();
+        let replication_factor = 1;
+        let mut ingester = Ingester::try_new(
+            self_node_id.clone(),
+            ingester_pool,
+            wal_dir_path,
+            disk_capacity,
+            memory_capacity,
+            rate_limiter_settings,
+            replication_factor,
+        )
+        .await
+        .unwrap();
+
+        let queue_id_01 = queue_id("test-index:0", "test-source:0", 1);
+        let queue_id_02 = queue_id("test-index:0", "test-source:0", 2);
+        let queue_id_03 = queue_id("test-index:1", "test-source:1", 3);
+
+        let mut state_guard = ingester.state.write().await;
+        for queue_id in &[&queue_id_01, &queue_id_02, &queue_id_03] {
+            ingester
+                .create_shard(&mut state_guard, queue_id, &self_node_id, None)
+                .await
+                .unwrap();
+            let records = [
+                MRecord::new_doc("test-doc-010").encode(),
+                MRecord::new_doc("test-doc-011").encode(),
+            ]
+            .into_iter();
+            state_guard
+                .mrecordlog
+                .append_records(&queue_id_01, None, records)
+                .await
+                .unwrap();
+        }
+
+        drop(state_guard);
+
+        let client_id = "test-client".to_string();
+        let open_fetch_stream_request = OpenFetchStreamRequest {
+            client_id: client_id.clone(),
+            index_uid: "test-index:0".to_string(),
+            source_id: "test-source:0".to_string(),
+            shard_id: 1,
+            from_position_exclusive: None,
+            to_position_inclusive: None,
+        };
+
+        let mut fetch_stream = ingester
+            .open_fetch_stream(open_fetch_stream_request)
+            .await
+            .unwrap();
+        let fetch_response = fetch_stream.next().await.unwrap().unwrap();
+        assert_eq!(
+            fetch_response.from_position_exclusive(),
+            Position::Beginning
+        );
+
+        let close_shard_1 = ClosedShards {
+            index_uid: "test-index:0".to_string(),
+            source_id: "test-source:0".to_string(),
+            shard_ids: vec![1, 2],
+        };
+        let close_shard_2 = ClosedShards {
+            index_uid: "test-index:1".to_string(),
+            source_id: "test-source:1".to_string(),
+            shard_ids: vec![3],
+        };
+        let close_shard_with_no_queue = ClosedShards {
+            index_uid: "test-index:2".to_string(),
+            source_id: "test-source:2".to_string(),
+            shard_ids: vec![4],
+        };
+        let closed_shards = vec![
+            close_shard_1.clone(),
+            close_shard_2.clone(),
+            close_shard_with_no_queue,
+        ];
+        let close_shards_request = CloseShardsRequest {
+            closed_shards: closed_shards.clone(),
+        };
+        ingester.close_shards(close_shards_request).await.unwrap();
+
+        // Check that shards are closed and EOF records are appended.
+        let state_guard = ingester.state.read().await;
+        for shard in state_guard.shards.values() {
+            shard.assert_is_closed();
+        }
+        for closed_shards in [&close_shard_1, &close_shard_2] {
+            for queue_id in closed_shards.queue_ids() {
+                let last_position = state_guard
+                    .mrecordlog
+                    .range(
+                        &queue_id,
+                        FetchRange::new(Position::Beginning, Position::Beginning),
+                    )
+                    .unwrap()
+                    .last()
+                    .unwrap();
+                assert!(is_eof_mrecord(&last_position.1));
+            }
+        }
+
+        // Check that fetch task is notified.
+        // Note: fetch stream should not block if the close shard call notified the fetch task.
+        let fetch_response =
+            tokio::time::timeout(std::time::Duration::from_millis(50), fetch_stream.next())
+                .await
+                .unwrap()
+                .unwrap()
+                .unwrap();
+        assert_eq!(fetch_response.to_position_inclusive(), Position::Eof);
     }
 }
diff --git a/quickwit/quickwit-ingest/src/ingest_v2/mod.rs b/quickwit/quickwit-ingest/src/ingest_v2/mod.rs
index 16e4317ab70..345de74ca08 100644
--- a/quickwit/quickwit-ingest/src/ingest_v2/mod.rs
+++ b/quickwit/quickwit-ingest/src/ingest_v2/mod.rs
@@ -21,6 +21,8 @@ mod fetch;
 mod ingester;
 mod models;
 mod mrecord;
+mod mrecordlog_utils;
+mod rate_limiter;
 mod replication;
 mod router;
 mod shard_table;
@@ -28,13 +30,17 @@ mod shard_table;
 mod test_utils;
 mod workbench;
 
+use bytesize::ByteSize;
 use quickwit_common::tower::Pool;
 use quickwit_proto::ingest::ingester::IngesterServiceClient;
+use quickwit_proto::ingest::DocBatchV2;
 use quickwit_proto::types::NodeId;
 
 pub use self::fetch::{FetchStreamError, MultiFetchStream};
 pub use self::ingester::Ingester;
+use self::mrecord::MRECORD_HEADER_LEN;
 pub use self::mrecord::{decoded_mrecords, MRecord};
+pub use self::rate_limiter::RateLimiterSettings;
 pub use self::router::IngestRouter;
 
 pub type IngesterPool = Pool<NodeId, IngesterServiceClient>;
@@ -45,3 +51,28 @@ pub type ClientId = String;
 pub type LeaderId = NodeId;
 
 pub type FollowerId = NodeId;
+
+pub(super) fn estimate_size(doc_batch: &DocBatchV2) -> ByteSize {
+    let estimate = doc_batch.num_bytes() + doc_batch.num_docs() * MRECORD_HEADER_LEN;
+    ByteSize(estimate as u64)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_estimate_size() {
+        let doc_batch = DocBatchV2 {
+            doc_buffer: Vec::new().into(),
+            doc_lengths: Vec::new(),
+        };
+        assert_eq!(estimate_size(&doc_batch), ByteSize(0));
+
+        let doc_batch = DocBatchV2 {
+            doc_buffer: vec![0u8; 100].into(),
+            doc_lengths: vec![10, 20, 30],
+        };
+        assert_eq!(estimate_size(&doc_batch), ByteSize(106));
+    }
+}
diff --git a/quickwit/quickwit-ingest/src/ingest_v2/models.rs b/quickwit/quickwit-ingest/src/ingest_v2/models.rs
index 577e28ebade..9cf73a1acb4 100644
--- a/quickwit/quickwit-ingest/src/ingest_v2/models.rs
+++ b/quickwit/quickwit-ingest/src/ingest_v2/models.rs
@@ -105,28 +105,19 @@ impl fmt::Debug for SoloShard {
     }
 }
 
-impl Default for SoloShard {
-    fn default() -> Self {
-        let (new_records_tx, new_records_rx) = watch::channel(());
-        Self {
-            shard_state: ShardState::Open,
-            replication_position_inclusive: Position::Beginning,
-            new_records_tx,
-            new_records_rx,
-        }
-    }
-}
-
 impl SoloShard {
     pub fn new(shard_state: ShardState, replication_position_inclusive: Position) -> Self {
+        let (new_records_tx, new_records_rx) = watch::channel(());
         Self {
             shard_state,
             replication_position_inclusive,
-            ..Default::default()
+            new_records_tx,
+            new_records_rx,
         }
     }
 }
 
+#[derive(Debug)]
 pub(super) enum IngesterShard {
     /// A primary shard hosted on a leader and replicated on a follower.
     Primary(PrimaryShard),
@@ -146,6 +137,15 @@ impl IngesterShard {
         .is_closed()
     }
 
+    pub fn close(&mut self) {
+        let shard_state = match self {
+            IngesterShard::Primary(primary_shard) => &mut primary_shard.shard_state,
+            IngesterShard::Replica(replica_shard) => &mut replica_shard.shard_state,
+            IngesterShard::Solo(solo_shard) => &mut solo_shard.shard_state,
+        };
+        *shard_state = ShardState::Closed;
+    }
+
     pub fn replication_position_inclusive(&self) -> Position {
         match self {
             IngesterShard::Primary(primary_shard) => &primary_shard.replication_position_inclusive,
@@ -192,3 +192,18 @@ impl IngesterShard {
         .expect("channel should be open");
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_new_solo_shard() {
+        let solo_shard = SoloShard::new(ShardState::Closed, Position::from(42u64));
+        assert_eq!(solo_shard.shard_state, ShardState::Closed);
+        assert_eq!(
+            solo_shard.replication_position_inclusive,
+            Position::from(42u64)
+        );
+    }
+}
diff --git a/quickwit/quickwit-ingest/src/ingest_v2/mrecord.rs b/quickwit/quickwit-ingest/src/ingest_v2/mrecord.rs
index 0d32ba4b52d..d73b7b83dc8 100644
--- a/quickwit/quickwit-ingest/src/ingest_v2/mrecord.rs
+++ b/quickwit/quickwit-ingest/src/ingest_v2/mrecord.rs
@@ -28,14 +28,17 @@ pub enum HeaderVersion {
     V0 = 0,
 }
 
+/// Length of the header of a [`MRecord`] in bytes.
+pub(super) const MRECORD_HEADER_LEN: usize = 2;
+
 /// `Doc` header v0 composed of the header version and the `Doc = 0` record type.
-const DOC_HEADER_V0: &[u8; 2] = &[HeaderVersion::V0 as u8, 0];
+const DOC_HEADER_V0: &[u8; MRECORD_HEADER_LEN] = &[HeaderVersion::V0 as u8, 0];
 
 /// `Commit` header v0 composed of the header version and the `Commit = 1` record type.
-const COMMIT_HEADER_V0: &[u8; 2] = &[HeaderVersion::V0 as u8, 1];
+const COMMIT_HEADER_V0: &[u8; MRECORD_HEADER_LEN] = &[HeaderVersion::V0 as u8, 1];
 
 /// `Eof` header v0 composed of the header version and the `Eof = 2` record type.
-const EOF_HEADER_V0: &[u8; 2] = &[HeaderVersion::V0 as u8, 2];
+const EOF_HEADER_V0: &[u8; MRECORD_HEADER_LEN] = &[HeaderVersion::V0 as u8, 2];
 
 #[derive(Debug, Clone, Eq, PartialEq)]
 pub enum MRecord {
diff --git a/quickwit/quickwit-ingest/src/ingest_v2/mrecordlog_utils.rs b/quickwit/quickwit-ingest/src/ingest_v2/mrecordlog_utils.rs
new file mode 100644
index 00000000000..15cf05a0465
--- /dev/null
+++ b/quickwit/quickwit-ingest/src/ingest_v2/mrecordlog_utils.rs
@@ -0,0 +1,159 @@
+// Copyright (C) 2023 Quickwit, Inc.
+//
+// Quickwit is offered under the AGPL v3.0 and as commercial software.
+// For commercial licensing, contact us at hello@quickwit.io.
+//
+// AGPL:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+use bytesize::ByteSize;
+use mrecordlog::error::{AppendError, MissingQueue};
+use mrecordlog::MultiRecordLog;
+use quickwit_proto::types::QueueId;
+use tracing::warn;
+
+use super::mrecord::is_eof_mrecord;
+use crate::MRecord;
+
+/// Appends an EOF record to the queue if it is empty or the last record is not an EOF
+/// record.
+pub(super) async fn append_eof_record_if_necessary(
+    mrecordlog: &mut MultiRecordLog,
+    queue_id: &QueueId,
+) {
+    let should_append_eof_record = match mrecordlog.last_record(queue_id) {
+        Ok(Some((_, last_mrecord))) => !is_eof_mrecord(&last_mrecord),
+        Ok(None) => true,
+        Err(MissingQueue(_)) => {
+            warn!("failed to append EOF record to queue `{queue_id}`: queue does not exist");
+            return;
+        }
+    };
+    if should_append_eof_record {
+        match mrecordlog
+            .append_record(queue_id, None, MRecord::Eof.encode())
+            .await
+        {
+            Ok(_) | Err(AppendError::MissingQueue(_)) => {}
+            Err(error) => {
+                warn!("failed to append EOF record to queue `{queue_id}`: {error}");
+            }
+        }
+    }
+}
+
+/// Error returned when the mrecordlog does not have enough capacity to store some records.
+#[derive(Debug, Clone, Copy, thiserror::Error)]
+pub(super) enum NotEnoughCapacityError {
+    #[error(
+        "write-ahead log is full, capacity: usage: {usage}, capacity: {capacity}, requested: \
+         {requested}"
+    )]
+    Disk {
+        usage: ByteSize,
+        capacity: ByteSize,
+        requested: ByteSize,
+    },
+    #[error(
+        "write-ahead log memory buffer is full, usage: {usage}, capacity: {capacity}, requested: \
+         {requested}"
+    )]
+    Memory {
+        usage: ByteSize,
+        capacity: ByteSize,
+        requested: ByteSize,
+    },
+}
+
+/// Checks whether the log has enough capacity to store some records.
+pub(super) fn check_enough_capacity(
+    mrecordlog: &MultiRecordLog,
+    disk_capacity: ByteSize,
+    memory_capacity: ByteSize,
+    requested_capacity: ByteSize,
+) -> Result<(), NotEnoughCapacityError> {
+    let disk_usage = ByteSize(mrecordlog.disk_usage() as u64);
+
+    if disk_usage + requested_capacity > disk_capacity {
+        return Err(NotEnoughCapacityError::Disk {
+            usage: disk_usage,
+            capacity: disk_capacity,
+            requested: requested_capacity,
+        });
+    }
+    let memory_usage = ByteSize(mrecordlog.memory_usage() as u64);
+
+    if memory_usage + requested_capacity > memory_capacity {
+        return Err(NotEnoughCapacityError::Memory {
+            usage: memory_usage,
+            capacity: memory_capacity,
+            requested: requested_capacity,
+        });
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_append_eof_record_if_necessary() {
+        let tempdir = tempfile::tempdir().unwrap();
+        let mut mrecordlog = MultiRecordLog::open(tempdir.path()).await.unwrap();
+
+        append_eof_record_if_necessary(&mut mrecordlog, &"queue-not-found".to_string()).await;
+
+        mrecordlog.create_queue("test-queue").await.unwrap();
+        append_eof_record_if_necessary(&mut mrecordlog, &"test-queue".to_string()).await;
+
+        let (last_position, last_record) = mrecordlog.last_record("test-queue").unwrap().unwrap();
+        assert_eq!(last_position, 0);
+        assert!(is_eof_mrecord(&last_record));
+
+        append_eof_record_if_necessary(&mut mrecordlog, &"test-queue".to_string()).await;
+        let (last_position, last_record) = mrecordlog.last_record("test-queue").unwrap().unwrap();
+        assert_eq!(last_position, 0);
+        assert!(is_eof_mrecord(&last_record));
+
+        mrecordlog.truncate("test-queue", 0).await.unwrap();
+
+        append_eof_record_if_necessary(&mut mrecordlog, &"test-queue".to_string()).await;
+        let (last_position, last_record) = mrecordlog.last_record("test-queue").unwrap().unwrap();
+        assert_eq!(last_position, 1);
+        assert!(is_eof_mrecord(&last_record));
+    }
+
+    #[tokio::test]
+    async fn test_check_enough_capacity() {
+        let tempdir = tempfile::tempdir().unwrap();
+        let mrecordlog = MultiRecordLog::open(tempdir.path()).await.unwrap();
+
+        let disk_error =
+            check_enough_capacity(&mrecordlog, ByteSize(0), ByteSize(0), ByteSize(12)).unwrap_err();
+
+        assert!(matches!(disk_error, NotEnoughCapacityError::Disk { .. }));
+
+        let memory_error =
+            check_enough_capacity(&mrecordlog, ByteSize::mb(256), ByteSize(11), ByteSize(12))
+                .unwrap_err();
+
+        assert!(matches!(
+            memory_error,
+            NotEnoughCapacityError::Memory { .. }
+        ));
+
+        check_enough_capacity(&mrecordlog, ByteSize::mb(256), ByteSize(12), ByteSize(12)).unwrap();
+    }
+}
diff --git a/quickwit/quickwit-ingest/src/ingest_v2/rate_limiter.rs b/quickwit/quickwit-ingest/src/ingest_v2/rate_limiter.rs
new file mode 100644
index 00000000000..1f310eca3b9
--- /dev/null
+++ b/quickwit/quickwit-ingest/src/ingest_v2/rate_limiter.rs
@@ -0,0 +1,179 @@
+// Copyright (C) 2023 Quickwit, Inc.
+//
+// Quickwit is offered under the AGPL v3.0 and as commercial software.
+// For commercial licensing, contact us at hello@quickwit.io.
+//
+// AGPL:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+use std::time::{Duration, Instant};
+
+use bytesize::ByteSize;
+use quickwit_common::tower::{ConstantRate, Rate};
+
+#[derive(Debug, Clone, Copy)]
+pub struct RateLimiterSettings {
+    // After a long period of inactivity, the rate limiter can accumulate some "credits"
+    // up to what we call a `burst_limit`.
+    //
+    // Until these credits are expired, the rate limiter may exceed temporarily its rate limit.
+    pub burst_limit: ByteSize,
+    pub rate_limit: ConstantRate,
+    // The refill period has an effect on the resolution at which the
+    // rate limiting is enforced.
+    //
+    // `Instant::now()` is guaranteed to be called at most once per refill_period.
+    pub refill_period: Duration,
+}
+
+impl Default for RateLimiterSettings {
+    fn default() -> Self {
+        // 10 MB burst limit.
+        let burst_limit = ByteSize::mb(10);
+        // 5 MB/s rate limit.
+        let rate_limit = ConstantRate::bytes_per_sec(ByteSize::mb(5));
+        // Refill every 100ms.
+        let refill_period = Duration::from_millis(100);
+
+        Self {
+            burst_limit,
+            rate_limit,
+            refill_period,
+        }
+    }
+}
+
+/// A bursty token-based rate limiter.
+#[derive(Debug, Clone)]
+pub(super) struct RateLimiter {
+    capacity: u64,
+    available: u64,
+    refill_amount: u64,
+    refill_period: Duration,
+    refill_period_micros: u64,
+    refill_at: Instant,
+}
+
+impl RateLimiter {
+    pub fn from_settings(settings: RateLimiterSettings) -> Self {
+        let capacity = settings.burst_limit.as_u64();
+
+        let work = settings.rate_limit.work() as u128;
+        let refill_period = settings.refill_period;
+        let rate_limit_period = settings.rate_limit.period();
+        let refill_amount = work * refill_period.as_nanos() / rate_limit_period.as_nanos();
+
+        Self {
+            capacity,
+            available: capacity,
+            refill_amount: refill_amount as u64,
+            refill_period,
+            refill_period_micros: refill_period.as_micros() as u64,
+            refill_at: Instant::now() + refill_period,
+        }
+    }
+
+    pub fn acquire(&mut self, capacity: ByteSize) -> bool {
+        if self.acquire_inner(capacity.as_u64()) {
+            true
+        } else {
+            self.refill(Instant::now());
+            self.acquire_inner(capacity.as_u64())
+        }
+    }
+
+    fn acquire_inner(&mut self, capacity: u64) -> bool {
+        if self.available >= capacity {
+            self.available -= capacity;
+            true
+        } else {
+            false
+        }
+    }
+
+    fn refill(&mut self, now: Instant) {
+        if now < self.refill_at {
+            return;
+        }
+        let elapsed = (now - self.refill_at).as_micros() as u64;
+        // More than one refill period may have elapsed so we need to take that into account.
+        let refill = self.refill_amount + self.refill_amount * elapsed / self.refill_period_micros;
+        self.available = std::cmp::min(self.available + refill, self.capacity);
+        self.refill_at = now + self.refill_period;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_rate_limiter() {
+        let settings = RateLimiterSettings {
+            burst_limit: ByteSize::mb(2),
+            rate_limit: ConstantRate::bytes_per_sec(ByteSize::mb(1)),
+            refill_period: Duration::from_millis(100),
+        };
+        let mut rate_limiter = RateLimiter::from_settings(settings);
+        assert_eq!(rate_limiter.capacity, ByteSize::mb(2).as_u64());
+        assert_eq!(rate_limiter.available, ByteSize::mb(2).as_u64());
+        assert_eq!(rate_limiter.refill_amount, ByteSize::kb(100).as_u64());
+        assert_eq!(rate_limiter.refill_period, Duration::from_millis(100));
+
+        assert!(rate_limiter.acquire(ByteSize::mb(1)));
+        assert!(rate_limiter.acquire(ByteSize::mb(1)));
+        assert!(!rate_limiter.acquire(ByteSize::kb(1)));
+
+        std::thread::sleep(Duration::from_millis(100));
+
+        assert!(rate_limiter.acquire(ByteSize::kb(100)));
+        assert!(!rate_limiter.acquire(ByteSize::kb(20)));
+
+        std::thread::sleep(Duration::from_millis(250));
+
+        assert!(rate_limiter.acquire(ByteSize::kb(125)));
+        assert!(rate_limiter.acquire(ByteSize::kb(125)));
+        assert!(!rate_limiter.acquire(ByteSize::kb(20)));
+    }
+
+    #[test]
+    fn test_rate_limiter_refill() {
+        let settings = RateLimiterSettings {
+            burst_limit: ByteSize::mb(2),
+            rate_limit: ConstantRate::bytes_per_sec(ByteSize::mb(1)),
+            refill_period: Duration::from_millis(100),
+        };
+        let mut rate_limiter = RateLimiter::from_settings(settings);
+
+        rate_limiter.available = 0;
+        let now = Instant::now();
+        rate_limiter.refill(now);
+        assert_eq!(rate_limiter.available, 0);
+
+        rate_limiter.available = 0;
+        let now = now + Duration::from_millis(100);
+        rate_limiter.refill(now);
+        assert_eq!(rate_limiter.available, ByteSize::kb(100).as_u64());
+
+        rate_limiter.available = 0;
+        let now = now + Duration::from_millis(110);
+        rate_limiter.refill(now);
+        assert_eq!(rate_limiter.available, ByteSize::kb(110).as_u64());
+
+        rate_limiter.available = 0;
+        let now = now + Duration::from_millis(210);
+        rate_limiter.refill(now);
+        assert_eq!(rate_limiter.available, ByteSize::kb(210).as_u64());
+    }
+}
diff --git a/quickwit/quickwit-ingest/src/ingest_v2/replication.rs b/quickwit/quickwit-ingest/src/ingest_v2/replication.rs
index e6fe41bbe68..4124a6f480d 100644
--- a/quickwit/quickwit-ingest/src/ingest_v2/replication.rs
+++ b/quickwit/quickwit-ingest/src/ingest_v2/replication.rs
@@ -22,22 +22,26 @@ use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
 
+use bytesize::ByteSize;
 use futures::{Future, StreamExt};
 use quickwit_common::ServiceStream;
 use quickwit_proto::ingest::ingester::{
-    ack_replication_message, syn_replication_message, AckReplicationMessage, ReplicateRequest,
-    ReplicateResponse, ReplicateSuccess, SynReplicationMessage,
+    ack_replication_message, syn_replication_message, AckReplicationMessage, ReplicateFailure,
+    ReplicateFailureReason, ReplicateRequest, ReplicateResponse, ReplicateSuccess,
+    SynReplicationMessage,
 };
 use quickwit_proto::ingest::{CommitTypeV2, IngestV2Error, IngestV2Result};
 use quickwit_proto::types::{NodeId, Position};
 use tokio::sync::mpsc::error::TryRecvError;
 use tokio::sync::{mpsc, oneshot, RwLock};
 use tokio::task::JoinHandle;
-use tracing::error;
+use tracing::{error, warn};
 
 use super::ingester::IngesterState;
 use super::models::{IngesterShard, ReplicaShard};
 use super::mrecord::MRecord;
+use super::mrecordlog_utils::check_enough_capacity;
+use crate::estimate_size;
 use crate::metrics::INGEST_METRICS;
 
 pub(super) const SYN_REPLICATION_STREAM_CAPACITY: usize = 5;
@@ -265,6 +269,8 @@ pub(super) struct ReplicationTask {
     syn_replication_stream: ServiceStream<SynReplicationMessage>,
     ack_replication_stream_tx: mpsc::UnboundedSender<IngestV2Result<AckReplicationMessage>>,
     current_replication_seqno: ReplicationSeqNo,
+    disk_capacity: ByteSize,
+    memory_capacity: ByteSize,
 }
 
 impl ReplicationTask {
@@ -274,6 +280,8 @@ impl ReplicationTask {
         state: Arc<RwLock<IngesterState>>,
         syn_replication_stream: ServiceStream<SynReplicationMessage>,
         ack_replication_stream_tx: mpsc::UnboundedSender<IngestV2Result<AckReplicationMessage>>,
+        disk_capacity: ByteSize,
+        memory_capacity: ByteSize,
     ) -> ReplicationTaskHandle {
         let mut replication_task = Self {
             leader_id,
@@ -282,6 +290,8 @@ impl ReplicationTask {
             syn_replication_stream,
             ack_replication_stream_tx,
             current_replication_seqno: 0,
+            disk_capacity,
+            memory_capacity,
         };
         let join_handle = tokio::spawn(async move { replication_task.run().await });
         ReplicationTaskHandle { join_handle }
@@ -314,7 +324,9 @@ impl ReplicationTask {
 
         let commit_type = replicate_request.commit_type();
         let force_commit = commit_type == CommitTypeV2::Force;
+
         let mut replicate_successes = Vec::with_capacity(replicate_request.subrequests.len());
+        let mut replicate_failures = Vec::new();
 
         let mut state_guard = self.state.write().await;
 
@@ -323,34 +335,75 @@ impl ReplicationTask {
             let from_position_exclusive = subrequest.from_position_exclusive();
             let to_position_inclusive = subrequest.to_position_inclusive();
 
-            let replica_shard: &mut IngesterShard =
-                if from_position_exclusive == Position::Beginning {
-                    // Initialize the replica shard and corresponding mrecordlog queue.
-                    state_guard
-                        .mrecordlog
-                        .create_queue(&queue_id)
-                        .await
-                        .expect("TODO");
-                    let leader_id: NodeId = replicate_request.leader_id.clone().into();
-                    let replica_shard = ReplicaShard::new(leader_id);
-                    let shard = IngesterShard::Replica(replica_shard);
-                    state_guard.shards.entry(queue_id.clone()).or_insert(shard)
-                } else {
-                    state_guard
-                        .shards
-                        .get_mut(&queue_id)
-                        .expect("replica shard should be initialized")
-                };
+            let replica_shard: &IngesterShard = if from_position_exclusive == Position::Beginning {
+                // Initialize the replica shard and corresponding mrecordlog queue.
+                state_guard
+                    .mrecordlog
+                    .create_queue(&queue_id)
+                    .await
+                    .expect("TODO");
+                let leader_id: NodeId = replicate_request.leader_id.clone().into();
+                let replica_shard = ReplicaShard::new(leader_id);
+                let shard = IngesterShard::Replica(replica_shard);
+                state_guard.shards.entry(queue_id.clone()).or_insert(shard)
+            } else {
+                state_guard
+                    .shards
+                    .get(&queue_id)
+                    .expect("replica shard should be initialized")
+            };
             if replica_shard.is_closed() {
-                // TODO
+                let replicate_failure = ReplicateFailure {
+                    subrequest_id: subrequest.subrequest_id,
+                    index_uid: subrequest.index_uid,
+                    source_id: subrequest.source_id,
+                    shard_id: subrequest.shard_id,
+                    reason: ReplicateFailureReason::ShardClosed as i32,
+                };
+                replicate_failures.push(replicate_failure);
+                continue;
             }
             if replica_shard.replication_position_inclusive() != from_position_exclusive {
                 // TODO
             }
-            let doc_batch = subrequest
-                .doc_batch
-                .expect("leader should not send empty replicate subrequests");
+            let doc_batch = match subrequest.doc_batch {
+                Some(doc_batch) if !doc_batch.is_empty() => doc_batch,
+                _ => {
+                    warn!("received empty replicate request");
 
+                    let replicate_success = ReplicateSuccess {
+                        subrequest_id: subrequest.subrequest_id,
+                        index_uid: subrequest.index_uid,
+                        source_id: subrequest.source_id,
+                        shard_id: subrequest.shard_id,
+                        replication_position_inclusive: Some(
+                            replica_shard.replication_position_inclusive(),
+                        ),
+                    };
+                    replicate_successes.push(replicate_success);
+                    continue;
+                }
+            };
+            let requested_capacity = estimate_size(&doc_batch);
+
+            if let Err(error) = check_enough_capacity(
+                &state_guard.mrecordlog,
+                self.disk_capacity,
+                self.memory_capacity,
+                requested_capacity,
+            ) {
+                warn!("failed to replicate records: {error}");
+
+                let replicate_failure = ReplicateFailure {
+                    subrequest_id: subrequest.subrequest_id,
+                    index_uid: subrequest.index_uid,
+                    source_id: subrequest.source_id,
+                    shard_id: subrequest.shard_id,
+                    reason: ReplicateFailureReason::ResourceExhausted as i32,
+                };
+                replicate_failures.push(replicate_failure);
+                continue;
+            }
             let current_position_inclusive: Position = if force_commit {
                 let encoded_mrecords = doc_batch
                     .docs()
@@ -380,17 +433,16 @@ impl ReplicationTask {
                 .replicated_num_docs_total
                 .inc_by(batch_num_docs);
 
-            let replica_shard = state_guard
-                .shards
-                .get_mut(&queue_id)
-                .expect("replica shard should exist");
-
             if current_position_inclusive != to_position_inclusive {
                 return Err(IngestV2Error::Internal(format!(
                     "bad replica position: expected {to_position_inclusive:?}, got \
                      {current_position_inclusive:?}"
                 )));
             }
+            let replica_shard = state_guard
+                .shards
+                .get_mut(&queue_id)
+                .expect("replica shard should be initialized");
             replica_shard.set_replication_position_inclusive(current_position_inclusive.clone());
 
             let replicate_success = ReplicateSuccess {
@@ -406,7 +458,7 @@ impl ReplicationTask {
         let replicate_response = ReplicateResponse {
             follower_id,
             successes: replicate_successes,
-            failures: Vec::new(),
+            failures: replicate_failures,
             replication_seqno: replicate_request.replication_seqno,
         };
         Ok(replicate_response)
@@ -465,7 +517,7 @@ mod tests {
 
     use mrecordlog::MultiRecordLog;
     use quickwit_proto::ingest::ingester::{ReplicateSubrequest, ReplicateSuccess};
-    use quickwit_proto::ingest::DocBatchV2;
+    use quickwit_proto::ingest::{DocBatchV2, ShardState};
     use quickwit_proto::types::queue_id;
 
     use super::*;
@@ -622,6 +674,7 @@ mod tests {
         let state = Arc::new(RwLock::new(IngesterState {
             mrecordlog,
             shards: HashMap::new(),
+            rate_limiters: HashMap::new(),
             replication_streams: HashMap::new(),
             replication_tasks: HashMap::new(),
         }));
@@ -629,12 +682,18 @@ mod tests {
             ServiceStream::new_bounded(SYN_REPLICATION_STREAM_CAPACITY);
         let (ack_replication_stream_tx, mut ack_replication_stream) =
             ServiceStream::new_unbounded();
+
+        let disk_capacity = ByteSize::mb(256);
+        let memory_capacity = ByteSize::mb(1);
+
         let _replication_task_handle = ReplicationTask::spawn(
             leader_id,
             follower_id,
             state.clone(),
             syn_replication_stream,
             ack_replication_stream_tx,
+            disk_capacity,
+            memory_capacity,
         );
         let replicate_request = ReplicateRequest {
             leader_id: "test-leader".to_string(),
@@ -769,4 +828,153 @@ mod tests {
             &[(0, "\0\0test-doc-foo"), (1, "\0\0test-doc-moo")],
         );
     }
+
+    #[tokio::test]
+    async fn test_replication_task_shard_closed() {
+        let leader_id: NodeId = "test-leader".into();
+        let follower_id: NodeId = "test-follower".into();
+        let tempdir = tempfile::tempdir().unwrap();
+        let mrecordlog = MultiRecordLog::open(tempdir.path()).await.unwrap();
+        let state = Arc::new(RwLock::new(IngesterState {
+            mrecordlog,
+            shards: HashMap::new(),
+            rate_limiters: HashMap::new(),
+            replication_streams: HashMap::new(),
+            replication_tasks: HashMap::new(),
+        }));
+        let (syn_replication_stream_tx, syn_replication_stream) =
+            ServiceStream::new_bounded(SYN_REPLICATION_STREAM_CAPACITY);
+        let (ack_replication_stream_tx, mut ack_replication_stream) =
+            ServiceStream::new_unbounded();
+
+        let disk_capacity = ByteSize::mb(256);
+        let memory_capacity = ByteSize::mb(1);
+
+        let _replication_task_handle = ReplicationTask::spawn(
+            leader_id.clone(),
+            follower_id,
+            state.clone(),
+            syn_replication_stream,
+            ack_replication_stream_tx,
+            disk_capacity,
+            memory_capacity,
+        );
+
+        let queue_id_01 = queue_id("test-index:0", "test-source", 1);
+
+        let mut replica_shard = ReplicaShard::new(leader_id);
+        replica_shard.shard_state = ShardState::Closed;
+        let shard = IngesterShard::Replica(replica_shard);
+
+        state
+            .write()
+            .await
+            .shards
+            .insert(queue_id_01.clone(), shard);
+
+        let replicate_request = ReplicateRequest {
+            leader_id: "test-leader".to_string(),
+            follower_id: "test-follower".to_string(),
+            commit_type: CommitTypeV2::Auto as i32,
+            subrequests: vec![ReplicateSubrequest {
+                subrequest_id: 0,
+                index_uid: "test-index:0".to_string(),
+                source_id: "test-source".to_string(),
+                shard_id: 1,
+                doc_batch: Some(DocBatchV2::for_test(["test-doc-foo"])),
+                from_position_exclusive: Position::from(0u64).into(),
+                to_position_inclusive: Some(Position::from(1u64)),
+            }],
+            replication_seqno: 0,
+        };
+        let syn_replication_message =
+            SynReplicationMessage::new_replicate_request(replicate_request);
+        syn_replication_stream_tx
+            .send(syn_replication_message)
+            .await
+            .unwrap();
+        let ack_replication_message = ack_replication_stream.next().await.unwrap().unwrap();
+        let replicate_response = into_replicate_response(ack_replication_message);
+
+        assert_eq!(replicate_response.follower_id, "test-follower");
+        assert_eq!(replicate_response.successes.len(), 0);
+        assert_eq!(replicate_response.failures.len(), 1);
+
+        let replicate_failure = &replicate_response.failures[0];
+        assert_eq!(replicate_failure.index_uid, "test-index:0");
+        assert_eq!(replicate_failure.source_id, "test-source");
+        assert_eq!(replicate_failure.shard_id, 1);
+        assert_eq!(
+            replicate_failure.reason(),
+            ReplicateFailureReason::ShardClosed
+        );
+    }
+
+    #[tokio::test]
+    async fn test_replication_task_resource_exhausted() {
+        let leader_id: NodeId = "test-leader".into();
+        let follower_id: NodeId = "test-follower".into();
+        let tempdir = tempfile::tempdir().unwrap();
+        let mrecordlog = MultiRecordLog::open(tempdir.path()).await.unwrap();
+        let state = Arc::new(RwLock::new(IngesterState {
+            mrecordlog,
+            shards: HashMap::new(),
+            rate_limiters: HashMap::new(),
+            replication_streams: HashMap::new(),
+            replication_tasks: HashMap::new(),
+        }));
+        let (syn_replication_stream_tx, syn_replication_stream) =
+            ServiceStream::new_bounded(SYN_REPLICATION_STREAM_CAPACITY);
+        let (ack_replication_stream_tx, mut ack_replication_stream) =
+            ServiceStream::new_unbounded();
+
+        let disk_capacity = ByteSize(0);
+        let memory_capacity = ByteSize(0);
+
+        let _replication_task_handle = ReplicationTask::spawn(
+            leader_id,
+            follower_id,
+            state.clone(),
+            syn_replication_stream,
+            ack_replication_stream_tx,
+            disk_capacity,
+            memory_capacity,
+        );
+        let replicate_request = ReplicateRequest {
+            leader_id: "test-leader".to_string(),
+            follower_id: "test-follower".to_string(),
+            commit_type: CommitTypeV2::Auto as i32,
+            subrequests: vec![ReplicateSubrequest {
+                subrequest_id: 0,
+                index_uid: "test-index:0".to_string(),
+                source_id: "test-source".to_string(),
+                shard_id: 1,
+                doc_batch: Some(DocBatchV2::for_test(["test-doc-foo"])),
+                from_position_exclusive: None,
+                to_position_inclusive: Some(Position::from(0u64)),
+            }],
+            replication_seqno: 0,
+        };
+        let syn_replication_message =
+            SynReplicationMessage::new_replicate_request(replicate_request);
+        syn_replication_stream_tx
+            .send(syn_replication_message)
+            .await
+            .unwrap();
+        let ack_replication_message = ack_replication_stream.next().await.unwrap().unwrap();
+        let replicate_response = into_replicate_response(ack_replication_message);
+
+        assert_eq!(replicate_response.follower_id, "test-follower");
+        assert_eq!(replicate_response.successes.len(), 0);
+        assert_eq!(replicate_response.failures.len(), 1);
+
+        let replicate_failure_0 = &replicate_response.failures[0];
+        assert_eq!(replicate_failure_0.index_uid, "test-index:0");
+        assert_eq!(replicate_failure_0.source_id, "test-source");
+        assert_eq!(replicate_failure_0.shard_id, 1);
+        assert_eq!(
+            replicate_failure_0.reason(),
+            ReplicateFailureReason::ResourceExhausted
+        );
+    }
 }
diff --git a/quickwit/quickwit-ingest/src/ingest_v2/router.rs b/quickwit/quickwit-ingest/src/ingest_v2/router.rs
index 1f1967b3580..71ec7f481c6 100644
--- a/quickwit/quickwit-ingest/src/ingest_v2/router.rs
+++ b/quickwit/quickwit-ingest/src/ingest_v2/router.rs
@@ -26,7 +26,7 @@ use async_trait::async_trait;
 use futures::stream::FuturesUnordered;
 use futures::{Future, StreamExt};
 use quickwit_proto::control_plane::{
-    ClosedShards, ControlPlaneService, ControlPlaneServiceClient, GetOrCreateOpenShardsRequest,
+    ControlPlaneService, ControlPlaneServiceClient, GetOrCreateOpenShardsRequest,
     GetOrCreateOpenShardsSubrequest,
 };
 use quickwit_proto::ingest::ingester::{
@@ -35,10 +35,10 @@ use quickwit_proto::ingest::ingester::{
 use quickwit_proto::ingest::router::{
     IngestRequestV2, IngestResponseV2, IngestRouterService, IngestSubrequest,
 };
-use quickwit_proto::ingest::{CommitTypeV2, IngestV2Error, IngestV2Result};
+use quickwit_proto::ingest::{ClosedShards, CommitTypeV2, IngestV2Error, IngestV2Result};
 use quickwit_proto::types::{IndexUid, NodeId, ShardId, SourceId, SubrequestId};
 use tokio::sync::RwLock;
-use tracing::{error, warn};
+use tracing::{error, info, warn};
 
 use super::ingester::PERSIST_REQUEST_TIMEOUT;
 use super::shard_table::ShardTable;
@@ -133,6 +133,18 @@ impl IngestRouter {
                 get_open_shards_subrequests.push(subrequest);
             }
         }
+        if !closed_shards.is_empty() {
+            info!(
+                "reporting {} closed shard(s) to control-plane",
+                closed_shards.len()
+            )
+        }
+        if !unavailable_leaders.is_empty() {
+            info!(
+                "reporting {} unavailable leader(s) to control-plane",
+                unavailable_leaders.len()
+            );
+        }
         GetOrCreateOpenShardsRequest {
             subrequests: get_open_shards_subrequests,
             closed_shards,
diff --git a/quickwit/quickwit-ingest/src/ingest_v2/shard_table.rs b/quickwit/quickwit-ingest/src/ingest_v2/shard_table.rs
index 3f5ba077c59..8109ff5388d 100644
--- a/quickwit/quickwit-ingest/src/ingest_v2/shard_table.rs
+++ b/quickwit/quickwit-ingest/src/ingest_v2/shard_table.rs
@@ -20,8 +20,7 @@
 use std::collections::{HashMap, HashSet};
 use std::sync::atomic::{AtomicUsize, Ordering};
 
-use quickwit_proto::control_plane::ClosedShards;
-use quickwit_proto::ingest::{Shard, ShardState};
+use quickwit_proto::ingest::{ClosedShards, Shard, ShardState};
 use quickwit_proto::types::{IndexId, IndexUid, NodeId, ShardId, SourceId};
 use tracing::warn;
 
diff --git a/quickwit/quickwit-ingest/src/lib.rs b/quickwit/quickwit-ingest/src/lib.rs
index 263243f27d8..fc070982785 100644
--- a/quickwit/quickwit-ingest/src/lib.rs
+++ b/quickwit/quickwit-ingest/src/lib.rs
@@ -72,8 +72,8 @@ pub async fn init_ingest_api(
     }
     let ingest_api_actor = IngestApiService::with_queues_dir(
         queues_dir_path,
-        config.max_queue_memory_usage.get_bytes() as usize,
-        config.max_queue_disk_usage.get_bytes() as usize,
+        config.max_queue_memory_usage.as_u64() as usize,
+        config.max_queue_disk_usage.as_u64() as usize,
     )
     .await
     .with_context(|| {
@@ -127,7 +127,7 @@ impl CommitType {
 #[cfg(test)]
 mod tests {
 
-    use byte_unit::Byte;
+    use bytesize::ByteSize;
     use quickwit_actors::AskError;
 
     use super::*;
@@ -218,8 +218,8 @@ mod tests {
         get_ingest_api_service(&queues_dir_path).await.unwrap_err();
 
         let ingest_api_config = IngestApiConfig {
-            max_queue_memory_usage: Byte::from_bytes(1200),
-            max_queue_disk_usage: Byte::from_bytes(1024 * 1024 * 256),
+            max_queue_memory_usage: ByteSize(1200),
+            max_queue_disk_usage: ByteSize::mib(256),
             ..Default::default()
         };
         init_ingest_api(&universe, &queues_dir_path, &ingest_api_config)
diff --git a/quickwit/quickwit-ingest/src/queue.rs b/quickwit/quickwit-ingest/src/queue.rs
index df5e80e593c..cb3e7e9eacb 100644
--- a/quickwit/quickwit-ingest/src/queue.rs
+++ b/quickwit/quickwit-ingest/src/queue.rs
@@ -209,11 +209,11 @@ impl Queues {
     }
 
     pub(crate) fn disk_usage(&self) -> usize {
-        self.record_log.on_disk_size()
+        self.record_log.disk_usage()
     }
 
     pub(crate) fn memory_usage(&self) -> usize {
-        self.record_log.in_memory_size()
+        self.record_log.memory_usage()
     }
 }
 
diff --git a/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs b/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs
index 4c94f5d9851..2d832c9d0b3 100644
--- a/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs
+++ b/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs
@@ -108,7 +108,7 @@ macro_rules! ingest_json {
     };
 }
 
-pub async fn ingest_with_retry(
+pub(crate) async fn ingest_with_retry(
     client: &QuickwitClient,
     index_id: &str,
     ingest_source: IngestSource,
@@ -201,6 +201,10 @@ impl ClusterSandbox {
         })
     }
 
+    pub fn enable_ingest_v2(&mut self) {
+        self.searcher_rest_client.enable_ingest_v2();
+    }
+
     // Starts one node that runs all the services.
     pub async fn start_standalone_node() -> anyhow::Result<Self> {
         let temp_dir = tempfile::tempdir()?;
@@ -286,13 +290,13 @@ impl ClusterSandbox {
     pub async fn wait_for_splits(
         &self,
         index_id: &str,
-        split_states: Option<Vec<SplitState>>,
+        split_states_filter: Option<Vec<SplitState>>,
         required_splits_num: usize,
     ) -> anyhow::Result<()> {
         wait_until_predicate(
             || {
                 let splits_query_params = ListSplitsQueryParams {
-                    split_states: split_states.clone(),
+                    split_states: split_states_filter.clone(),
                     ..Default::default()
                 };
                 async move {
@@ -322,7 +326,7 @@ impl ClusterSandbox {
                 }
             },
             Duration::from_secs(10),
-            Duration::from_millis(100),
+            Duration::from_millis(500),
         )
         .await?;
         Ok(())
diff --git a/quickwit/quickwit-integration-tests/src/test_utils/mod.rs b/quickwit/quickwit-integration-tests/src/test_utils/mod.rs
index 24be4084f9b..79c23571c88 100644
--- a/quickwit/quickwit-integration-tests/src/test_utils/mod.rs
+++ b/quickwit/quickwit-integration-tests/src/test_utils/mod.rs
@@ -19,4 +19,4 @@
 
 mod cluster_sandbox;
 
-pub use cluster_sandbox::{build_node_configs, ingest_with_retry, ClusterSandbox};
+pub(crate) use cluster_sandbox::{ingest_with_retry, ClusterSandbox};
diff --git a/quickwit/quickwit-integration-tests/src/tests/index_tests.rs b/quickwit/quickwit-integration-tests/src/tests/index_tests.rs
index e887cb82281..688114cdb98 100644
--- a/quickwit/quickwit-integration-tests/src/tests/index_tests.rs
+++ b/quickwit/quickwit-integration-tests/src/tests/index_tests.rs
@@ -23,9 +23,11 @@ use std::time::Duration;
 use bytes::Bytes;
 use quickwit_common::test_utils::wait_until_predicate;
 use quickwit_config::service::QuickwitService;
+use quickwit_config::ConfigFormat;
 use quickwit_indexing::actors::INDEXING_DIR_NAME;
 use quickwit_janitor::actors::DELETE_SERVICE_TASK_DIR_NAME;
 use quickwit_metastore::SplitState;
+use quickwit_rest_client::error::{ApiError, Error};
 use quickwit_rest_client::rest_client::CommitType;
 use quickwit_serve::SearchRequestQueryString;
 use serde_json::json;
@@ -60,11 +62,7 @@ async fn test_restarting_standalone_server() {
     sandbox
         .indexer_rest_client
         .indexes()
-        .create(
-            index_config.clone(),
-            quickwit_config::ConfigFormat::Yaml,
-            false,
-        )
+        .create(index_config.clone(), ConfigFormat::Yaml, false)
         .await
         .unwrap();
 
@@ -102,7 +100,7 @@ async fn test_restarting_standalone_server() {
     sandbox
         .indexer_rest_client
         .indexes()
-        .create(index_config, quickwit_config::ConfigFormat::Yaml, false)
+        .create(index_config, ConfigFormat::Yaml, false)
         .await
         .unwrap();
 
@@ -215,36 +213,124 @@ async fn test_restarting_standalone_server() {
     sandbox.shutdown().await.unwrap();
 }
 
+const TEST_INDEX_CONFIG: &str = r#"
+    version: 0.6
+    index_id: test_index
+    doc_mapping:
+      field_mappings:
+      - name: body
+        type: text
+    indexing_settings:
+      commit_timeout_secs: 1
+      merge_policy:
+        type: stable_log
+        merge_factor: 4
+        max_merge_factor: 4
+"#;
+
+#[tokio::test]
+async fn test_ingest_v2_index_not_found() {
+    // This tests checks what happens when we try to ingest into a non-existing index.
+    quickwit_common::setup_logging_for_tests();
+    let nodes_services = &[
+        HashSet::from_iter([QuickwitService::Indexer, QuickwitService::Janitor]),
+        HashSet::from_iter([QuickwitService::Indexer, QuickwitService::Janitor]),
+        HashSet::from_iter([
+            QuickwitService::ControlPlane,
+            QuickwitService::Metastore,
+            QuickwitService::Searcher,
+        ]),
+    ];
+    let mut sandbox = ClusterSandbox::start_cluster_nodes(&nodes_services[..])
+        .await
+        .unwrap();
+    sandbox.enable_ingest_v2();
+    sandbox.wait_for_cluster_num_ready_nodes(3).await.unwrap();
+    let missing_index_err: Error = sandbox
+        .indexer_rest_client
+        .ingest(
+            "missing_index",
+            ingest_json!({"body": "doc1"}),
+            None,
+            None,
+            CommitType::WaitFor,
+        )
+        .await
+        .unwrap_err();
+    let Error::Api(ApiError { message, code }) = missing_index_err else {
+        panic!("Expected an API error.");
+    };
+    assert_eq!(code, 404u16);
+    let error_message = message.unwrap();
+    assert_eq!(error_message, "index `missing_index` not found");
+    sandbox.shutdown().await.unwrap();
+}
+
+#[tokio::test]
+async fn test_ingest_v2_happy_path() {
+    // This tests checks our happy path for ingesting one doc.
+    quickwit_common::setup_logging_for_tests();
+    let nodes_services = &[
+        HashSet::from_iter([QuickwitService::Indexer, QuickwitService::Janitor]),
+        HashSet::from_iter([QuickwitService::Indexer, QuickwitService::Janitor]),
+        HashSet::from_iter([
+            QuickwitService::ControlPlane,
+            QuickwitService::Metastore,
+            QuickwitService::Searcher,
+        ]),
+    ];
+    let mut sandbox = ClusterSandbox::start_cluster_nodes(&nodes_services[..])
+        .await
+        .unwrap();
+    sandbox.enable_ingest_v2();
+    sandbox.wait_for_cluster_num_ready_nodes(3).await.unwrap();
+    sandbox
+        .indexer_rest_client
+        .indexes()
+        .create(TEST_INDEX_CONFIG.into(), ConfigFormat::Yaml, false)
+        .await
+        .unwrap();
+    sandbox
+        .indexer_rest_client
+        .sources("test_index")
+        .toggle("_ingest-source", true)
+        .await
+        .unwrap();
+    sandbox
+        .indexer_rest_client
+        .ingest(
+            "test_index",
+            ingest_json!({"body": "doc1"}),
+            None,
+            None,
+            CommitType::WaitFor,
+        )
+        .await
+        .unwrap();
+    let search_req = SearchRequestQueryString {
+        query: "*".to_string(),
+        ..Default::default()
+    };
+    let search_result = sandbox
+        .indexer_rest_client
+        .search("test_index", search_req)
+        .await
+        .unwrap();
+    assert_eq!(search_result.num_hits, 1);
+    sandbox.shutdown().await.unwrap();
+}
+
 #[tokio::test]
 async fn test_commit_modes() {
     quickwit_common::setup_logging_for_tests();
     let sandbox = ClusterSandbox::start_standalone_node().await.unwrap();
-    let index_id = "test_commit_modes_index";
+    let index_id = "test_index";
 
     // Create index
     sandbox
         .indexer_rest_client
         .indexes()
-        .create(
-            r#"
-            version: 0.6
-            index_id: test_commit_modes_index
-            doc_mapping:
-              field_mappings:
-              - name: body
-                type: text
-            indexing_settings:
-              commit_timeout_secs: 1
-              merge_policy:
-                type: stable_log
-                merge_factor: 4
-                max_merge_factor: 4
-
-            "#
-            .into(),
-            quickwit_config::ConfigFormat::Yaml,
-            false,
-        )
+        .create(TEST_INDEX_CONFIG.into(), ConfigFormat::Yaml, false)
         .await
         .unwrap();
 
@@ -391,7 +477,7 @@ async fn test_very_large_index_name() {
                 "#,
             )
             .into(),
-            quickwit_config::ConfigFormat::Yaml,
+            ConfigFormat::Yaml,
             false,
         )
         .await
@@ -447,7 +533,7 @@ async fn test_very_large_index_name() {
                     "#,
             )
             .into(),
-            quickwit_config::ConfigFormat::Yaml,
+            ConfigFormat::Yaml,
             false,
         )
         .await
@@ -484,7 +570,7 @@ async fn test_shutdown() {
               commit_timeout_secs: 1
             "#
             .into(),
-            quickwit_config::ConfigFormat::Yaml,
+            ConfigFormat::Yaml,
             false,
         )
         .await
diff --git a/quickwit/quickwit-janitor/src/actors/delete_task_pipeline.rs b/quickwit/quickwit-janitor/src/actors/delete_task_pipeline.rs
index 23e4bf43eb3..78196e48bce 100644
--- a/quickwit/quickwit-janitor/src/actors/delete_task_pipeline.rs
+++ b/quickwit/quickwit-janitor/src/actors/delete_task_pipeline.rs
@@ -199,7 +199,7 @@ impl DeleteTaskPipeline {
             .resources
             .max_merge_write_throughput
             .as_ref()
-            .map(|bytes_per_sec| bytes_per_sec.get_bytes() as f64)
+            .map(|bytes_per_sec| bytes_per_sec.as_u64() as f64)
             .unwrap_or(f64::INFINITY);
         let delete_executor_io_controls = IoControls::default()
             .set_throughput_limit(throughput_limit)
diff --git a/quickwit/quickwit-janitor/src/actors/garbage_collector.rs b/quickwit/quickwit-janitor/src/actors/garbage_collector.rs
index 61f151b5664..5e80b11d179 100644
--- a/quickwit/quickwit-janitor/src/actors/garbage_collector.rs
+++ b/quickwit/quickwit-janitor/src/actors/garbage_collector.rs
@@ -161,7 +161,7 @@ impl GarbageCollector {
                 self.counters.num_deleted_files += deleted_file_entries.len();
                 self.counters.num_deleted_bytes += deleted_file_entries
                     .iter()
-                    .map(|entry| entry.file_size_bytes.get_bytes() as usize)
+                    .map(|entry| entry.file_size_bytes.as_u64() as usize)
                     .sum::<usize>();
             }
         }
diff --git a/quickwit/quickwit-metastore/Cargo.toml b/quickwit/quickwit-metastore/Cargo.toml
index d09f7ff4e8e..96bc49a3618 100644
--- a/quickwit/quickwit-metastore/Cargo.toml
+++ b/quickwit/quickwit-metastore/Cargo.toml
@@ -12,7 +12,7 @@ documentation = "https://quickwit.io/docs/"
 [dependencies]
 anyhow = { workspace = true }
 async-trait = { workspace = true }
-byte-unit = { workspace = true }
+bytesize = { workspace = true }
 futures = { workspace = true }
 http = { workspace = true }
 itertools = { workspace = true }
@@ -20,12 +20,12 @@ mockall = { workspace = true, optional = true }
 once_cell = { workspace = true }
 rand = { workspace = true }
 regex = { workspace = true }
+sea-query = { workspace = true, optional = true }
+sea-query-binder = { workspace = true, optional = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 serde_with = { workspace = true }
 sqlx = { workspace = true, optional = true }
-sea-query = { workspace = true }
-sea-query-binder = { workspace = true }
 tempfile = { workspace = true, optional = true }
 thiserror = { workspace = true }
 time = { workspace = true }
@@ -60,5 +60,5 @@ quickwit-storage = { workspace = true, features = ["testsuite"] }
 [features]
 azure = ["quickwit-storage/azure"]
 ci-test = []
-postgres = ["quickwit-proto/postgres", "sqlx"]
+postgres = ["quickwit-proto/postgres", "sea-query", "sea-query-binder", "sqlx"]
 testsuite = ["mockall", "tempfile", "quickwit-config/testsuite"]
diff --git a/quickwit/quickwit-metastore/src/split_metadata.rs b/quickwit/quickwit-metastore/src/split_metadata.rs
index d8f78ffc565..d31f3d2a60e 100644
--- a/quickwit/quickwit-metastore/src/split_metadata.rs
+++ b/quickwit/quickwit-metastore/src/split_metadata.rs
@@ -24,7 +24,7 @@ use std::path::PathBuf;
 use std::str::FromStr;
 use std::time::Duration;
 
-use byte_unit::Byte;
+use bytesize::ByteSize;
 use quickwit_proto::types::IndexUid;
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DurationMilliSeconds};
@@ -231,9 +231,9 @@ impl SplitMetadata {
         let file_name = quickwit_common::split_file(self.split_id());
 
         SplitInfo {
-            uncompressed_docs_size_bytes: Byte::from_bytes(self.uncompressed_docs_size_in_bytes),
+            uncompressed_docs_size_bytes: ByteSize(self.uncompressed_docs_size_in_bytes),
             file_name: PathBuf::from(file_name),
-            file_size_bytes: Byte::from_bytes(self.footer_offsets.end),
+            file_size_bytes: ByteSize(self.footer_offsets.end),
             split_id: self.split_id.clone(),
             num_docs: self.num_docs,
         }
@@ -249,13 +249,13 @@ pub struct SplitInfo {
     pub num_docs: usize,
     /// The sum of the sizes of the original JSON payloads in bytes.
     #[schema(value_type = u64)]
-    pub uncompressed_docs_size_bytes: Byte,
+    pub uncompressed_docs_size_bytes: ByteSize,
     /// The name of the split file on disk.
     #[schema(value_type = String)]
     pub file_name: PathBuf,
     /// The size of the split file on disk in bytes.
     #[schema(value_type = u64)]
-    pub file_size_bytes: Byte,
+    pub file_size_bytes: ByteSize,
 }
 
 #[cfg(any(test, feature = "testsuite"))]
diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.expected.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.expected.json
index 169cca9abcf..91c552514d7 100644
--- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.expected.json
+++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.expected.json
@@ -98,7 +98,7 @@
           "type": "stable_log"
         },
         "resources": {
-          "heap_size": 3
+          "heap_size": "50.0 MB"
         },
         "split_num_docs_target": 10000001
       },
diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.json
index de49a1ab545..a0c2f309a9f 100644
--- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.json
+++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.json
@@ -86,7 +86,7 @@
           "type": "stable_log"
         },
         "resources": {
-          "heap_size": 3
+          "heap_size": 50000000
         },
         "split_num_docs_target": 10000001
       },
diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.expected.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.expected.json
index b9abd44b004..239836b57ff 100644
--- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.expected.json
+++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.expected.json
@@ -98,7 +98,7 @@
           "type": "stable_log"
         },
         "resources": {
-          "heap_size": 3
+          "heap_size": "50.0 MB"
         },
         "split_num_docs_target": 10000001
       },
diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.json
index 9977998242d..f4f7d26d1f5 100644
--- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.json
+++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.json
@@ -86,7 +86,7 @@
           "type": "stable_log"
         },
         "resources": {
-          "heap_size": 3
+          "heap_size": 50000000
         },
         "split_num_docs_target": 10000001
       },
diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.expected.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.expected.json
index 3a48e362219..c8a478a3c3e 100644
--- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.expected.json
+++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.expected.json
@@ -40,6 +40,7 @@
           },
           {
             "fast": true,
+            "fast_precision": "seconds",
             "indexed": true,
             "input_formats": [
               "rfc3339",
@@ -47,7 +48,6 @@
             ],
             "name": "timestamp",
             "output_format": "rfc3339",
-            "fast_precision": "seconds",
             "stored": true,
             "type": "datetime"
           },
@@ -105,7 +105,7 @@
           "type": "stable_log"
         },
         "resources": {
-          "heap_size": 3
+          "heap_size": "50.0 MB"
         },
         "split_num_docs_target": 10000001
       },
diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.json
index 3a48e362219..c8a478a3c3e 100644
--- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.json
+++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.json
@@ -40,6 +40,7 @@
           },
           {
             "fast": true,
+            "fast_precision": "seconds",
             "indexed": true,
             "input_formats": [
               "rfc3339",
@@ -47,7 +48,6 @@
             ],
             "name": "timestamp",
             "output_format": "rfc3339",
-            "fast_precision": "seconds",
             "stored": true,
             "type": "datetime"
           },
@@ -105,7 +105,7 @@
           "type": "stable_log"
         },
         "resources": {
-          "heap_size": 3
+          "heap_size": "50.0 MB"
         },
         "split_num_docs_target": 10000001
       },
diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.expected.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.expected.json
index b2edb60f832..1962205d42d 100644
--- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.expected.json
+++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.expected.json
@@ -87,7 +87,7 @@
         "type": "stable_log"
       },
       "resources": {
-        "heap_size": 3
+        "heap_size": "50.0 MB"
       },
       "split_num_docs_target": 10000001
     },
diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.json
index b3c3cdf99e6..a6316834104 100644
--- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.json
+++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.json
@@ -72,7 +72,7 @@
         "type": "stable_log"
       },
       "resources": {
-        "heap_size": 3
+        "heap_size": 50000000
       },
       "split_num_docs_target": 10000001
     },
diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.expected.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.expected.json
index b2edb60f832..1962205d42d 100644
--- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.expected.json
+++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.expected.json
@@ -87,7 +87,7 @@
         "type": "stable_log"
       },
       "resources": {
-        "heap_size": 3
+        "heap_size": "50.0 MB"
       },
       "split_num_docs_target": 10000001
     },
diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.json
index 87fc824fc15..66233c904d7 100644
--- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.json
+++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.json
@@ -72,7 +72,7 @@
         "type": "stable_log"
       },
       "resources": {
-        "heap_size": 3
+        "heap_size": 50000000
       },
       "split_num_docs_target": 10000001
     },
diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.expected.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.expected.json
index d38a9e51e02..f6522a1ba38 100644
--- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.expected.json
+++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.expected.json
@@ -29,6 +29,7 @@
         },
         {
           "fast": true,
+          "fast_precision": "seconds",
           "indexed": true,
           "input_formats": [
             "rfc3339",
@@ -36,7 +37,6 @@
           ],
           "name": "timestamp",
           "output_format": "rfc3339",
-          "fast_precision": "seconds",
           "stored": true,
           "type": "datetime"
         },
@@ -94,7 +94,7 @@
         "type": "stable_log"
       },
       "resources": {
-        "heap_size": 3
+        "heap_size": "50.0 MB"
       },
       "split_num_docs_target": 10000001
     },
diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.json
index d38a9e51e02..f6522a1ba38 100644
--- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.json
+++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.json
@@ -29,6 +29,7 @@
         },
         {
           "fast": true,
+          "fast_precision": "seconds",
           "indexed": true,
           "input_formats": [
             "rfc3339",
@@ -36,7 +37,6 @@
           ],
           "name": "timestamp",
           "output_format": "rfc3339",
-          "fast_precision": "seconds",
           "stored": true,
           "type": "datetime"
         },
@@ -94,7 +94,7 @@
         "type": "stable_log"
       },
       "resources": {
-        "heap_size": 3
+        "heap_size": "50.0 MB"
       },
       "split_num_docs_target": 10000001
     },
diff --git a/quickwit/quickwit-proto/protos/quickwit/control_plane.proto b/quickwit/quickwit-proto/protos/quickwit/control_plane.proto
index 32e9686c7e5..ed630375612 100644
--- a/quickwit/quickwit-proto/protos/quickwit/control_plane.proto
+++ b/quickwit/quickwit-proto/protos/quickwit/control_plane.proto
@@ -65,7 +65,7 @@ service ControlPlaneService {
 
 message GetOrCreateOpenShardsRequest {
   repeated GetOrCreateOpenShardsSubrequest subrequests = 1;
-  repeated ClosedShards closed_shards = 2;
+  repeated quickwit.ingest.ClosedShards closed_shards = 2;
   repeated string unavailable_leaders = 3;
 }
 
@@ -75,12 +75,6 @@ message GetOrCreateOpenShardsSubrequest {
   string source_id = 3;
 }
 
-message ClosedShards {
-  string index_uid = 1;
-  string source_id = 2;
-  repeated uint64 shard_ids = 3;
-}
-
 message GetOrCreateOpenShardsResponse {
   repeated GetOrCreateOpenShardsSuccess successes = 1;
   repeated GetOrCreateOpenShardsFailure failures = 2;
diff --git a/quickwit/quickwit-proto/protos/quickwit/ingest.proto b/quickwit/quickwit-proto/protos/quickwit/ingest.proto
index e6b0288759f..e4653bd9b16 100644
--- a/quickwit/quickwit-proto/protos/quickwit/ingest.proto
+++ b/quickwit/quickwit-proto/protos/quickwit/ingest.proto
@@ -78,3 +78,9 @@ message Shard {
   // For instance, if an indexer goes rogue, eventually the control plane will detect it and assign the shard to another indexer, which will override the publish token.
   optional string publish_token = 10;
 }
+
+message ClosedShards {
+  string index_uid = 1;
+  string source_id = 2;
+  repeated uint64 shard_ids = 3;
+}
diff --git a/quickwit/quickwit-proto/protos/quickwit/ingester.proto b/quickwit/quickwit-proto/protos/quickwit/ingester.proto
index c7dd2e9c6be..109888c60f8 100644
--- a/quickwit/quickwit-proto/protos/quickwit/ingester.proto
+++ b/quickwit/quickwit-proto/protos/quickwit/ingester.proto
@@ -35,6 +35,8 @@ service IngesterService {
 
   // rpc OpenWatchStream(OpenWatchStreamRequest) returns (stream WatchMessage);
 
+  rpc CloseShards(CloseShardsRequest) returns (CloseShardsResponse);
+
   // Pings an ingester to check if it is ready to host shards and serve requests.
   rpc Ping(PingRequest) returns (PingResponse);
 
@@ -143,13 +145,19 @@ message ReplicateSuccess {
   quickwit.ingest.Position replication_position_inclusive = 5;
 }
 
+enum ReplicateFailureReason {
+  REPLICATE_FAILURE_REASON_UNSPECIFIED = 0;
+  REPLICATE_FAILURE_REASON_SHARD_CLOSED = 1;
+  reserved 2; // REPLICATE_FAILURE_REASON_RATE_LIMITED = 2;
+  REPLICATE_FAILURE_REASON_RESOURCE_EXHAUSTED = 3;
+}
+
 message ReplicateFailure {
   uint32 subrequest_id = 1;
   string index_uid = 2;
   string source_id = 3;
   uint64 shard_id = 4;
-  // ingest.DocBatchV2 doc_batch = 4;
-  // ingest.IngestError error = 5;
+  ReplicateFailureReason reason = 5;
 }
 
 message TruncateRequest {
@@ -186,6 +194,13 @@ message FetchResponseV2 {
   quickwit.ingest.Position to_position_inclusive = 6;
 }
 
+message CloseShardsRequest {
+  repeated quickwit.ingest.ClosedShards closed_shards = 1;
+}
+
+message CloseShardsResponse {
+}
+
 message PingRequest {
   string leader_id = 1;
   optional string follower_id = 2;
diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs
index 5e7c01caade..e276eb95832 100644
--- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs
+++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs
@@ -5,7 +5,7 @@ pub struct GetOrCreateOpenShardsRequest {
     #[prost(message, repeated, tag = "1")]
     pub subrequests: ::prost::alloc::vec::Vec<GetOrCreateOpenShardsSubrequest>,
     #[prost(message, repeated, tag = "2")]
-    pub closed_shards: ::prost::alloc::vec::Vec<ClosedShards>,
+    pub closed_shards: ::prost::alloc::vec::Vec<super::ingest::ClosedShards>,
     #[prost(string, repeated, tag = "3")]
     pub unavailable_leaders: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
 }
@@ -23,17 +23,6 @@ pub struct GetOrCreateOpenShardsSubrequest {
 #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)]
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
-pub struct ClosedShards {
-    #[prost(string, tag = "1")]
-    pub index_uid: ::prost::alloc::string::String,
-    #[prost(string, tag = "2")]
-    pub source_id: ::prost::alloc::string::String,
-    #[prost(uint64, repeated, tag = "3")]
-    pub shard_ids: ::prost::alloc::vec::Vec<u64>,
-}
-#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)]
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
 pub struct GetOrCreateOpenShardsResponse {
     #[prost(message, repeated, tag = "1")]
     pub successes: ::prost::alloc::vec::Vec<GetOrCreateOpenShardsSuccess>,
diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.ingester.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.ingester.rs
index 8ceb73fc7ee..53857e8ac79 100644
--- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.ingester.rs
+++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.ingester.rs
@@ -194,10 +194,10 @@ pub struct ReplicateFailure {
     pub index_uid: ::prost::alloc::string::String,
     #[prost(string, tag = "3")]
     pub source_id: ::prost::alloc::string::String,
-    /// ingest.DocBatchV2 doc_batch = 4;
-    /// ingest.IngestError error = 5;
     #[prost(uint64, tag = "4")]
     pub shard_id: u64,
+    #[prost(enumeration = "ReplicateFailureReason", tag = "5")]
+    pub reason: i32,
 }
 #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)]
 #[allow(clippy::derive_partial_eq_without_eq)]
@@ -263,6 +263,17 @@ pub struct FetchResponseV2 {
 #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)]
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
+pub struct CloseShardsRequest {
+    #[prost(message, repeated, tag = "1")]
+    pub closed_shards: ::prost::alloc::vec::Vec<super::ClosedShards>,
+}
+#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct CloseShardsResponse {}
+#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
 pub struct PingRequest {
     #[prost(string, tag = "1")]
     pub leader_id: ::prost::alloc::string::String,
@@ -309,6 +320,43 @@ impl PersistFailureReason {
         }
     }
 }
+#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)]
+#[serde(rename_all = "snake_case")]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
+#[repr(i32)]
+pub enum ReplicateFailureReason {
+    Unspecified = 0,
+    ShardClosed = 1,
+    ResourceExhausted = 3,
+}
+impl ReplicateFailureReason {
+    /// String value of the enum field names used in the ProtoBuf definition.
+    ///
+    /// The values are not transformed in any way and thus are considered stable
+    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
+    pub fn as_str_name(&self) -> &'static str {
+        match self {
+            ReplicateFailureReason::Unspecified => "REPLICATE_FAILURE_REASON_UNSPECIFIED",
+            ReplicateFailureReason::ShardClosed => {
+                "REPLICATE_FAILURE_REASON_SHARD_CLOSED"
+            }
+            ReplicateFailureReason::ResourceExhausted => {
+                "REPLICATE_FAILURE_REASON_RESOURCE_EXHAUSTED"
+            }
+        }
+    }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "REPLICATE_FAILURE_REASON_UNSPECIFIED" => Some(Self::Unspecified),
+            "REPLICATE_FAILURE_REASON_SHARD_CLOSED" => Some(Self::ShardClosed),
+            "REPLICATE_FAILURE_REASON_RESOURCE_EXHAUSTED" => {
+                Some(Self::ResourceExhausted)
+            }
+            _ => None,
+        }
+    }
+}
 /// BEGIN quickwit-codegen
 #[allow(unused_imports)]
 use std::str::FromStr;
@@ -334,6 +382,10 @@ pub trait IngesterService: std::fmt::Debug + dyn_clone::DynClone + Send + Sync +
         &mut self,
         request: OpenFetchStreamRequest,
     ) -> crate::ingest::IngestV2Result<IngesterServiceStream<FetchResponseV2>>;
+    async fn close_shards(
+        &mut self,
+        request: CloseShardsRequest,
+    ) -> crate::ingest::IngestV2Result<CloseShardsResponse>;
     /// Pings an ingester to check if it is ready to host shards and serve requests.
     async fn ping(
         &mut self,
@@ -442,6 +494,12 @@ impl IngesterService for IngesterServiceClient {
     ) -> crate::ingest::IngestV2Result<IngesterServiceStream<FetchResponseV2>> {
         self.inner.open_fetch_stream(request).await
     }
+    async fn close_shards(
+        &mut self,
+        request: CloseShardsRequest,
+    ) -> crate::ingest::IngestV2Result<CloseShardsResponse> {
+        self.inner.close_shards(request).await
+    }
     async fn ping(
         &mut self,
         request: PingRequest,
@@ -486,6 +544,12 @@ pub mod ingester_service_mock {
         > {
             self.inner.lock().await.open_fetch_stream(request).await
         }
+        async fn close_shards(
+            &mut self,
+            request: super::CloseShardsRequest,
+        ) -> crate::ingest::IngestV2Result<super::CloseShardsResponse> {
+            self.inner.lock().await.close_shards(request).await
+        }
         async fn ping(
             &mut self,
             request: super::PingRequest,
@@ -563,6 +627,22 @@ impl tower::Service<OpenFetchStreamRequest> for Box<dyn IngesterService> {
         Box::pin(fut)
     }
 }
+impl tower::Service<CloseShardsRequest> for Box<dyn IngesterService> {
+    type Response = CloseShardsResponse;
+    type Error = crate::ingest::IngestV2Error;
+    type Future = BoxFuture<Self::Response, Self::Error>;
+    fn poll_ready(
+        &mut self,
+        _cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Result<(), Self::Error>> {
+        std::task::Poll::Ready(Ok(()))
+    }
+    fn call(&mut self, request: CloseShardsRequest) -> Self::Future {
+        let mut svc = self.clone();
+        let fut = async move { svc.close_shards(request).await };
+        Box::pin(fut)
+    }
+}
 impl tower::Service<PingRequest> for Box<dyn IngesterService> {
     type Response = PingResponse;
     type Error = crate::ingest::IngestV2Error;
@@ -614,6 +694,11 @@ struct IngesterServiceTowerBlock {
         IngesterServiceStream<FetchResponseV2>,
         crate::ingest::IngestV2Error,
     >,
+    close_shards_svc: quickwit_common::tower::BoxService<
+        CloseShardsRequest,
+        CloseShardsResponse,
+        crate::ingest::IngestV2Error,
+    >,
     ping_svc: quickwit_common::tower::BoxService<
         PingRequest,
         PingResponse,
@@ -632,6 +717,7 @@ impl Clone for IngesterServiceTowerBlock {
             persist_svc: self.persist_svc.clone(),
             open_replication_stream_svc: self.open_replication_stream_svc.clone(),
             open_fetch_stream_svc: self.open_fetch_stream_svc.clone(),
+            close_shards_svc: self.close_shards_svc.clone(),
             ping_svc: self.ping_svc.clone(),
             truncate_svc: self.truncate_svc.clone(),
         }
@@ -657,6 +743,12 @@ impl IngesterService for IngesterServiceTowerBlock {
     ) -> crate::ingest::IngestV2Result<IngesterServiceStream<FetchResponseV2>> {
         self.open_fetch_stream_svc.ready().await?.call(request).await
     }
+    async fn close_shards(
+        &mut self,
+        request: CloseShardsRequest,
+    ) -> crate::ingest::IngestV2Result<CloseShardsResponse> {
+        self.close_shards_svc.ready().await?.call(request).await
+    }
     async fn ping(
         &mut self,
         request: PingRequest,
@@ -700,6 +792,15 @@ pub struct IngesterServiceTowerBlockBuilder {
         >,
     >,
     #[allow(clippy::type_complexity)]
+    close_shards_layer: Option<
+        quickwit_common::tower::BoxLayer<
+            Box<dyn IngesterService>,
+            CloseShardsRequest,
+            CloseShardsResponse,
+            crate::ingest::IngestV2Error,
+        >,
+    >,
+    #[allow(clippy::type_complexity)]
     ping_layer: Option<
         quickwit_common::tower::BoxLayer<
             Box<dyn IngesterService>,
@@ -742,6 +843,12 @@ impl IngesterServiceTowerBlockBuilder {
                 Error = crate::ingest::IngestV2Error,
             > + Clone + Send + Sync + 'static,
         <L::Service as tower::Service<OpenFetchStreamRequest>>::Future: Send + 'static,
+        L::Service: tower::Service<
+                CloseShardsRequest,
+                Response = CloseShardsResponse,
+                Error = crate::ingest::IngestV2Error,
+            > + Clone + Send + Sync + 'static,
+        <L::Service as tower::Service<CloseShardsRequest>>::Future: Send + 'static,
         L::Service: tower::Service<
                 PingRequest,
                 Response = PingResponse,
@@ -764,6 +871,10 @@ impl IngesterServiceTowerBlockBuilder {
             .open_fetch_stream_layer = Some(
             quickwit_common::tower::BoxLayer::new(layer.clone()),
         );
+        self
+            .close_shards_layer = Some(
+            quickwit_common::tower::BoxLayer::new(layer.clone()),
+        );
         self.ping_layer = Some(quickwit_common::tower::BoxLayer::new(layer.clone()));
         self.truncate_layer = Some(quickwit_common::tower::BoxLayer::new(layer));
         self
@@ -815,6 +926,19 @@ impl IngesterServiceTowerBlockBuilder {
         );
         self
     }
+    pub fn close_shards_layer<L>(mut self, layer: L) -> Self
+    where
+        L: tower::Layer<Box<dyn IngesterService>> + Send + Sync + 'static,
+        L::Service: tower::Service<
+                CloseShardsRequest,
+                Response = CloseShardsResponse,
+                Error = crate::ingest::IngestV2Error,
+            > + Clone + Send + Sync + 'static,
+        <L::Service as tower::Service<CloseShardsRequest>>::Future: Send + 'static,
+    {
+        self.close_shards_layer = Some(quickwit_common::tower::BoxLayer::new(layer));
+        self
+    }
     pub fn ping_layer<L>(mut self, layer: L) -> Self
     where
         L: tower::Layer<Box<dyn IngesterService>> + Send + Sync + 'static,
@@ -895,6 +1019,11 @@ impl IngesterServiceTowerBlockBuilder {
         } else {
             quickwit_common::tower::BoxService::new(boxed_instance.clone())
         };
+        let close_shards_svc = if let Some(layer) = self.close_shards_layer {
+            layer.layer(boxed_instance.clone())
+        } else {
+            quickwit_common::tower::BoxService::new(boxed_instance.clone())
+        };
         let ping_svc = if let Some(layer) = self.ping_layer {
             layer.layer(boxed_instance.clone())
         } else {
@@ -910,6 +1039,7 @@ impl IngesterServiceTowerBlockBuilder {
             persist_svc,
             open_replication_stream_svc,
             open_fetch_stream_svc,
+            close_shards_svc,
             ping_svc,
             truncate_svc,
         };
@@ -1012,6 +1142,12 @@ where
                 crate::ingest::IngestV2Error,
             >,
         >
+        + tower::Service<
+            CloseShardsRequest,
+            Response = CloseShardsResponse,
+            Error = crate::ingest::IngestV2Error,
+            Future = BoxFuture<CloseShardsResponse, crate::ingest::IngestV2Error>,
+        >
         + tower::Service<
             PingRequest,
             Response = PingResponse,
@@ -1043,6 +1179,12 @@ where
     ) -> crate::ingest::IngestV2Result<IngesterServiceStream<FetchResponseV2>> {
         self.call(request).await
     }
+    async fn close_shards(
+        &mut self,
+        request: CloseShardsRequest,
+    ) -> crate::ingest::IngestV2Result<CloseShardsResponse> {
+        self.call(request).await
+    }
     async fn ping(
         &mut self,
         request: PingRequest,
@@ -1128,6 +1270,16 @@ where
             })
             .map_err(|error| error.into())
     }
+    async fn close_shards(
+        &mut self,
+        request: CloseShardsRequest,
+    ) -> crate::ingest::IngestV2Result<CloseShardsResponse> {
+        self.inner
+            .close_shards(request)
+            .await
+            .map(|response| response.into_inner())
+            .map_err(|error| error.into())
+    }
     async fn ping(
         &mut self,
         request: PingRequest,
@@ -1206,6 +1358,17 @@ for IngesterServiceGrpcServerAdapter {
             .map(|stream| tonic::Response::new(stream.map_err(|error| error.into())))
             .map_err(|error| error.into())
     }
+    async fn close_shards(
+        &self,
+        request: tonic::Request<CloseShardsRequest>,
+    ) -> Result<tonic::Response<CloseShardsResponse>, tonic::Status> {
+        self.inner
+            .clone()
+            .close_shards(request.into_inner())
+            .await
+            .map(tonic::Response::new)
+            .map_err(|error| error.into())
+    }
     async fn ping(
         &self,
         request: tonic::Request<PingRequest>,
@@ -1409,6 +1572,36 @@ pub mod ingester_service_grpc_client {
                 );
             self.inner.server_streaming(req, path, codec).await
         }
+        pub async fn close_shards(
+            &mut self,
+            request: impl tonic::IntoRequest<super::CloseShardsRequest>,
+        ) -> std::result::Result<
+            tonic::Response<super::CloseShardsResponse>,
+            tonic::Status,
+        > {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| {
+                    tonic::Status::new(
+                        tonic::Code::Unknown,
+                        format!("Service was not ready: {}", e.into()),
+                    )
+                })?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static(
+                "/quickwit.ingest.ingester.IngesterService/CloseShards",
+            );
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(
+                    GrpcMethod::new(
+                        "quickwit.ingest.ingester.IngesterService",
+                        "CloseShards",
+                    ),
+                );
+            self.inner.unary(req, path, codec).await
+        }
         /// Pings an ingester to check if it is ready to host shards and serve requests.
         pub async fn ping(
             &mut self,
@@ -1507,6 +1700,13 @@ pub mod ingester_service_grpc_server {
             tonic::Response<Self::OpenFetchStreamStream>,
             tonic::Status,
         >;
+        async fn close_shards(
+            &self,
+            request: tonic::Request<super::CloseShardsRequest>,
+        ) -> std::result::Result<
+            tonic::Response<super::CloseShardsResponse>,
+            tonic::Status,
+        >;
         /// Pings an ingester to check if it is ready to host shards and serve requests.
         async fn ping(
             &self,
@@ -1741,6 +1941,52 @@ pub mod ingester_service_grpc_server {
                     };
                     Box::pin(fut)
                 }
+                "/quickwit.ingest.ingester.IngesterService/CloseShards" => {
+                    #[allow(non_camel_case_types)]
+                    struct CloseShardsSvc<T: IngesterServiceGrpc>(pub Arc<T>);
+                    impl<
+                        T: IngesterServiceGrpc,
+                    > tonic::server::UnaryService<super::CloseShardsRequest>
+                    for CloseShardsSvc<T> {
+                        type Response = super::CloseShardsResponse;
+                        type Future = BoxFuture<
+                            tonic::Response<Self::Response>,
+                            tonic::Status,
+                        >;
+                        fn call(
+                            &mut self,
+                            request: tonic::Request<super::CloseShardsRequest>,
+                        ) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move {
+                                (*inner).close_shards(request).await
+                            };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let inner = inner.0;
+                        let method = CloseShardsSvc(inner);
+                        let codec = tonic::codec::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(
+                                accept_compression_encodings,
+                                send_compression_encodings,
+                            )
+                            .apply_max_message_size_config(
+                                max_decoding_message_size,
+                                max_encoding_message_size,
+                            );
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
                 "/quickwit.ingest.ingester.IngesterService/Ping" => {
                     #[allow(non_camel_case_types)]
                     struct PingSvc<T: IngesterServiceGrpc>(pub Arc<T>);
diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.rs
index e339aad99a0..6ad8e0e259d 100644
--- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.rs
+++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.rs
@@ -52,6 +52,17 @@ pub struct Shard {
     pub publish_token: ::core::option::Option<::prost::alloc::string::String>,
 }
 #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ClosedShards {
+    #[prost(string, tag = "1")]
+    pub index_uid: ::prost::alloc::string::String,
+    #[prost(string, tag = "2")]
+    pub source_id: ::prost::alloc::string::String,
+    #[prost(uint64, repeated, tag = "3")]
+    pub shard_ids: ::prost::alloc::vec::Vec<u64>,
+}
+#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)]
 #[serde(rename_all = "snake_case")]
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
 #[repr(i32)]
diff --git a/quickwit/quickwit-proto/src/indexing/mod.rs b/quickwit/quickwit-proto/src/indexing/mod.rs
index 4d380315f53..f55c4a83ebd 100644
--- a/quickwit/quickwit-proto/src/indexing/mod.rs
+++ b/quickwit/quickwit-proto/src/indexing/mod.rs
@@ -17,8 +17,9 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 
-use std::fmt::Formatter;
+use std::fmt::{Display, Formatter};
 use std::hash::Hash;
+use std::ops::{Add, Mul, Sub};
 use std::{fmt, io};
 
 use anyhow::anyhow;
@@ -150,13 +151,13 @@ pub struct IndexingPipelineId {
     pub pipeline_ord: usize,
 }
 
-impl fmt::Display for IndexingPipelineId {
+impl Display for IndexingPipelineId {
     fn fmt(&self, f: &mut Formatter) -> fmt::Result {
         write!(f, "{}:{}", self.index_uid, &self.source_id)
     }
 }
 
-impl fmt::Display for IndexingTask {
+impl Display for IndexingTask {
     fn fmt(&self, f: &mut Formatter) -> fmt::Result {
         write!(f, "{}:{}", self.index_uid, &self.source_id)
     }
@@ -209,13 +210,146 @@ impl TryFrom<&str> for IndexingTask {
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, utoipa::ToSchema)]
 pub struct PipelineMetrics {
-    pub cpu_millis: u16,
+    pub cpu_millis: CpuCapacity,
     pub throughput_mb_per_sec: u16,
 }
 
-impl fmt::Display for PipelineMetrics {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}m,{}MB/s", self.cpu_millis, self.throughput_mb_per_sec)
+impl Display for PipelineMetrics {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        write!(f, "{},{}MB/s", self.cpu_millis, self.throughput_mb_per_sec)
+    }
+}
+
+/// One full pipeline (including merging) is assumed to consume 4 CPU threads.
+/// The actual number somewhere between 3 and 4.
+pub const PIPELINE_FULL_CAPACITY: CpuCapacity = CpuCapacity::from_cpu_millis(4_000u32);
+
+/// The CpuCapacity represents an amount of CPU resource available.
+///
+/// It is usually expressed in CPU millis (For instance, one full CPU thread is
+/// displayed as `1000m`).
+#[derive(
+    Copy, Clone, Debug, Eq, PartialEq, Deserialize, Serialize, Ord, PartialOrd, utoipa::ToSchema,
+)]
+#[serde(
+    into = "CpuCapacityForSerialization",
+    try_from = "CpuCapacityForSerialization"
+)]
+pub struct CpuCapacity(u32);
+
+/// Short helper function to build `CpuCapacity`.
+#[inline(always)]
+pub const fn mcpu(milli_cpus: u32) -> CpuCapacity {
+    CpuCapacity::from_cpu_millis(milli_cpus)
+}
+
+impl CpuCapacity {
+    #[inline(always)]
+    pub const fn from_cpu_millis(cpu_millis: u32) -> CpuCapacity {
+        CpuCapacity(cpu_millis)
+    }
+
+    #[inline(always)]
+    pub fn cpu_millis(self) -> u32 {
+        self.0
+    }
+
+    #[inline(always)]
+    pub fn zero() -> CpuCapacity {
+        CpuCapacity::from_cpu_millis(0u32)
+    }
+
+    #[inline(always)]
+    pub fn one_cpu_thread() -> CpuCapacity {
+        CpuCapacity::from_cpu_millis(1_000u32)
+    }
+}
+
+impl Sub<CpuCapacity> for CpuCapacity {
+    type Output = CpuCapacity;
+
+    #[inline(always)]
+    fn sub(self, rhs: CpuCapacity) -> Self::Output {
+        CpuCapacity::from_cpu_millis(self.0 - rhs.0)
+    }
+}
+
+impl Add<CpuCapacity> for CpuCapacity {
+    type Output = CpuCapacity;
+
+    #[inline(always)]
+    fn add(self, rhs: CpuCapacity) -> Self::Output {
+        CpuCapacity::from_cpu_millis(self.0 + rhs.0)
+    }
+}
+
+impl Mul<u32> for CpuCapacity {
+    type Output = CpuCapacity;
+
+    #[inline(always)]
+    fn mul(self, rhs: u32) -> CpuCapacity {
+        CpuCapacity::from_cpu_millis(self.0 * rhs)
+    }
+}
+
+impl Mul<f32> for CpuCapacity {
+    type Output = CpuCapacity;
+
+    #[inline(always)]
+    fn mul(self, scale: f32) -> CpuCapacity {
+        CpuCapacity::from_cpu_millis((self.0 as f32 * scale) as u32)
+    }
+}
+
+impl Display for CpuCapacity {
+    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
+        write!(f, "{}m", self.0)
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(untagged)]
+enum CpuCapacityForSerialization {
+    Float(f32),
+    MilliCpuWithUnit(String),
+}
+
+impl TryFrom<CpuCapacityForSerialization> for CpuCapacity {
+    type Error = String;
+
+    fn try_from(
+        cpu_capacity_for_serialization: CpuCapacityForSerialization,
+    ) -> Result<CpuCapacity, Self::Error> {
+        match cpu_capacity_for_serialization {
+            CpuCapacityForSerialization::Float(cpu_capacity) => {
+                Ok(CpuCapacity((cpu_capacity * 1000.0f32) as u32))
+            }
+            CpuCapacityForSerialization::MilliCpuWithUnit(cpu_capacity_str) => {
+                Self::from_str(&cpu_capacity_str)
+            }
+        }
+    }
+}
+
+impl FromStr for CpuCapacity {
+    type Err = String;
+
+    fn from_str(cpu_capacity_str: &str) -> Result<Self, Self::Err> {
+        let Some(milli_cpus_without_unit_str) = cpu_capacity_str.strip_suffix('m') else {
+            return Err(format!(
+                "invalid cpu capacity: `{cpu_capacity_str}`. String format expects a trailing 'm'."
+            ));
+        };
+        let milli_cpus: u32 = milli_cpus_without_unit_str
+            .parse::<u32>()
+            .map_err(|_err| format!("invalid cpu capacity: `{cpu_capacity_str}`."))?;
+        Ok(CpuCapacity(milli_cpus))
+    }
+}
+
+impl From<CpuCapacity> for CpuCapacityForSerialization {
+    fn from(cpu_capacity: CpuCapacity) -> CpuCapacityForSerialization {
+        CpuCapacityForSerialization::MilliCpuWithUnit(format!("{}m", cpu_capacity.0))
     }
 }
 
@@ -223,6 +357,37 @@ impl fmt::Display for PipelineMetrics {
 mod tests {
     use super::*;
 
+    #[test]
+    fn test_cpu_capacity_serialization() {
+        assert_eq!(CpuCapacity::from_str("2000m").unwrap(), mcpu(2000));
+        assert_eq!(CpuCapacity::from_cpu_millis(2500), mcpu(2500));
+        assert_eq!(
+            CpuCapacity::from_str("2.5").unwrap_err(),
+            "invalid cpu capacity: `2.5`. String format expects a trailing 'm'."
+        );
+        assert_eq!(
+            serde_json::from_value::<CpuCapacity>(serde_json::Value::String("1200m".to_string()))
+                .unwrap(),
+            mcpu(1200)
+        );
+        assert_eq!(
+            serde_json::from_value::<CpuCapacity>(serde_json::Value::Number(
+                serde_json::Number::from_f64(1.2f64).unwrap()
+            ))
+            .unwrap(),
+            mcpu(1200)
+        );
+        assert_eq!(
+            serde_json::from_value::<CpuCapacity>(serde_json::Value::Number(
+                serde_json::Number::from(1u32)
+            ))
+            .unwrap(),
+            mcpu(1000)
+        );
+        assert_eq!(CpuCapacity::from_cpu_millis(2500).to_string(), "2500m");
+        assert_eq!(serde_json::to_string(&mcpu(2500)).unwrap(), "\"2500m\"");
+    }
+
     #[test]
     fn test_indexing_task_serialization() {
         let original = IndexingTask {
diff --git a/quickwit/quickwit-proto/src/ingest/mod.rs b/quickwit/quickwit-proto/src/ingest/mod.rs
index ef5f54ec368..2563cba0394 100644
--- a/quickwit/quickwit-proto/src/ingest/mod.rs
+++ b/quickwit/quickwit-proto/src/ingest/mod.rs
@@ -23,7 +23,7 @@ use self::ingester::FetchResponseV2;
 use super::types::NodeId;
 use super::{ServiceError, ServiceErrorCode};
 use crate::control_plane::ControlPlaneError;
-use crate::types::{queue_id, Position};
+use crate::types::{queue_id, Position, QueueId};
 
 pub mod ingester;
 pub mod router;
@@ -192,3 +192,11 @@ impl ShardState {
         *self == ShardState::Closed
     }
 }
+
+impl ClosedShards {
+    pub fn queue_ids(&self) -> impl Iterator<Item = QueueId> + '_ {
+        self.shard_ids
+            .iter()
+            .map(|shard_id| queue_id(&self.index_uid, &self.source_id, *shard_id))
+    }
+}
diff --git a/quickwit/quickwit-query/benches/multilang_tokenizers_bench.rs b/quickwit/quickwit-query/benches/multilang_tokenizers_bench.rs
index 0f4a5b7496f..705cbbb3520 100644
--- a/quickwit/quickwit-query/benches/multilang_tokenizers_bench.rs
+++ b/quickwit/quickwit-query/benches/multilang_tokenizers_bench.rs
@@ -69,9 +69,11 @@ fn process_tokens(analyzer: &mut TextAnalyzer, text: &str) -> Vec<Token> {
 pub fn tokenizers_throughput_benchmark(c: &mut Criterion) {
     let mut group = c.benchmark_group("multilang");
     let tokenizer_manager = create_default_quickwit_tokenizer_manager();
-    let mut default_tokenizer = tokenizer_manager.get("default").unwrap();
-    let mut multilang_tokenizer = tokenizer_manager.get("multilang").unwrap();
-    let mut chinese_tokenizer = tokenizer_manager.get("chinese_compatible").unwrap();
+    let mut default_tokenizer = tokenizer_manager.get_tokenizer("default").unwrap();
+    let mut multilang_tokenizer = tokenizer_manager.get_tokenizer("multilang").unwrap();
+    let mut chinese_tokenizer = tokenizer_manager
+        .get_tokenizer("chinese_compatible")
+        .unwrap();
 
     group
         .throughput(Throughput::Bytes(ASCII_SHORT.len() as u64))
diff --git a/quickwit/quickwit-query/src/query_ast/bool_query.rs b/quickwit/quickwit-query/src/query_ast/bool_query.rs
index 30ec9cac473..9394aaa030a 100644
--- a/quickwit/quickwit-query/src/query_ast/bool_query.rs
+++ b/quickwit/quickwit-query/src/query_ast/bool_query.rs
@@ -19,10 +19,10 @@
 
 use serde::{Deserialize, Serialize};
 use tantivy::schema::Schema as TantivySchema;
-use tantivy::tokenizer::TokenizerManager;
 
 use super::{BuildTantivyAst, TantivyQueryAst};
 use crate::query_ast::QueryAst;
+use crate::tokenizers::TokenizerManager;
 use crate::InvalidQuery;
 
 /// # Unsupported features
diff --git a/quickwit/quickwit-query/src/query_ast/field_presence.rs b/quickwit/quickwit-query/src/query_ast/field_presence.rs
index be68c6a8d38..0f5eaabd070 100644
--- a/quickwit/quickwit-query/src/query_ast/field_presence.rs
+++ b/quickwit/quickwit-query/src/query_ast/field_presence.rs
@@ -21,11 +21,11 @@ use quickwit_common::shared_consts::FIELD_PRESENCE_FIELD_NAME;
 use quickwit_common::PathHasher;
 use serde::{Deserialize, Serialize};
 use tantivy::schema::{Field, IndexRecordOption, Schema as TantivySchema};
-use tantivy::tokenizer::TokenizerManager;
 use tantivy::Term;
 
 use crate::query_ast::tantivy_query_ast::TantivyQueryAst;
 use crate::query_ast::{BuildTantivyAst, QueryAst};
+use crate::tokenizers::TokenizerManager;
 use crate::{find_field_or_hit_dynamic, InvalidQuery};
 
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
diff --git a/quickwit/quickwit-query/src/query_ast/full_text_query.rs b/quickwit/quickwit-query/src/query_ast/full_text_query.rs
index be2e74dba43..a3ca2929caa 100644
--- a/quickwit/quickwit-query/src/query_ast/full_text_query.rs
+++ b/quickwit/quickwit-query/src/query_ast/full_text_query.rs
@@ -28,12 +28,13 @@ use tantivy::schema::{
     Field, FieldType, IndexRecordOption, JsonObjectOptions, Schema as TantivySchema,
     TextFieldIndexing,
 };
-use tantivy::tokenizer::{TextAnalyzer, TokenStream, TokenizerManager};
+use tantivy::tokenizer::{TextAnalyzer, TokenStream};
 use tantivy::Term;
 
 use crate::query_ast::tantivy_query_ast::{TantivyBoolQuery, TantivyQueryAst};
 use crate::query_ast::utils::full_text_query;
 use crate::query_ast::{BuildTantivyAst, QueryAst};
+use crate::tokenizers::TokenizerManager;
 use crate::{find_field_or_hit_dynamic, BooleanOperand, InvalidQuery, MatchAllOrNone};
 
 #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Clone)]
@@ -59,7 +60,7 @@ impl FullTextParams {
             .as_deref()
             .unwrap_or(text_field_indexing.tokenizer());
         tokenizer_manager
-            .get(tokenizer_name)
+            .get_tokenizer(tokenizer_name)
             .with_context(|| format!("no tokenizer named `{}` is registered", tokenizer_name))
     }
 
@@ -182,6 +183,8 @@ pub enum FullTextMode {
     },
     BoolPrefix {
         operator: BooleanOperand,
+        // max_expansions correspond to the fuzzy stop of query evalution. It's not the same as the
+        // max_expansions of a PhrasePrefixQuery, where it's used for the range expansion.
         max_expansions: u32,
     },
     // Act as Phrase with slop 0 if the field has positions,
@@ -255,11 +258,15 @@ impl FullTextQuery {
     ///
     /// This strange method is used to identify which term range should be warmed up for
     /// phrase prefix queries.
-    pub fn get_last_term(
+    pub fn get_prefix_term(
         &self,
         schema: &TantivySchema,
         tokenizer_manager: &TokenizerManager,
     ) -> Option<Term> {
+        if !matches!(self.params.mode, FullTextMode::BoolPrefix { .. }) {
+            return None;
+        };
+
         let (field, field_entry, json_path) =
             find_field_or_hit_dynamic(&self.field, schema).ok()?;
         let field_type: &FieldType = field_entry.field_type();
diff --git a/quickwit/quickwit-query/src/query_ast/mod.rs b/quickwit/quickwit-query/src/query_ast/mod.rs
index 50696d6a1aa..e815248b16f 100644
--- a/quickwit/quickwit-query/src/query_ast/mod.rs
+++ b/quickwit/quickwit-query/src/query_ast/mod.rs
@@ -20,7 +20,8 @@
 use serde::{Deserialize, Serialize};
 use tantivy::query::BoostQuery as TantivyBoostQuery;
 use tantivy::schema::Schema as TantivySchema;
-use tantivy::tokenizer::TokenizerManager;
+
+use crate::tokenizers::TokenizerManager;
 
 mod bool_query;
 mod field_presence;
@@ -33,6 +34,7 @@ mod term_set_query;
 mod user_input_query;
 pub(crate) mod utils;
 mod visitor;
+mod wildcard_query;
 
 pub use bool_query::BoolQuery;
 pub use field_presence::FieldPresenceQuery;
@@ -44,6 +46,7 @@ pub use term_query::TermQuery;
 pub use term_set_query::TermSetQuery;
 pub use user_input_query::UserInputQuery;
 pub use visitor::QueryAstVisitor;
+pub use wildcard_query::WildcardQuery;
 
 use crate::{BooleanOperand, InvalidQuery, NotNaNf32};
 
@@ -59,6 +62,7 @@ pub enum QueryAst {
     PhrasePrefix(PhrasePrefixQuery),
     Range(RangeQuery),
     UserInput(UserInputQuery),
+    Wildcard(WildcardQuery),
     MatchAll,
     MatchNone,
     Boost {
@@ -98,7 +102,8 @@ impl QueryAst {
             | ast @ QueryAst::MatchAll
             | ast @ QueryAst::MatchNone
             | ast @ QueryAst::FieldPresence(_)
-            | ast @ QueryAst::Range(_) => Ok(ast),
+            | ast @ QueryAst::Range(_)
+            | ast @ QueryAst::Wildcard(_) => Ok(ast),
             QueryAst::UserInput(user_text_query) => {
                 user_text_query.parse_user_query(default_search_fields)
             }
@@ -236,6 +241,12 @@ impl BuildTantivyAst for QueryAst {
                 search_fields,
                 with_validation,
             ),
+            QueryAst::Wildcard(wildcard) => wildcard.build_tantivy_ast_call(
+                schema,
+                tokenizer_manager,
+                search_fields,
+                with_validation,
+            ),
         }
     }
 }
diff --git a/quickwit/quickwit-query/src/query_ast/phrase_prefix_query.rs b/quickwit/quickwit-query/src/query_ast/phrase_prefix_query.rs
index ace1f2b1ad8..c7d67ea0617 100644
--- a/quickwit/quickwit-query/src/query_ast/phrase_prefix_query.rs
+++ b/quickwit/quickwit-query/src/query_ast/phrase_prefix_query.rs
@@ -20,11 +20,11 @@
 use serde::{Deserialize, Serialize};
 use tantivy::query::PhrasePrefixQuery as TantivyPhrasePrefixQuery;
 use tantivy::schema::{Field, FieldType, Schema as TantivySchema};
-use tantivy::tokenizer::TokenizerManager;
 use tantivy::Term;
 
 use crate::query_ast::tantivy_query_ast::TantivyQueryAst;
 use crate::query_ast::{BuildTantivyAst, FullTextParams, QueryAst};
+use crate::tokenizers::TokenizerManager;
 use crate::{find_field_or_hit_dynamic, InvalidQuery};
 
 /// The PhraseQuery node is meant to be tokenized and searched.
@@ -57,20 +57,19 @@ impl PhrasePrefixQuery {
                         field_entry.name()
                     ))
                 })?;
-                if !text_field_indexing.index_option().has_positions() {
-                    return Err(InvalidQuery::SchemaError(
-                        "trying to run a phrase prefix query on a field which does not have \
-                         positions indexed"
-                            .to_string(),
-                    ));
-                }
-
                 let terms = self.params.tokenize_text_into_terms(
                     field,
                     &self.phrase,
                     text_field_indexing,
                     tokenizer_manager,
                 )?;
+                if !text_field_indexing.index_option().has_positions() && terms.len() > 1 {
+                    return Err(InvalidQuery::SchemaError(
+                        "trying to run a phrase prefix query on a field which does not have \
+                         positions indexed"
+                            .to_string(),
+                    ));
+                }
                 Ok((field, terms))
             }
             FieldType::JsonObject(json_options) => {
@@ -81,13 +80,6 @@ impl PhrasePrefixQuery {
                             field_entry.name()
                         ))
                     })?;
-                if !text_field_indexing.index_option().has_positions() {
-                    return Err(InvalidQuery::SchemaError(
-                        "trying to run a PhrasePrefix query on a field which does not have \
-                         positions indexed"
-                            .to_string(),
-                    ));
-                }
                 let terms = self.params.tokenize_text_into_terms_json(
                     field,
                     json_path,
@@ -95,6 +87,13 @@ impl PhrasePrefixQuery {
                     json_options,
                     tokenizer_manager,
                 )?;
+                if !text_field_indexing.index_option().has_positions() && terms.len() > 1 {
+                    return Err(InvalidQuery::SchemaError(
+                        "trying to run a PhrasePrefix query on a field which does not have \
+                         positions indexed"
+                            .to_string(),
+                    ));
+                }
                 Ok((field, terms))
             }
             _ => Err(InvalidQuery::SchemaError(
diff --git a/quickwit/quickwit-query/src/query_ast/range_query.rs b/quickwit/quickwit-query/src/query_ast/range_query.rs
index a053675821a..16ada6047fe 100644
--- a/quickwit/quickwit-query/src/query_ast/range_query.rs
+++ b/quickwit/quickwit-query/src/query_ast/range_query.rs
@@ -24,12 +24,13 @@ use tantivy::query::{
     FastFieldRangeWeight as TantivyFastFieldRangeQuery, RangeQuery as TantivyRangeQuery,
 };
 use tantivy::schema::Schema as TantivySchema;
-use tantivy::tokenizer::TokenizerManager;
+use tantivy::DateTime;
 
 use super::QueryAst;
 use crate::json_literal::InterpretUserInput;
 use crate::query_ast::tantivy_query_ast::{TantivyBoolQuery, TantivyQueryAst};
 use crate::query_ast::BuildTantivyAst;
+use crate::tokenizers::TokenizerManager;
 use crate::{InvalidQuery, JsonLiteral};
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
@@ -266,11 +267,19 @@ impl BuildTantivyAst for RangeQuery {
                     field_name: field_entry.name().to_string(),
                 });
             }
-            tantivy::schema::FieldType::Date(_) => {
+            tantivy::schema::FieldType::Date(date_options) => {
                 let (lower_bound, upper_bound) =
                     convert_bounds(&self.lower_bound, &self.upper_bound, field_entry.name())?;
-                TantivyRangeQuery::new_date_bounds(self.field.clone(), lower_bound, upper_bound)
-                    .into()
+                let truncate_datetime =
+                    |date: &DateTime| date.truncate(date_options.get_precision());
+                let truncated_lower_bound = map_bound(&lower_bound, truncate_datetime);
+                let truncated_upper_bound = map_bound(&upper_bound, truncate_datetime);
+                TantivyFastFieldRangeQuery::new::<DateTime>(
+                    self.field.clone(),
+                    truncated_lower_bound,
+                    truncated_upper_bound,
+                )
+                .into()
             }
             tantivy::schema::FieldType::Facet(_) => {
                 return Err(InvalidQuery::RangeQueryNotSupportedForField {
@@ -333,11 +342,20 @@ impl BuildTantivyAst for RangeQuery {
     }
 }
 
+fn map_bound<TFrom, TTo>(bound: &Bound<TFrom>, transform: impl Fn(&TFrom) -> TTo) -> Bound<TTo> {
+    match bound {
+        Bound::Excluded(ref from_val) => Bound::Excluded(transform(from_val)),
+        Bound::Included(ref from_val) => Bound::Included(transform(from_val)),
+        Bound::Unbounded => Bound::Unbounded,
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::ops::Bound;
 
     use tantivy::schema::{Schema, FAST, STORED, TEXT};
+    use tantivy::DateOptions;
 
     use super::RangeQuery;
     use crate::query_ast::tantivy_query_ast::TantivyBoolQuery;
@@ -352,6 +370,10 @@ mod tests {
         schema_builder.add_u64_field("my_u64_field", FAST);
         schema_builder.add_f64_field("my_f64_field", FAST);
         schema_builder.add_text_field("my_str_field", FAST);
+        let date_options = DateOptions::default()
+            .set_fast()
+            .set_precision(tantivy::DateTimePrecision::Milliseconds);
+        schema_builder.add_date_field("my_date_field", date_options);
         schema_builder.add_u64_field("my_u64_not_fastfield", STORED);
         if dynamic_mode {
             schema_builder.add_json_field("_dynamic", TEXT | STORED | FAST);
@@ -359,12 +381,17 @@ mod tests {
         schema_builder.build()
     }
 
-    fn test_range_query_typed_field_util(field: &str, expected: &str) {
+    fn test_range_query_typed_field_util(
+        field: &str,
+        lower_value: JsonLiteral,
+        upper_value: JsonLiteral,
+        expected: &str,
+    ) {
         let schema = make_schema(false);
         let range_query = RangeQuery {
             field: field.to_string(),
-            lower_bound: Bound::Included(JsonLiteral::String("1980".to_string())),
-            upper_bound: Bound::Included(JsonLiteral::String("1989".to_string())),
+            lower_bound: Bound::Included(lower_value),
+            upper_bound: Bound::Included(upper_value),
         };
         let tantivy_ast = range_query
             .build_tantivy_ast_call(
@@ -384,17 +411,23 @@ mod tests {
     fn test_range_query_typed_field() {
         test_range_query_typed_field_util(
             "my_i64_field",
+            JsonLiteral::String("1980".to_string()),
+            JsonLiteral::String("1989".to_string()),
             "FastFieldRangeWeight { field: \"my_i64_field\", lower_bound: \
              Included(9223372036854777788), upper_bound: Included(9223372036854777797), \
              column_type_opt: Some(I64) }",
         );
         test_range_query_typed_field_util(
             "my_u64_field",
+            JsonLiteral::String("1980".to_string()),
+            JsonLiteral::String("1989".to_string()),
             "FastFieldRangeWeight { field: \"my_u64_field\", lower_bound: Included(1980), \
              upper_bound: Included(1989), column_type_opt: Some(U64) }",
         );
         test_range_query_typed_field_util(
             "my_f64_field",
+            JsonLiteral::String("1980".to_string()),
+            JsonLiteral::String("1989".to_string()),
             "FastFieldRangeWeight { field: \"my_f64_field\", lower_bound: \
              Included(13879794984393113600), upper_bound: Included(13879834566811713536), \
              column_type_opt: Some(F64) }",
diff --git a/quickwit/quickwit-query/src/query_ast/term_query.rs b/quickwit/quickwit-query/src/query_ast/term_query.rs
index 0f62da9544d..cb1129fc69f 100644
--- a/quickwit/quickwit-query/src/query_ast/term_query.rs
+++ b/quickwit/quickwit-query/src/query_ast/term_query.rs
@@ -21,10 +21,10 @@ use std::collections::HashMap;
 
 use serde::{Deserialize, Serialize};
 use tantivy::schema::Schema as TantivySchema;
-use tantivy::tokenizer::TokenizerManager;
 
 use super::{BuildTantivyAst, QueryAst};
 use crate::query_ast::{FullTextParams, TantivyQueryAst};
+use crate::tokenizers::TokenizerManager;
 use crate::{BooleanOperand, InvalidQuery};
 
 /// The TermQuery acts exactly like a FullTextQuery with
diff --git a/quickwit/quickwit-query/src/query_ast/term_set_query.rs b/quickwit/quickwit-query/src/query_ast/term_set_query.rs
index d27597e37bc..b9c93ba36e5 100644
--- a/quickwit/quickwit-query/src/query_ast/term_set_query.rs
+++ b/quickwit/quickwit-query/src/query_ast/term_set_query.rs
@@ -21,10 +21,10 @@ use std::collections::{BTreeSet, HashMap, HashSet};
 
 use serde::{Deserialize, Serialize};
 use tantivy::schema::Schema as TantivySchema;
-use tantivy::tokenizer::TokenizerManager;
 use tantivy::Term;
 
 use crate::query_ast::{BuildTantivyAst, QueryAst, TantivyQueryAst, TermQuery};
+use crate::tokenizers::TokenizerManager;
 use crate::InvalidQuery;
 
 /// TermSetQuery matches the same document set as if it was a union of
diff --git a/quickwit/quickwit-query/src/query_ast/user_input_query.rs b/quickwit/quickwit-query/src/query_ast/user_input_query.rs
index cab0d9cea9a..910e37ae254 100644
--- a/quickwit/quickwit-query/src/query_ast/user_input_query.rs
+++ b/quickwit/quickwit-query/src/query_ast/user_input_query.rs
@@ -26,13 +26,13 @@ use tantivy::query_grammar::{
     Delimiter, Occur, UserInputAst, UserInputBound, UserInputLeaf, UserInputLiteral,
 };
 use tantivy::schema::Schema as TantivySchema;
-use tantivy::tokenizer::TokenizerManager;
 
 use crate::not_nan_f32::NotNaNf32;
 use crate::query_ast::tantivy_query_ast::TantivyQueryAst;
 use crate::query_ast::{
     self, BuildTantivyAst, FieldPresenceQuery, FullTextMode, FullTextParams, QueryAst,
 };
+use crate::tokenizers::TokenizerManager;
 use crate::{BooleanOperand, InvalidQuery, JsonLiteral};
 
 const DEFAULT_PHRASE_QUERY_MAX_EXPANSION: u32 = 50;
@@ -182,6 +182,31 @@ fn convert_user_input_ast_to_query_ast(
     }
 }
 
+fn is_wildcard(phrase: &str) -> bool {
+    use std::ops::ControlFlow;
+    enum State {
+        Normal,
+        Escaped,
+    }
+
+    phrase
+        .chars()
+        .try_fold(State::Normal, |state, c| match state {
+            State::Escaped => ControlFlow::Continue(State::Normal),
+            State::Normal => {
+                if c == '*' || c == '?' {
+                    // we are in a wildcard query
+                    ControlFlow::Break(())
+                } else if c == '\\' {
+                    ControlFlow::Continue(State::Escaped)
+                } else {
+                    ControlFlow::Continue(State::Normal)
+                }
+            }
+        })
+        .is_break()
+}
+
 fn convert_user_input_literal(
     user_input_literal: UserInputLiteral,
     default_search_fields: &[String],
@@ -216,24 +241,32 @@ fn convert_user_input_literal(
         mode,
         zero_terms_query: crate::MatchAllOrNone::MatchNone,
     };
+    let wildcard = delimiter == Delimiter::None && is_wildcard(&phrase);
     let mut phrase_queries: Vec<QueryAst> = field_names
         .into_iter()
         .map(|field_name| {
             if prefix {
-                return query_ast::PhrasePrefixQuery {
+                query_ast::PhrasePrefixQuery {
                     field: field_name,
                     phrase: phrase.clone(),
                     params: full_text_params.clone(),
                     max_expansions: DEFAULT_PHRASE_QUERY_MAX_EXPANSION,
                 }
-                .into();
-            }
-            query_ast::FullTextQuery {
-                field: field_name,
-                text: phrase.clone(),
-                params: full_text_params.clone(),
+                .into()
+            } else if wildcard {
+                query_ast::WildcardQuery {
+                    field: field_name,
+                    value: phrase.clone(),
+                }
+                .into()
+            } else {
+                query_ast::FullTextQuery {
+                    field: field_name,
+                    text: phrase.clone(),
+                    params: full_text_params.clone(),
+                }
+                .into()
             }
-            .into()
         })
         .collect();
     if phrase_queries.is_empty() {
diff --git a/quickwit/quickwit-query/src/query_ast/utils.rs b/quickwit/quickwit-query/src/query_ast/utils.rs
index 8033424c158..b6be400f393 100644
--- a/quickwit/quickwit-query/src/query_ast/utils.rs
+++ b/quickwit/quickwit-query/src/query_ast/utils.rs
@@ -23,11 +23,12 @@ use tantivy::schema::{
     Field, FieldEntry, FieldType, IndexRecordOption, JsonObjectOptions, Schema as TantivySchema,
     Type,
 };
-use tantivy::{tokenizer, Term};
+use tantivy::Term;
 
 use crate::json_literal::InterpretUserInput;
 use crate::query_ast::full_text_query::FullTextParams;
 use crate::query_ast::tantivy_query_ast::{TantivyBoolQuery, TantivyQueryAst};
+use crate::tokenizers::TokenizerManager;
 use crate::InvalidQuery;
 
 const DYNAMIC_FIELD_NAME: &str = "_dynamic";
@@ -75,7 +76,7 @@ pub(crate) fn full_text_query(
     text_query: &str,
     full_text_params: &FullTextParams,
     schema: &TantivySchema,
-    tokenizer_manager: &tokenizer::TokenizerManager,
+    tokenizer_manager: &TokenizerManager,
 ) -> Result<TantivyQueryAst, InvalidQuery> {
     let (field, field_entry, path) = find_field_or_hit_dynamic(full_path, schema)?;
     compute_query_with_field(
@@ -108,7 +109,7 @@ fn compute_query_with_field(
     json_path: &str,
     value: &str,
     full_text_params: &FullTextParams,
-    tokenizer_manager: &tokenizer::TokenizerManager,
+    tokenizer_manager: &TokenizerManager,
 ) -> Result<TantivyQueryAst, InvalidQuery> {
     let field_type = field_entry.field_type();
     match field_type {
@@ -182,7 +183,7 @@ fn compute_tantivy_ast_query_for_json(
     text: &str,
     full_text_params: &FullTextParams,
     json_options: &JsonObjectOptions,
-    tokenizer_manager: &tokenizer::TokenizerManager,
+    tokenizer_manager: &TokenizerManager,
 ) -> Result<TantivyQueryAst, InvalidQuery> {
     let mut bool_query = TantivyBoolQuery::default();
     let mut term = Term::with_capacity(100);
diff --git a/quickwit/quickwit-query/src/query_ast/visitor.rs b/quickwit/quickwit-query/src/query_ast/visitor.rs
index 398f8d9852d..8c60153ac9d 100644
--- a/quickwit/quickwit-query/src/query_ast/visitor.rs
+++ b/quickwit/quickwit-query/src/query_ast/visitor.rs
@@ -22,6 +22,7 @@ use crate::query_ast::field_presence::FieldPresenceQuery;
 use crate::query_ast::user_input_query::UserInputQuery;
 use crate::query_ast::{
     BoolQuery, FullTextQuery, PhrasePrefixQuery, QueryAst, RangeQuery, TermQuery, TermSetQuery,
+    WildcardQuery,
 };
 
 /// Simple trait to implement a Visitor over the QueryAst.
@@ -43,6 +44,7 @@ pub trait QueryAstVisitor<'a> {
             QueryAst::Boost { underlying, boost } => self.visit_boost(underlying, *boost),
             QueryAst::UserInput(user_text_query) => self.visit_user_text(user_text_query),
             QueryAst::FieldPresence(exists) => self.visit_exists(exists),
+            QueryAst::Wildcard(wildcard) => self.visit_wildcard(wildcard),
         }
     }
 
@@ -105,4 +107,8 @@ pub trait QueryAstVisitor<'a> {
     fn visit_exists(&mut self, _exists_query: &'a FieldPresenceQuery) -> Result<(), Self::Err> {
         Ok(())
     }
+
+    fn visit_wildcard(&mut self, _wildcard_query: &'a WildcardQuery) -> Result<(), Self::Err> {
+        Ok(())
+    }
 }
diff --git a/quickwit/quickwit-query/src/query_ast/wildcard_query.rs b/quickwit/quickwit-query/src/query_ast/wildcard_query.rs
new file mode 100644
index 00000000000..6e69d242fbe
--- /dev/null
+++ b/quickwit/quickwit-query/src/query_ast/wildcard_query.rs
@@ -0,0 +1,203 @@
+// Copyright (C) 2023 Quickwit, Inc.
+//
+// Quickwit is offered under the AGPL v3.0 and as commercial software.
+// For commercial licensing, contact us at hello@quickwit.io.
+//
+// AGPL:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+use anyhow::{anyhow, bail, Context};
+use serde::{Deserialize, Serialize};
+use tantivy::json_utils::JsonTermWriter;
+use tantivy::schema::{Field, FieldType, Schema as TantivySchema};
+use tantivy::Term;
+
+use super::{BuildTantivyAst, QueryAst};
+use crate::query_ast::TantivyQueryAst;
+use crate::tokenizers::TokenizerManager;
+use crate::{find_field_or_hit_dynamic, InvalidQuery};
+
+/// A Wildcard query allows to match 'bond' with a query like 'b*d'.
+///
+/// At the moment, only wildcard at end of term is supported.
+#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)]
+pub struct WildcardQuery {
+    pub field: String,
+    pub value: String,
+}
+
+impl From<WildcardQuery> for QueryAst {
+    fn from(wildcard_query: WildcardQuery) -> Self {
+        Self::Wildcard(wildcard_query)
+    }
+}
+
+impl WildcardQuery {
+    #[cfg(test)]
+    pub fn from_field_value(field: impl ToString, value: impl ToString) -> Self {
+        Self {
+            field: field.to_string(),
+            value: value.to_string(),
+        }
+    }
+}
+
+fn extract_unique_token(mut tokens: Vec<Term>) -> anyhow::Result<Term> {
+    let term = tokens
+        .pop()
+        .with_context(|| "wildcard query generated no term")?;
+    if !tokens.is_empty() {
+        anyhow::bail!("wildcard query generated more than one term");
+    }
+    Ok(term)
+}
+
+fn unescape_with_final_wildcard(phrase: &str) -> anyhow::Result<String> {
+    enum State {
+        Normal,
+        Escaped,
+    }
+
+    // we keep this state outside of scan because we want to query if after
+    let mut saw_wildcard = false;
+    let saw_wildcard = &mut saw_wildcard;
+
+    let phrase = phrase
+        .chars()
+        .scan(State::Normal, |state, c| {
+            if *saw_wildcard {
+                return Some(Some(Err(anyhow!(
+                    "Wildcard iquery contains wildcard in non final position"
+                ))));
+            }
+            match state {
+                State::Escaped => {
+                    *state = State::Normal;
+                    Some(Some(Ok(c)))
+                }
+                State::Normal => {
+                    if c == '*' {
+                        *saw_wildcard = true;
+                        Some(None)
+                    } else if c == '\\' {
+                        *state = State::Escaped;
+                        Some(None)
+                    } else if c == '?' {
+                        Some(Some(Err(anyhow!("Wildcard query contains `?`"))))
+                    } else {
+                        Some(Some(Ok(c)))
+                    }
+                }
+            }
+        })
+        // we have an iterator of Option<Result<char, anyhow::Error>>
+        .flatten()
+        // we have an iterator of Result<char, anyhow::Error>
+        .collect::<Result<String, _>>()?;
+    if !*saw_wildcard {
+        bail!("Wildcard query doesn't contain a wildcard");
+    }
+    Ok(phrase)
+}
+
+impl WildcardQuery {
+    // TODO this method will probably disappear once we support the full semantic of
+    // wildcard queries
+    pub fn extract_prefix_term(
+        &self,
+        schema: &TantivySchema,
+        tokenizer_manager: &TokenizerManager,
+    ) -> Result<(Field, Term), InvalidQuery> {
+        let (field, field_entry, json_path) = find_field_or_hit_dynamic(&self.field, schema)?;
+        let field_type = field_entry.field_type();
+
+        let prefix = unescape_with_final_wildcard(&self.value)?;
+
+        match field_type {
+            FieldType::Str(ref text_options) => {
+                let text_field_indexing = text_options.get_indexing_options().ok_or_else(|| {
+                    InvalidQuery::SchemaError(format!(
+                        "field {} is not full-text searchable",
+                        field_entry.name()
+                    ))
+                })?;
+                let tokenizer_name = text_field_indexing.tokenizer();
+                let mut normalizer = tokenizer_manager
+                    .get_normalizer(tokenizer_name)
+                    .with_context(|| {
+                        format!("no tokenizer named `{}` is registered", tokenizer_name)
+                    })?;
+                let mut token_stream = normalizer.token_stream(&prefix);
+                let mut tokens = Vec::new();
+                token_stream.process(&mut |token| {
+                    let term: Term = Term::from_field_text(field, &token.text);
+                    tokens.push(term);
+                });
+                let term = extract_unique_token(tokens)?;
+                Ok((field, term))
+            }
+            FieldType::JsonObject(json_options) => {
+                let text_field_indexing =
+                    json_options.get_text_indexing_options().ok_or_else(|| {
+                        InvalidQuery::SchemaError(format!(
+                            "field {} is not full-text searchable",
+                            field_entry.name()
+                        ))
+                    })?;
+                let tokenizer_name = text_field_indexing.tokenizer();
+                let mut normalizer = tokenizer_manager
+                    .get_normalizer(tokenizer_name)
+                    .with_context(|| {
+                        format!("no tokenizer named `{}` is registered", tokenizer_name)
+                    })?;
+                let mut token_stream = normalizer.token_stream(&prefix);
+                let mut tokens = Vec::new();
+                let mut term = Term::with_capacity(100);
+                let mut json_term_writer = JsonTermWriter::from_field_and_json_path(
+                    field,
+                    json_path,
+                    json_options.is_expand_dots_enabled(),
+                    &mut term,
+                );
+
+                token_stream.process(&mut |token| {
+                    json_term_writer.set_str(&token.text);
+                    tokens.push(json_term_writer.term().clone());
+                });
+                let term = extract_unique_token(tokens)?;
+                Ok((field, term))
+            }
+            _ => Err(InvalidQuery::SchemaError(
+                "trying to run a Wildcard query on a non-text field".to_string(),
+            )),
+        }
+    }
+}
+
+impl BuildTantivyAst for WildcardQuery {
+    fn build_tantivy_ast_impl(
+        &self,
+        schema: &TantivySchema,
+        tokenizer_manager: &TokenizerManager,
+        _search_fields: &[String],
+        _with_validation: bool,
+    ) -> Result<TantivyQueryAst, InvalidQuery> {
+        let (_, term) = self.extract_prefix_term(schema, tokenizer_manager)?;
+
+        let mut phrase_prefix_query =
+            tantivy::query::PhrasePrefixQuery::new_with_offset(vec![(0, term)]);
+        phrase_prefix_query.set_max_expansions(u32::MAX);
+        Ok(phrase_prefix_query.into())
+    }
+}
diff --git a/quickwit/quickwit-query/src/tokenizers/chinese_compatible.rs b/quickwit/quickwit-query/src/tokenizers/chinese_compatible.rs
index 1f326081e11..785d3c4a33f 100644
--- a/quickwit/quickwit-query/src/tokenizers/chinese_compatible.rs
+++ b/quickwit/quickwit-query/src/tokenizers/chinese_compatible.rs
@@ -135,7 +135,9 @@ mod tests {
     fn test_chinese_tokenizer() {
         let text = "Hello world, 你好世界, bonjour monde";
         let tokenizer_manager = crate::create_default_quickwit_tokenizer_manager();
-        let mut tokenizer = tokenizer_manager.get("chinese_compatible").unwrap();
+        let mut tokenizer = tokenizer_manager
+            .get_tokenizer("chinese_compatible")
+            .unwrap();
         let mut text_stream = tokenizer.token_stream(text);
 
         let mut res = Vec::new();
@@ -210,7 +212,9 @@ mod tests {
     fn test_chinese_tokenizer_no_space() {
         let text = "Hello你好bonjour";
         let tokenizer_manager = crate::create_default_quickwit_tokenizer_manager();
-        let mut tokenizer = tokenizer_manager.get("chinese_compatible").unwrap();
+        let mut tokenizer = tokenizer_manager
+            .get_tokenizer("chinese_compatible")
+            .unwrap();
         let mut text_stream = tokenizer.token_stream(text);
 
         let mut res = Vec::new();
@@ -256,8 +260,8 @@ mod tests {
         #[test]
         fn test_proptest_ascii_default_chinese_equal(text in "[ -~]{0,64}") {
             let tokenizer_manager = crate::create_default_quickwit_tokenizer_manager();
-            let mut cn_tok = tokenizer_manager.get("chinese_compatible").unwrap();
-            let mut default_tok = tokenizer_manager.get("default").unwrap();
+            let mut cn_tok = tokenizer_manager.get_tokenizer("chinese_compatible").unwrap();
+            let mut default_tok = tokenizer_manager.get_tokenizer("default").unwrap();
 
             let mut text_stream = cn_tok.token_stream(&text);
 
diff --git a/quickwit/quickwit-query/src/tokenizers/mod.rs b/quickwit/quickwit-query/src/tokenizers/mod.rs
index 07a27c86689..4b70eb6cce2 100644
--- a/quickwit/quickwit-query/src/tokenizers/mod.rs
+++ b/quickwit/quickwit-query/src/tokenizers/mod.rs
@@ -21,32 +21,57 @@ mod chinese_compatible;
 mod code_tokenizer;
 #[cfg(feature = "multilang")]
 mod multilang;
+mod tokenizer_manager;
 
 use once_cell::sync::Lazy;
 use tantivy::tokenizer::{
-    AsciiFoldingFilter, LowerCaser, RawTokenizer, RemoveLongFilter, TextAnalyzer, TokenizerManager,
+    AsciiFoldingFilter, Language, LowerCaser, RawTokenizer, RemoveLongFilter, SimpleTokenizer,
+    Stemmer, TextAnalyzer, WhitespaceTokenizer,
 };
 
 use self::chinese_compatible::ChineseTokenizer;
 pub use self::code_tokenizer::CodeTokenizer;
 #[cfg(feature = "multilang")]
 pub use self::multilang::MultiLangTokenizer;
+pub use self::tokenizer_manager::TokenizerManager;
 
 pub const DEFAULT_REMOVE_TOKEN_LENGTH: usize = 255;
 
 /// Quickwit's tokenizer/analyzer manager.
 pub fn create_default_quickwit_tokenizer_manager() -> TokenizerManager {
-    let tokenizer_manager = TokenizerManager::default();
+    let tokenizer_manager = TokenizerManager::new();
 
     let raw_tokenizer = TextAnalyzer::builder(RawTokenizer::default())
         .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH))
         .build();
-    tokenizer_manager.register("raw", raw_tokenizer);
+    tokenizer_manager.register("raw", raw_tokenizer, false);
+
+    let lower_case_tokenizer = TextAnalyzer::builder(RawTokenizer::default())
+        .filter(LowerCaser)
+        .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH))
+        .build();
+    tokenizer_manager.register("lowercase", lower_case_tokenizer, true);
+
+    let default_tokenizer = TextAnalyzer::builder(SimpleTokenizer::default())
+        .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH))
+        .filter(LowerCaser)
+        .build();
+    tokenizer_manager.register("default", default_tokenizer, true);
+
+    let en_stem_tokenizer = TextAnalyzer::builder(SimpleTokenizer::default())
+        .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH))
+        .filter(LowerCaser)
+        .filter(Stemmer::new(Language::English))
+        .build();
+    tokenizer_manager.register("en_stem", en_stem_tokenizer, true);
+
+    tokenizer_manager.register("whitespace", WhitespaceTokenizer::default(), false);
+
     let chinese_tokenizer = TextAnalyzer::builder(ChineseTokenizer)
         .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH))
         .filter(LowerCaser)
         .build();
-    tokenizer_manager.register("chinese_compatible", chinese_tokenizer);
+    tokenizer_manager.register("chinese_compatible", chinese_tokenizer, false);
     tokenizer_manager.register(
         "source_code_default",
         TextAnalyzer::builder(CodeTokenizer::default())
@@ -54,6 +79,7 @@ pub fn create_default_quickwit_tokenizer_manager() -> TokenizerManager {
             .filter(LowerCaser)
             .filter(AsciiFoldingFilter)
             .build(),
+        true,
     );
     #[cfg(feature = "multilang")]
     tokenizer_manager.register(
@@ -62,6 +88,7 @@ pub fn create_default_quickwit_tokenizer_manager() -> TokenizerManager {
             .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH))
             .filter(LowerCaser)
             .build(),
+        true,
     );
     tokenizer_manager
 }
@@ -75,8 +102,8 @@ fn create_quickwit_fastfield_normalizer_manager() -> TokenizerManager {
         .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH))
         .build();
     let tokenizer_manager = TokenizerManager::new();
-    tokenizer_manager.register("raw", raw_tokenizer);
-    tokenizer_manager.register("lowercase", lower_case_tokenizer);
+    tokenizer_manager.register("raw", raw_tokenizer, false);
+    tokenizer_manager.register("lowercase", lower_case_tokenizer, true);
     tokenizer_manager
 }
 
@@ -92,9 +119,11 @@ mod tests {
     #[test]
     fn test_tokenizers_in_manager() {
         let tokenizer_manager = super::create_default_quickwit_tokenizer_manager();
-        tokenizer_manager.get("chinese_compatible").unwrap();
-        tokenizer_manager.get("default").unwrap();
-        tokenizer_manager.get("raw").unwrap();
+        tokenizer_manager
+            .get_tokenizer("chinese_compatible")
+            .unwrap();
+        tokenizer_manager.get_tokenizer("default").unwrap();
+        tokenizer_manager.get_tokenizer("raw").unwrap();
     }
 
     #[test]
@@ -109,11 +138,11 @@ mod tests {
                             it, no one shall find it. I just need some more chars, now you may \
                             not pass.";
 
-        let mut tokenizer = tokenizer_manager.get("raw").unwrap();
+        let mut tokenizer = tokenizer_manager.get_tokenizer("raw").unwrap();
         let mut haiku_stream = tokenizer.token_stream(my_haiku);
         assert!(haiku_stream.advance());
         assert!(!haiku_stream.advance());
-        let mut other_tokenizer = tokenizer_manager.get("raw").unwrap();
+        let mut other_tokenizer = tokenizer_manager.get_tokenizer("raw").unwrap();
         let mut other_stream = other_tokenizer.token_stream(my_long_text);
         assert!(other_stream.advance());
         assert!(!other_stream.advance());
@@ -122,7 +151,7 @@ mod tests {
     #[test]
     fn test_code_tokenizer_in_tokenizer_manager() {
         let mut code_tokenizer = super::create_default_quickwit_tokenizer_manager()
-            .get("source_code_default")
+            .get_tokenizer("source_code_default")
             .unwrap();
         let mut token_stream = code_tokenizer.token_stream("PigCaféFactory2");
         let mut tokens = Vec::new();
diff --git a/quickwit/quickwit-query/src/tokenizers/tokenizer_manager.rs b/quickwit/quickwit-query/src/tokenizers/tokenizer_manager.rs
new file mode 100644
index 00000000000..7169d5ed740
--- /dev/null
+++ b/quickwit/quickwit-query/src/tokenizers/tokenizer_manager.rs
@@ -0,0 +1,103 @@
+// Copyright (C) 2023 Quickwit, Inc.
+//
+// Quickwit is offered under the AGPL v3.0 and as commercial software.
+// For commercial licensing, contact us at hello@quickwit.io.
+//
+// AGPL:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+use std::collections::HashMap;
+use std::sync::{Arc, RwLock};
+
+use tantivy::tokenizer::{
+    LowerCaser, RawTokenizer, RemoveLongFilter, TextAnalyzer,
+    TokenizerManager as TantivyTokenizerManager,
+};
+
+use crate::DEFAULT_REMOVE_TOKEN_LENGTH;
+
+const RAW_TOKENIZER_NAME: &str = "raw";
+const LOWERCASE_TOKENIZER_NAME: &str = "lowercase";
+
+#[derive(Clone)]
+pub struct TokenizerManager {
+    inner: TantivyTokenizerManager,
+    is_lowercaser: Arc<RwLock<HashMap<String, bool>>>,
+}
+
+impl TokenizerManager {
+    /// Creates an empty tokenizer manager.
+    pub fn new() -> Self {
+        let this = Self {
+            inner: TantivyTokenizerManager::new(),
+            is_lowercaser: Arc::new(RwLock::new(HashMap::new())),
+        };
+
+        // in practice these will almost always be overriden in
+        // create_default_quickwit_tokenizer_manager()
+        let raw_tokenizer = TextAnalyzer::builder(RawTokenizer::default())
+            .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH))
+            .build();
+        this.register(RAW_TOKENIZER_NAME, raw_tokenizer, false);
+        let lower_case_tokenizer = TextAnalyzer::builder(RawTokenizer::default())
+            .filter(LowerCaser)
+            .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH))
+            .build();
+        this.register(LOWERCASE_TOKENIZER_NAME, lower_case_tokenizer, true);
+
+        this
+    }
+
+    /// Registers a new tokenizer associated with a given name.
+    pub fn register<T>(&self, tokenizer_name: &str, tokenizer: T, does_lowercasing: bool)
+    where TextAnalyzer: From<T> {
+        self.inner.register(tokenizer_name, tokenizer);
+        self.is_lowercaser
+            .write()
+            .unwrap()
+            .insert(tokenizer_name.to_string(), does_lowercasing);
+    }
+
+    /// Accessing a tokenizer given its name.
+    pub fn get_tokenizer(&self, tokenizer_name: &str) -> Option<TextAnalyzer> {
+        self.inner.get(tokenizer_name)
+    }
+
+    /// Query whether a given tokenizer does lowercasing
+    pub fn get_normalizer(&self, tokenizer_name: &str) -> Option<TextAnalyzer> {
+        let use_lowercaser = self
+            .is_lowercaser
+            .read()
+            .unwrap()
+            .get(tokenizer_name)
+            .copied()?;
+        let analyzer = if use_lowercaser {
+            LOWERCASE_TOKENIZER_NAME
+        } else {
+            RAW_TOKENIZER_NAME
+        };
+        self.get_tokenizer(analyzer)
+    }
+
+    /// Get the inner TokenizerManager
+    pub fn tantivy_manager(&self) -> &TantivyTokenizerManager {
+        &self.inner
+    }
+}
+
+impl Default for TokenizerManager {
+    fn default() -> Self {
+        Self::new()
+    }
+}
diff --git a/quickwit/quickwit-rest-client/src/models.rs b/quickwit/quickwit-rest-client/src/models.rs
index 72e2fb1b804..6416a8ec286 100644
--- a/quickwit/quickwit-rest-client/src/models.rs
+++ b/quickwit/quickwit-rest-client/src/models.rs
@@ -26,6 +26,7 @@ use serde::de::DeserializeOwned;
 
 use crate::error::{ApiError, Error, ErrorResponsePayload};
 
+#[derive(Debug)]
 pub struct ApiResponse {
     inner: reqwest::Response,
 }
diff --git a/quickwit/quickwit-rest-client/src/rest_client.rs b/quickwit/quickwit-rest-client/src/rest_client.rs
index 3125c318702..6b333131b93 100644
--- a/quickwit/quickwit-rest-client/src/rest_client.rs
+++ b/quickwit/quickwit-rest-client/src/rest_client.rs
@@ -31,6 +31,7 @@ use reqwest::header::{HeaderMap, HeaderValue, CONTENT_TYPE};
 use reqwest::{Client, ClientBuilder, Method, StatusCode, Url};
 use serde::Serialize;
 use serde_json::json;
+use tracing::warn;
 
 use crate::error::Error;
 use crate::models::{ApiResponse, IngestSource, Timeout};
@@ -119,6 +120,8 @@ pub struct QuickwitClientBuilder {
     ingest_timeout: Timeout,
     /// Timeout for the ingest operations that require waiting for commit.
     commit_timeout: Timeout,
+    /// Experimental: if true, use the ingest v2 endpoint.
+    ingest_v2: bool,
 }
 
 impl QuickwitClientBuilder {
@@ -130,6 +133,7 @@ impl QuickwitClientBuilder {
             search_timeout: DEFAULT_CLIENT_SEARCH_TIMEOUT,
             ingest_timeout: DEFAULT_CLIENT_INGEST_TIMEOUT,
             commit_timeout: DEFAULT_CLIENT_COMMIT_TIMEOUT,
+            ingest_v2: false,
         }
     }
 
@@ -143,6 +147,12 @@ impl QuickwitClientBuilder {
         self
     }
 
+    pub fn enable_ingest_v2(mut self) -> Self {
+        warn!("ingest v2 experimental feature enabled!");
+        self.ingest_v2 = true;
+        self
+    }
+
     pub fn search_timeout(mut self, timeout: Timeout) -> Self {
         self.search_timeout = timeout;
         self
@@ -160,13 +170,14 @@ impl QuickwitClientBuilder {
 
     pub fn build(self) -> QuickwitClient {
         let transport = Transport::new(self.base_url, self.connect_timeout);
-        QuickwitClient::new(
+        QuickwitClient {
             transport,
-            self.timeout,
-            self.search_timeout,
-            self.ingest_timeout,
-            self.commit_timeout,
-        )
+            timeout: self.timeout,
+            search_timeout: self.search_timeout,
+            ingest_timeout: self.ingest_timeout,
+            commit_timeout: self.commit_timeout,
+            ingest_v2: self.ingest_v2,
+        }
     }
 }
 
@@ -181,23 +192,14 @@ pub struct QuickwitClient {
     ingest_timeout: Timeout,
     /// Timeout for the ingest operations that require waiting for commit.
     commit_timeout: Timeout,
+    // TODO remove me after Quickwit 0.7 release.
+    // If true, rely on ingest v2
+    ingest_v2: bool,
 }
 
 impl QuickwitClient {
-    fn new(
-        transport: Transport,
-        timeout: Timeout,
-        search_timeout: Timeout,
-        ingest_timeout: Timeout,
-        commit_timeout: Timeout,
-    ) -> Self {
-        Self {
-            transport,
-            timeout,
-            search_timeout,
-            ingest_timeout,
-            commit_timeout,
-        }
+    pub fn enable_ingest_v2(&mut self) {
+        self.ingest_v2 = true;
     }
 
     pub async fn search(
@@ -258,7 +260,11 @@ impl QuickwitClient {
         on_ingest_event: Option<&(dyn Fn(IngestEvent) + Sync)>,
         last_block_commit: CommitType,
     ) -> Result<(), Error> {
-        let ingest_path = format!("{index_id}/ingest");
+        let ingest_path = if self.ingest_v2 {
+            format!("{index_id}/ingest-v2")
+        } else {
+            format!("{index_id}/ingest")
+        };
         let batch_size_limit = batch_size_limit_opt.unwrap_or(INGEST_CONTENT_LENGTH_LIMIT);
         let mut batch_reader = match ingest_source {
             IngestSource::File(filepath) => {
@@ -286,7 +292,6 @@ impl QuickwitClient {
                         timeout,
                     )
                     .await?;
-
                 if response.status_code() == StatusCode::TOO_MANY_REQUESTS {
                     if let Some(event_fn) = &on_ingest_event {
                         event_fn(IngestEvent::Sleep)
diff --git a/quickwit/quickwit-search/src/collector.rs b/quickwit/quickwit-search/src/collector.rs
index d91fbf1b8e0..bd0982ed6eb 100644
--- a/quickwit/quickwit-search/src/collector.rs
+++ b/quickwit/quickwit-search/src/collector.rs
@@ -1169,7 +1169,7 @@ mod tests {
         fn max_num_partitions(&self) -> std::num::NonZeroU32 {
             unimplemented!()
         }
-        fn tokenizer_manager(&self) -> &tantivy::tokenizer::TokenizerManager {
+        fn tokenizer_manager(&self) -> &quickwit_query::tokenizers::TokenizerManager {
             unimplemented!()
         }
 
@@ -1227,8 +1227,9 @@ mod tests {
     }
 
     fn make_index() -> tantivy::Index {
+        use tantivy::indexer::UserOperation;
         use tantivy::schema::{NumericOptions, Schema};
-        use tantivy::{Index, UserOperation};
+        use tantivy::Index;
 
         let dataset = sort_dataset();
 
diff --git a/quickwit/quickwit-search/src/leaf.rs b/quickwit/quickwit-search/src/leaf.rs
index c11e86636fb..b6d3c85b4b6 100644
--- a/quickwit/quickwit-search/src/leaf.rs
+++ b/quickwit/quickwit-search/src/leaf.rs
@@ -33,13 +33,13 @@ use quickwit_proto::search::{
     SearchRequest, SortOrder, SortValue, SplitIdAndFooterOffsets, SplitSearchError,
 };
 use quickwit_query::query_ast::QueryAst;
+use quickwit_query::tokenizers::TokenizerManager;
 use quickwit_storage::{
     wrap_storage_with_cache, BundleStorage, MemorySizedCache, OwnedBytes, SplitCache, Storage,
 };
 use tantivy::directory::FileSlice;
 use tantivy::fastfield::FastFieldReaders;
 use tantivy::schema::{Field, FieldType};
-use tantivy::tokenizer::TokenizerManager;
 use tantivy::{Index, ReloadPolicy, Searcher, Term};
 use tracing::*;
 
@@ -130,10 +130,12 @@ pub(crate) async fn open_index_with_caches(
     };
     let mut index = Index::open(hot_directory)?;
     if let Some(tokenizer_manager) = tokenizer_manager {
-        index.set_tokenizers(tokenizer_manager.clone());
+        index.set_tokenizers(tokenizer_manager.tantivy_manager().clone());
     }
     index.set_fast_field_tokenizers(
-        quickwit_query::get_quickwit_fastfield_normalizer_manager().clone(),
+        quickwit_query::get_quickwit_fastfield_normalizer_manager()
+            .tantivy_manager()
+            .clone(),
     );
     Ok(index)
 }
diff --git a/quickwit/quickwit-search/src/root.rs b/quickwit/quickwit-search/src/root.rs
index bbbe67f3ade..2a916f00fa4 100644
--- a/quickwit/quickwit-search/src/root.rs
+++ b/quickwit/quickwit-search/src/root.rs
@@ -392,7 +392,7 @@ fn get_scroll_ttl_duration(search_request: &SearchRequest) -> crate::Result<Opti
     Ok(Some(scroll_ttl))
 }
 
-#[instrument(skip_all)]
+#[instrument(level = "debug", skip_all)]
 async fn search_partial_hits_phase_with_scroll(
     searcher_context: &SearcherContext,
     indexes_metas_for_leaf_search: &IndexesMetasForLeafSearch,
@@ -457,7 +457,7 @@ async fn search_partial_hits_phase_with_scroll(
     }
 }
 
-#[instrument(skip_all)]
+#[instrument(level = "debug", skip_all)]
 pub(crate) async fn search_partial_hits_phase(
     searcher_context: &SearcherContext,
     indexes_metas_for_leaf_search: &IndexesMetasForLeafSearch,
@@ -498,8 +498,13 @@ pub(crate) async fn search_partial_hits_phase(
     .await
     .context("failed to merge leaf search responses")?
     .map_err(|error: TantivyError| crate::SearchError::Internal(error.to_string()))?;
-    debug!(leaf_search_response = ?leaf_search_response, "Merged leaf search response.");
-
+    debug!(
+        num_hits = leaf_search_response.num_hits,
+        failed_splits = ?leaf_search_response.failed_splits,
+        num_attempted_splits = leaf_search_response.num_attempted_splits,
+        has_intermediate_aggregation_result = leaf_search_response.intermediate_aggregation_result.is_some(),
+        "Merged leaf search response."
+    );
     if !leaf_search_response.failed_splits.is_empty() {
         error!(failed_splits = ?leaf_search_response.failed_splits, "Leaf search response contains at least one failed split.");
         let errors: String = leaf_search_response.failed_splits.iter().join(", ");
@@ -1061,7 +1066,10 @@ pub async fn root_list_terms(
         merged_iter.collect()
     };
 
-    debug!(leaf_list_terms_response = ?leaf_list_terms_response, "Merged leaf search response.");
+    debug!(
+        leaf_list_terms_response_count = leaf_list_terms_response.len(),
+        "Merged leaf search response."
+    );
 
     let elapsed = start_instant.elapsed();
 
diff --git a/quickwit/quickwit-search/src/service.rs b/quickwit/quickwit-search/src/service.rs
index ce28e2962a1..48d0d6e2e05 100644
--- a/quickwit/quickwit-search/src/service.rs
+++ b/quickwit/quickwit-search/src/service.rs
@@ -431,7 +431,7 @@ impl SearcherContext {
 
     /// Creates a new searcher context, given a searcher config, and an optional `SplitCache`.
     pub fn new(searcher_config: SearcherConfig, split_cache_opt: Option<Arc<SplitCache>>) -> Self {
-        let capacity_in_bytes = searcher_config.split_footer_cache_capacity.get_bytes() as usize;
+        let capacity_in_bytes = searcher_config.split_footer_cache_capacity.as_u64() as usize;
         let global_split_footer_cache = MemorySizedCache::with_capacity_in_bytes(
             capacity_in_bytes,
             &quickwit_storage::STORAGE_METRICS.split_footer_cache,
@@ -441,12 +441,10 @@ impl SearcherContext {
         ));
         let split_stream_semaphore =
             Semaphore::new(searcher_config.max_num_concurrent_split_streams);
-        let fast_field_cache_capacity =
-            searcher_config.fast_field_cache_capacity.get_bytes() as usize;
+        let fast_field_cache_capacity = searcher_config.fast_field_cache_capacity.as_u64() as usize;
         let storage_long_term_cache = Arc::new(QuickwitCache::new(fast_field_cache_capacity));
-        let leaf_search_cache = LeafSearchCache::new(
-            searcher_config.partial_request_cache_capacity.get_bytes() as usize,
-        );
+        let leaf_search_cache =
+            LeafSearchCache::new(searcher_config.partial_request_cache_capacity.as_u64() as usize);
 
         Self {
             searcher_config,
@@ -462,7 +460,7 @@ impl SearcherContext {
     /// Returns a new instance to track the aggregation memory usage.
     pub fn get_aggregation_limits(&self) -> AggregationLimits {
         AggregationLimits::new(
-            Some(self.searcher_config.aggregation_memory_limit.get_bytes()),
+            Some(self.searcher_config.aggregation_memory_limit.as_u64()),
             Some(self.searcher_config.aggregation_bucket_limit),
         )
     }
diff --git a/quickwit/quickwit-serve/Cargo.toml b/quickwit/quickwit-serve/Cargo.toml
index 1fe8d67d367..4454bd5d311 100644
--- a/quickwit/quickwit-serve/Cargo.toml
+++ b/quickwit/quickwit-serve/Cargo.toml
@@ -13,7 +13,7 @@ documentation = "https://quickwit.io/docs/"
 anyhow = { workspace = true }
 async-trait = { workspace = true }
 bytes = { workspace = true }
-byte-unit = { workspace = true }
+bytesize = { workspace = true }
 elasticsearch-dsl = "0.4.15"
 futures = { workspace = true }
 futures-util = { workspace = true }
diff --git a/quickwit/quickwit-serve/src/elastic_search_api/filter.rs b/quickwit/quickwit-serve/src/elastic_search_api/filter.rs
index eae0bbfdb1c..3208d7e3e8f 100644
--- a/quickwit/quickwit-serve/src/elastic_search_api/filter.rs
+++ b/quickwit/quickwit-serve/src/elastic_search_api/filter.rs
@@ -17,8 +17,8 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 
-use byte_unit::Byte;
 use bytes::Bytes;
+use bytesize::ByteSize;
 use serde::de::DeserializeOwned;
 use warp::reject::LengthRequired;
 use warp::{Filter, Rejection};
@@ -29,8 +29,8 @@ use crate::elastic_search_api::model::{
 };
 use crate::search_api::extract_index_id_patterns;
 
-const BODY_LENGTH_LIMIT: Byte = byte_unit::Byte::from_bytes(1_000_000);
-const CONTENT_LENGTH_LIMIT: Byte = byte_unit::Byte::from_bytes(10 * 1024 * 1024); // 10MiB
+const BODY_LENGTH_LIMIT: ByteSize = ByteSize::mb(1);
+const CONTENT_LENGTH_LIMIT: ByteSize = ByteSize::mib(10);
 
 // TODO: Make all elastic endpoint models `utoipa` compatible
 // and register them here.
@@ -71,7 +71,7 @@ pub(crate) fn elastic_bulk_filter(
     warp::path!("_elastic" / "_bulk")
         .and(warp::post())
         .and(warp::body::content_length_limit(
-            CONTENT_LENGTH_LIMIT.get_bytes(),
+            CONTENT_LENGTH_LIMIT.as_u64(),
         ))
         .and(warp::body::bytes())
         .and(serde_qs::warp::query(serde_qs::Config::default()))
@@ -80,7 +80,7 @@ pub(crate) fn elastic_bulk_filter(
 /// Like the warp json filter, but accepts an empty body and interprets it as `T::default`.
 fn json_or_empty<T: DeserializeOwned + Send + Default>(
 ) -> impl Filter<Extract = (T,), Error = Rejection> + Copy {
-    warp::body::content_length_limit(BODY_LENGTH_LIMIT.get_bytes())
+    warp::body::content_length_limit(BODY_LENGTH_LIMIT.as_u64())
         .and(warp::body::bytes().and_then(|buf: Bytes| async move {
             if buf.is_empty() {
                 return Ok(T::default());
@@ -128,7 +128,7 @@ pub(crate) fn elastic_index_bulk_filter(
     warp::path!("_elastic" / String / "_bulk")
         .and(warp::post())
         .and(warp::body::content_length_limit(
-            CONTENT_LENGTH_LIMIT.get_bytes(),
+            CONTENT_LENGTH_LIMIT.as_u64(),
         ))
         .and(warp::body::bytes())
         .and(serde_qs::warp::query::<ElasticIngestOptions>(
@@ -140,9 +140,7 @@ pub(crate) fn elastic_index_bulk_filter(
 pub(crate) fn elastic_multi_search_filter(
 ) -> impl Filter<Extract = (Bytes, MultiSearchQueryParams), Error = Rejection> + Clone {
     warp::path!("_elastic" / "_msearch")
-        .and(warp::body::content_length_limit(
-            BODY_LENGTH_LIMIT.get_bytes(),
-        ))
+        .and(warp::body::content_length_limit(BODY_LENGTH_LIMIT.as_u64()))
         .and(warp::body::bytes())
         .and(warp::post())
         .and(serde_qs::warp::query(serde_qs::Config::default()))
@@ -162,9 +160,7 @@ fn merge_scroll_body_params(
 pub(crate) fn elastic_scroll_filter(
 ) -> impl Filter<Extract = (ScrollQueryParams,), Error = Rejection> + Clone {
     warp::path!("_elastic" / "_search" / "scroll")
-        .and(warp::body::content_length_limit(
-            BODY_LENGTH_LIMIT.get_bytes(),
-        ))
+        .and(warp::body::content_length_limit(BODY_LENGTH_LIMIT.as_u64()))
         .and(warp::get().or(warp::post()).unify())
         .and(serde_qs::warp::query(serde_qs::Config::default()))
         .and(json_or_empty())
diff --git a/quickwit/quickwit-serve/src/index_api/rest_handler.rs b/quickwit/quickwit-serve/src/index_api/rest_handler.rs
index c203bab2dfb..78d235fe610 100644
--- a/quickwit/quickwit-serve/src/index_api/rest_handler.rs
+++ b/quickwit/quickwit-serve/src/index_api/rest_handler.rs
@@ -439,7 +439,6 @@ fn mark_splits_for_deletion_handler(
 async fn get_indexes_metadatas(
     mut metastore: MetastoreServiceClient,
 ) -> MetastoreResult<Vec<IndexMetadata>> {
-    info!("get-indexes-metadatas");
     metastore
         .list_indexes_metadata(ListIndexesMetadataRequest::all())
         .await
@@ -1300,7 +1299,7 @@ mod tests {
             let resp_json: serde_json::Value = serde_json::from_slice(resp.body()).unwrap();
             let expected_response_json = serde_json::json!([{
                 "file_name": "split_1.split",
-                "file_size_bytes": 800,
+                "file_size_bytes": "800 B",
             }]);
             assert_json_include!(actual: resp_json, expected: expected_response_json);
         }
@@ -1314,7 +1313,7 @@ mod tests {
             let resp_json: serde_json::Value = serde_json::from_slice(resp.body()).unwrap();
             let expected_response_json = serde_json::json!([{
                 "file_name": "split_1.split",
-                "file_size_bytes": 800,
+                "file_size_bytes": "800 B",
             }]);
             assert_json_include!(actual: resp_json, expected: expected_response_json);
         }
diff --git a/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs b/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs
index efe95991828..9c0dd0ea912 100644
--- a/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs
+++ b/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs
@@ -24,8 +24,8 @@ use quickwit_ingest::{
     IngestServiceClient, IngestServiceError, TailRequest,
 };
 use quickwit_proto::ingest::router::{
-    IngestRequestV2, IngestResponseV2, IngestRouterService, IngestRouterServiceClient,
-    IngestSubrequest,
+    IngestFailureReason, IngestRequestV2, IngestResponseV2, IngestRouterService,
+    IngestRouterServiceClient, IngestSubrequest,
 };
 use quickwit_proto::ingest::{DocBatchV2, IngestV2Error};
 use quickwit_proto::types::IndexId;
@@ -79,7 +79,7 @@ fn ingest_filter(
     warp::path!(String / "ingest")
         .and(warp::post())
         .and(warp::body::content_length_limit(
-            config.content_length_limit,
+            config.content_length_limit.as_u64(),
         ))
         .and(warp::body::bytes())
         .and(serde_qs::warp::query::<IngestOptions>(
@@ -103,7 +103,7 @@ fn ingest_v2_filter(
     warp::path!(String / "ingest-v2")
         .and(warp::post())
         .and(warp::body::content_length_limit(
-            config.content_length_limit,
+            config.content_length_limit.as_u64(),
         ))
         .and(warp::body::bytes())
         .and(serde_qs::warp::query::<IngestOptions>(
@@ -127,7 +127,7 @@ async fn ingest_v2(
     body: Bytes,
     ingest_options: IngestOptions,
     mut ingest_router: IngestRouterServiceClient,
-) -> Result<IngestResponseV2, IngestV2Error> {
+) -> Result<IngestResponse, IngestServiceError> {
     let mut doc_buffer = BytesMut::new();
     let mut doc_lengths = Vec::new();
 
@@ -135,6 +135,7 @@ async fn ingest_v2(
         doc_lengths.push(line.len() as u32);
         doc_buffer.put(line);
     }
+    let num_docs = doc_lengths.len();
     let doc_batch = DocBatchV2 {
         doc_buffer: doc_buffer.freeze(),
         doc_lengths,
@@ -149,8 +150,46 @@ async fn ingest_v2(
         commit_type: ingest_options.commit_type as i32,
         subrequests: vec![subrequest],
     };
-    let response = ingest_router.ingest(request).await?;
-    Ok(response)
+    let response = ingest_router
+        .ingest(request)
+        .await
+        .map_err(|err: IngestV2Error| IngestServiceError::Internal(err.to_string()))?;
+    convert_ingest_response_v2(response, num_docs)
+}
+
+fn convert_ingest_response_v2(
+    mut response: IngestResponseV2,
+    num_docs: usize,
+) -> Result<IngestResponse, IngestServiceError> {
+    let num_responses = response.successes.len() + response.failures.len();
+    if num_responses != 1 {
+        return Err(IngestServiceError::Internal(format!(
+            "Expected a single failure/success, got {}.",
+            num_responses
+        )));
+    }
+    if response.successes.pop().is_some() {
+        return Ok(IngestResponse {
+            num_docs_for_processing: num_docs as u64,
+        });
+    }
+    let ingest_failure = response.failures.pop().unwrap();
+    Err(match ingest_failure.reason() {
+        IngestFailureReason::Unspecified => {
+            IngestServiceError::Internal("Unknown reason".to_string())
+        }
+        IngestFailureReason::IndexNotFound => IngestServiceError::IndexNotFound {
+            index_id: ingest_failure.index_id,
+        },
+        IngestFailureReason::SourceNotFound => IngestServiceError::Internal(format!(
+            "Ingest v2 source not found for index {}",
+            ingest_failure.index_id
+        )),
+        IngestFailureReason::Internal => IngestServiceError::Internal("Internal error".to_string()),
+        IngestFailureReason::NoShardsAvailable => IngestServiceError::Unavailable,
+        IngestFailureReason::RateLimited => IngestServiceError::RateLimited,
+        IngestFailureReason::ResourceExhausted => IngestServiceError::Unavailable,
+    })
 }
 
 #[utoipa::path(
@@ -230,7 +269,7 @@ pub(crate) fn lines(body: &Bytes) -> impl Iterator<Item = &[u8]> {
 pub(crate) mod tests {
     use std::time::Duration;
 
-    use byte_unit::Byte;
+    use bytesize::ByteSize;
     use quickwit_actors::{Mailbox, Universe};
     use quickwit_config::IngestApiConfig;
     use quickwit_ingest::{
@@ -333,7 +372,7 @@ pub(crate) mod tests {
     #[tokio::test]
     async fn test_ingest_api_return_429_if_above_limits() {
         let config = IngestApiConfig {
-            max_queue_memory_usage: Byte::from_bytes(1),
+            max_queue_memory_usage: ByteSize(1),
             ..Default::default()
         };
         let (universe, _temp_dir, ingest_service, _) =
@@ -355,7 +394,7 @@ pub(crate) mod tests {
     #[tokio::test]
     async fn test_ingest_api_return_413_if_above_content_limit() {
         let config = IngestApiConfig {
-            content_length_limit: 1,
+            content_length_limit: ByteSize(1),
             ..Default::default()
         };
         let (universe, _temp_dir, ingest_service, _) =
diff --git a/quickwit/quickwit-serve/src/lib.rs b/quickwit/quickwit-serve/src/lib.rs
index a838d8e567a..f244c12b88d 100644
--- a/quickwit/quickwit-serve/src/lib.rs
+++ b/quickwit/quickwit-serve/src/lib.rs
@@ -48,7 +48,7 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use anyhow::Context;
-use byte_unit::n_mib_bytes;
+use bytesize::ByteSize;
 pub use format::BodyFormat;
 use futures::{Stream, StreamExt};
 use itertools::Itertools;
@@ -69,7 +69,7 @@ use quickwit_indexing::actors::IndexingService;
 use quickwit_indexing::start_indexing_service;
 use quickwit_ingest::{
     start_ingest_api_service, GetMemoryCapacity, IngestApiService, IngestRequest, IngestRouter,
-    IngestServiceClient, Ingester, IngesterPool,
+    IngestServiceClient, Ingester, IngesterPool, RateLimiterSettings,
 };
 use quickwit_janitor::{start_janitor_service, JanitorService};
 use quickwit_metastore::{
@@ -176,7 +176,7 @@ async fn start_ingest_client_if_needed(
         )
         .await?;
         let num_buckets = NonZeroUsize::new(60).expect("60 should be non-zero");
-        let initial_rate = ConstantRate::new(n_mib_bytes!(50), Duration::from_secs(1));
+        let initial_rate = ConstantRate::new(ByteSize::mib(50).as_u64(), Duration::from_secs(1));
         let rate_estimator = SmaRateEstimator::new(
             num_buckets,
             Duration::from_secs(10),
@@ -184,7 +184,7 @@ async fn start_ingest_client_if_needed(
         )
         .with_initial_rate(initial_rate);
         let memory_capacity = ingest_api_service.ask(GetMemoryCapacity).await?;
-        let min_rate = ConstantRate::new(n_mib_bytes!(1), Duration::from_millis(100));
+        let min_rate = ConstantRate::new(ByteSize::mib(1).as_u64(), Duration::from_millis(100));
         let rate_modulator = RateModulator::new(rate_estimator.clone(), memory_capacity, min_rate);
         let ingest_service = IngestServiceClient::tower()
             .ingest_layer(
@@ -572,6 +572,9 @@ async fn setup_ingest_v2(
             self_node_id.clone(),
             ingester_pool.clone(),
             &wal_dir_path,
+            config.ingest_api_config.max_queue_disk_usage,
+            config.ingest_api_config.max_queue_memory_usage,
+            RateLimiterSettings::default(),
             replication_factor,
         )
         .await?;
@@ -694,7 +697,7 @@ fn setup_indexer_pool(
                 {
                     let node_id = node.node_id().to_string();
                     let indexing_tasks = node.indexing_tasks().to_vec();
-
+                    let indexing_capacity = node.indexing_capacity();
                     if node.is_self_node() {
                         if let Some(indexing_service_clone) = indexing_service_clone_opt {
                             let client =
@@ -704,6 +707,7 @@ fn setup_indexer_pool(
                                 IndexerNodeInfo {
                                     client,
                                     indexing_tasks,
+                                    indexing_capacity,
                                 },
                             ))
                         } else {
@@ -721,6 +725,7 @@ fn setup_indexer_pool(
                             IndexerNodeInfo {
                                 client,
                                 indexing_tasks,
+                                indexing_capacity,
                             },
                         ))
                     }
diff --git a/quickwit/quickwit-storage/Cargo.toml b/quickwit/quickwit-storage/Cargo.toml
index fa8d3b06bb5..49e25554045 100644
--- a/quickwit/quickwit-storage/Cargo.toml
+++ b/quickwit/quickwit-storage/Cargo.toml
@@ -13,8 +13,8 @@ documentation = "https://quickwit.io/docs/"
 anyhow = { workspace = true }
 async-trait = { workspace = true }
 base64 = { workspace = true }
-byte-unit = { workspace = true }
 bytes = { workspace = true }
+bytesize = { workspace = true }
 fnv = { workspace = true }
 futures = { workspace = true }
 hyper = { workspace = true }
diff --git a/quickwit/quickwit-storage/src/split_cache/split_table.rs b/quickwit/quickwit-storage/src/split_cache/split_table.rs
index 1cbc9594ebc..a1a0a91ae5f 100644
--- a/quickwit/quickwit-storage/src/split_cache/split_table.rs
+++ b/quickwit/quickwit-storage/src/split_cache/split_table.rs
@@ -369,7 +369,7 @@ impl SplitTable {
         {
             return true;
         }
-        if self.on_disk_bytes > self.limits.max_num_bytes.get_bytes() {
+        if self.on_disk_bytes > self.limits.max_num_bytes.as_u64() {
             return true;
         }
         false
@@ -451,7 +451,7 @@ pub(crate) struct DownloadOpportunity {
 mod tests {
     use std::num::NonZeroU32;
 
-    use byte_unit::Byte;
+    use bytesize::ByteSize;
     use quickwit_common::uri::Uri;
     use quickwit_config::SplitCacheLimits;
     use ulid::Ulid;
@@ -471,7 +471,7 @@ mod tests {
     fn test_split_table() {
         let mut split_table = SplitTable::with_limits_and_existing_splits(
             SplitCacheLimits {
-                max_num_bytes: Byte::from_bytes(1000),
+                max_num_bytes: ByteSize::kb(1),
                 max_num_splits: NonZeroU32::new(1).unwrap(),
                 num_concurrent_downloads: NonZeroU32::new(1).unwrap(),
             },
@@ -490,7 +490,7 @@ mod tests {
     fn test_split_table_prefer_last_touched() {
         let mut split_table = SplitTable::with_limits_and_existing_splits(
             SplitCacheLimits {
-                max_num_bytes: Byte::from_bytes(1000),
+                max_num_bytes: ByteSize::kb(1),
                 max_num_splits: NonZeroU32::new(1).unwrap(),
                 num_concurrent_downloads: NonZeroU32::new(1).unwrap(),
             },
@@ -511,7 +511,7 @@ mod tests {
     fn test_split_table_prefer_start_download_prevent_new_report() {
         let mut split_table = SplitTable::with_limits_and_existing_splits(
             SplitCacheLimits {
-                max_num_bytes: Byte::from_bytes(1000),
+                max_num_bytes: ByteSize::kb(1),
                 max_num_splits: NonZeroU32::new(1).unwrap(),
                 num_concurrent_downloads: NonZeroU32::new(1).unwrap(),
             },
@@ -540,7 +540,7 @@ mod tests {
     fn test_eviction_due_to_size() {
         let mut split_table = SplitTable::with_limits_and_existing_splits(
             SplitCacheLimits {
-                max_num_bytes: Byte::from_bytes(1_000_000),
+                max_num_bytes: ByteSize::mb(1),
                 max_num_splits: NonZeroU32::new(30).unwrap(),
                 num_concurrent_downloads: NonZeroU32::new(1).unwrap(),
             },
@@ -577,7 +577,7 @@ mod tests {
     fn test_eviction_due_to_num_splits() {
         let mut split_table = SplitTable::with_limits_and_existing_splits(
             SplitCacheLimits {
-                max_num_bytes: Byte::from_bytes(10_000_000),
+                max_num_bytes: ByteSize::mb(10),
                 max_num_splits: NonZeroU32::new(5).unwrap(),
                 num_concurrent_downloads: NonZeroU32::new(1).unwrap(),
             },
@@ -611,7 +611,7 @@ mod tests {
     fn test_failed_download_can_be_re_reported() {
         let mut split_table = SplitTable::with_limits_and_existing_splits(
             SplitCacheLimits {
-                max_num_bytes: Byte::from_bytes(10_000_000),
+                max_num_bytes: ByteSize::mb(10),
                 max_num_splits: NonZeroU32::new(5).unwrap(),
                 num_concurrent_downloads: NonZeroU32::new(1).unwrap(),
             },
@@ -640,7 +640,7 @@ mod tests {
     fn test_split_table_truncate_candidates() {
         let mut split_table = SplitTable::with_limits_and_existing_splits(
             SplitCacheLimits {
-                max_num_bytes: Byte::from_bytes(10_000_000),
+                max_num_bytes: ByteSize::mb(10),
                 max_num_splits: NonZeroU32::new(5).unwrap(),
                 num_concurrent_downloads: NonZeroU32::new(1).unwrap(),
             },
diff --git a/quickwit/quickwit-storage/src/split_cache/tests.rs b/quickwit/quickwit-storage/src/split_cache/tests.rs
index 9c85add52e8..8a63809f0c7 100644
--- a/quickwit/quickwit-storage/src/split_cache/tests.rs
+++ b/quickwit/quickwit-storage/src/split_cache/tests.rs
@@ -19,7 +19,7 @@
 
 use std::num::NonZeroU32;
 
-use byte_unit::Byte;
+use bytesize::ByteSize;
 use quickwit_common::uri::Uri;
 use quickwit_config::SplitCacheLimits;
 use ulid::Ulid;
@@ -31,7 +31,7 @@ const TEST_STORAGE_URI: &'static str = "s3://test";
 #[test]
 fn test_split_table() {
     let mut split_table = SplitTable::with_limits(SplitCacheLimits {
-        max_num_bytes: Byte::from_bytes(1000),
+        max_num_bytes: ByteSize::kb(1),
         max_num_splits: NonZeroU32::new(1).unwrap(),
         num_concurrent_downloads: NonZeroU32::new(1).unwrap(),
     });
@@ -46,7 +46,7 @@ fn test_split_table() {
 #[test]
 fn test_split_table_prefer_last_touched() {
     let mut split_table = SplitTable::with_limits(SplitCacheLimits {
-        max_num_bytes: Byte::from_bytes(1000),
+        max_num_bytes: ByteSize::kb(1),
         max_num_splits: NonZeroU32::new(1).unwrap(),
         num_concurrent_downloads: NonZeroU32::new(1).unwrap(),
     });
@@ -63,7 +63,7 @@ fn test_split_table_prefer_last_touched() {
 #[test]
 fn test_split_table_prefer_start_download_prevent_new_report() {
     let mut split_table = SplitTable::with_limits(SplitCacheLimits {
-        max_num_bytes: Byte::from_bytes(1000),
+        max_num_bytes: ByteSize::kb(1),
         max_num_splits: NonZeroU32::new(1).unwrap(),
         num_concurrent_downloads: NonZeroU32::new(1).unwrap(),
     });
@@ -89,7 +89,7 @@ fn test_split_table_prefer_start_download_prevent_new_report() {
 #[test]
 fn test_eviction_due_to_size() {
     let mut split_table = SplitTable::with_limits(SplitCacheLimits {
-        max_num_bytes: Byte::from_bytes(1_000_000),
+        max_num_bytes: ByteSize::mb(1),
         max_num_splits: NonZeroU32::new(30).unwrap(),
         num_concurrent_downloads: NonZeroU32::new(1).unwrap(),
     });
@@ -123,7 +123,7 @@ fn test_eviction_due_to_size() {
 #[test]
 fn test_eviction_due_to_num_splits() {
     let mut split_table = SplitTable::with_limits(SplitCacheLimits {
-        max_num_bytes: Byte::from_bytes(10_000_000),
+        max_num_bytes: ByteSize::mb(10),
         max_num_splits: NonZeroU32::new(5).unwrap(),
         num_concurrent_downloads: NonZeroU32::new(1).unwrap(),
     });
@@ -154,7 +154,7 @@ fn test_eviction_due_to_num_splits() {
 #[test]
 fn test_failed_download_can_be_re_reported() {
     let mut split_table = SplitTable::with_limits(SplitCacheLimits {
-        max_num_bytes: Byte::from_bytes(10_000_000),
+        max_num_bytes: ByteSize::mb(10),
         max_num_splits: NonZeroU32::new(5).unwrap(),
         num_concurrent_downloads: NonZeroU32::new(1).unwrap(),
     });
diff --git a/quickwit/quickwit-ui/yarn.lock b/quickwit/quickwit-ui/yarn.lock
index 0502aec3e7b..44c7b899f12 100644
--- a/quickwit/quickwit-ui/yarn.lock
+++ b/quickwit/quickwit-ui/yarn.lock
@@ -4301,9 +4301,9 @@ axe-core@^4.4.3:
   integrity sha512-lCZN5XRuOnpG4bpMq8v0khrWtUOn+i8lZSb6wHZH56ZfbIEv6XwJV84AAueh9/zi7qPVJ/E4yz6fmsiyOmXR4w==
 
 axios@^1.4.0:
-  version "1.5.1"
-  resolved "https://registry.yarnpkg.com/axios/-/axios-1.5.1.tgz#11fbaa11fc35f431193a9564109c88c1f27b585f"
-  integrity sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==
+  version "1.6.1"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.1.tgz#76550d644bf0a2d469a01f9244db6753208397d7"
+  integrity sha512-vfBmhDpKafglh0EldBEbVuoe7DyAavGSLWhuSm5ZSEKQnHhBf0xAAwybbNH1IkrJNGnS/VG4I5yxig1pCEXE4g==
   dependencies:
     follow-redirects "^1.15.0"
     form-data "^4.0.0"
diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/0005-query_string.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/0005-query_string_query.yaml
similarity index 75%
rename from quickwit/rest-api-tests/scenarii/es_compatibility/0005-query_string.yaml
rename to quickwit/rest-api-tests/scenarii/es_compatibility/0005-query_string_query.yaml
index cb3caf4fcf2..099ee2bc5d2 100644
--- a/quickwit/rest-api-tests/scenarii/es_compatibility/0005-query_string.yaml
+++ b/quickwit/rest-api-tests/scenarii/es_compatibility/0005-query_string_query.yaml
@@ -151,3 +151,51 @@ json:
       lenient: true
       query: "to AND the"
 status_code: 400
+---
+# trailing wildcard
+json:
+  query:
+    query_string:
+      default_field: payload.description
+      lenient: true
+      query: "Jour* AND unix"
+expected:
+  hits:
+    total:
+      value: 2
+---
+# trailing wildcard
+json:
+  query:
+    query_string:
+      default_field: payload.description
+      lenient: true
+      query: "jour* AND unix"
+expected:
+  hits:
+    total:
+      value: 2
+---
+# trailing wildcard
+json:
+  query:
+    query_string:
+      default_field: payload.description
+      lenient: true
+      query: "jour*"
+expected:
+  hits:
+    total:
+      value: 3
+---
+# trailing wildcard
+json:
+  query:
+    query_string:
+      default_field: payload.description
+      lenient: true
+      query: "jour\\*"
+expected:
+  hits:
+    total:
+      value: 1
diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/0007-range_queries.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/0007-range_queries.yaml
index a66284f6754..c3c625395c4 100644
--- a/quickwit/rest-api-tests/scenarii/es_compatibility/0007-range_queries.yaml
+++ b/quickwit/rest-api-tests/scenarii/es_compatibility/0007-range_queries.yaml
@@ -159,3 +159,31 @@ expected:
     total:
       value: 86
       relation: "eq"
+---
+# Timestamp field with milliseconds precision 2015-02-01T00:00:00.001
+json:
+  query:
+    range:
+      created_at:
+        gte: "2015-02-01T00:00:00.001Z"
+        lt: "2015-02-01T00:00:00.002Z"
+expected:
+  hits:
+    total:
+      value: 1
+      relation: "eq"
+---
+# Timestamp field with range in microseconds.
+# Datetime will be truncated at milliseconds as
+# defined in the doc mapper.
+json:
+  query:
+    range:
+      created_at:
+        gte: "2015-02-01T00:00:00.001999Z"
+        lte: "2015-02-01T00:00:00.001999Z"
+expected:
+  hits:
+    total:
+      value: 1
+      relation: "eq"
diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.quickwit.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.quickwit.yaml
index 24089348d94..5eea75dd277 100644
--- a/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.quickwit.yaml
+++ b/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.quickwit.yaml
@@ -27,6 +27,7 @@ json:
         - name: created_at
           type: datetime
           fast: true
+          fast_precision: milliseconds
     dynamic_mapping:
       expand_dots: true
       tokenizer: default
diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/gharchive-bulk.json.gz b/quickwit/rest-api-tests/scenarii/es_compatibility/gharchive-bulk.json.gz
index 1cb0a99376e..9b4bb284afc 100644
Binary files a/quickwit/rest-api-tests/scenarii/es_compatibility/gharchive-bulk.json.gz and b/quickwit/rest-api-tests/scenarii/es_compatibility/gharchive-bulk.json.gz differ