From bf9dbd48a6c3703c04bc270e7fb90e14913816a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:37:34 +0000 Subject: [PATCH 01/27] Bump vrl from v0.3.0 to v0.8.1 in /quickwit (#4080) * Bump vrl from v0.3.0 to v0.8.1 in /quickwit Bumps [vrl](https://github.com/vectordotdev/vrl) from v0.3.0 to v0.8.1. - [Changelog](https://github.com/vectordotdev/vrl/blob/main/CHANGELOG.md) - [Commits](https://github.com/vectordotdev/vrl/compare/113005bcee6cd7b5ea0a53a7db2fc45ba4bc4125...854852100d0ad7f52f970d2843bad09d3055ee56) --- updated-dependencies: - dependency-name: vrl dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Add bump VRL fixes --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Adrien Guillo --- quickwit/Cargo.lock | 458 +++++++++++------- quickwit/Cargo.toml | 168 +------ quickwit/quickwit-config/Cargo.toml | 5 +- .../quickwit-config/src/source_config/mod.rs | 4 +- quickwit/quickwit-indexing/Cargo.toml | 7 +- 5 files changed, 297 insertions(+), 345 deletions(-) diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 704c0470e3a..2d4cc4b9184 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -33,6 +33,16 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + [[package]] name = "aes" version = "0.8.3" @@ -1170,6 +1180,30 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chacha20" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + [[package]] name = "charset" version = "0.1.3" @@ -1214,9 +1248,9 @@ dependencies = [ [[package]] name = "chrono-tz" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1369bc6b9e9a7dfdae2055f6ec151fe9c554a9d23d357c0237cee2e25eaabb7" +checksum = "e23185c0e21df6ed832a12e2bda87c7d1def6842881fb634a8511ced741b0d76" dependencies = [ "chrono", "chrono-tz-build", @@ -1225,9 +1259,9 @@ dependencies = [ [[package]] name = "chrono-tz-build" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2f5ebdc942f57ed96d560a6d1a459bae5851102a25d5bf89dc04ae453e31ecf" +checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f" dependencies = [ "parse-zoneinfo", "phf", @@ -1282,6 +1316,7 @@ checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common", "inout", + "zeroize", ] [[package]] @@ -1353,6 +1388,20 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "community-id" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f6af96839c04974cf381e427792a99913ecf3f7bfb348f153dc8a8e5f9803ad" +dependencies = [ + "anyhow", + "base64 0.21.5", + "hex", + "lazy_static", + "num_enum 0.6.1", + "sha1", +] + [[package]] name = "concurrent-queue" version = "2.3.0" @@ -1602,9 +1651,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] +[[package]] +name = "crypto_secretbox" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d6cf87adf719ddf43a805e92c6870a531aedda35ff640442cbaf8674e141e1" +dependencies = [ + "aead", + "cipher", + "generic-array", + "poly1305", + "salsa20", + "subtle", + "zeroize", +] + [[package]] name = "csv" version = "1.3.0" @@ -1775,17 +1840,6 @@ dependencies = [ "serde", ] -[[package]] -name = "derivative" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "derive_more" version = "0.99.17" @@ -1858,14 +1912,14 @@ dependencies = [ [[package]] name = "dns-lookup" -version = "1.0.8" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53ecafc952c4528d9b51a458d1a8904b81783feff9fde08ab6ed2545ff396872" +checksum = "e5766087c2235fec47fafa4cfecc81e494ee679d0fd4a59887ea0919bfb0e4fc" dependencies = [ "cfg-if", "libc", - "socket2 0.4.10", - "winapi 0.3.9", + "socket2 0.5.5", + "windows-sys 0.48.0", ] [[package]] @@ -2408,6 +2462,7 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -2613,6 +2668,16 @@ dependencies = [ "async-trait", ] +[[package]] +name = "grok" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "273797968160270573071022613fc4aa28b91fe68f3eef6c96a1b2a1947ddfbd" +dependencies = [ + "glob", + "onig", +] + [[package]] name = "h2" version = "0.3.21" @@ -3178,9 +3243,9 @@ dependencies = [ [[package]] name = "lalrpop" -version = "0.19.12" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a1cbf952127589f2851ab2046af368fd20645491bb4b376f04b7f94d7a9837b" +checksum = "da4081d44f4611b66c6dd725e6de3169f9f63905421e8626fcb86b6a898998b8" dependencies = [ "ascii-canvas", "bit-set", @@ -3191,7 +3256,7 @@ dependencies = [ "lalrpop-util", "petgraph", "regex", - "regex-syntax 0.6.29", + "regex-syntax 0.7.5", "string_cache", "term", "tiny-keccak", @@ -3200,9 +3265,9 @@ dependencies = [ [[package]] name = "lalrpop-util" -version = "0.19.12" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3c48237b9604c5a4702de6b824e02006c3214327564636aef27c1028a8fa0ed" +checksum = "3f35c735096c0293d313e8f2a641627472b83d01b937177fe76e5e2708d31e0d" [[package]] name = "lazy_static" @@ -3905,7 +3970,16 @@ version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9" dependencies = [ - "num_enum_derive", + "num_enum_derive 0.5.11", +] + +[[package]] +name = "num_enum" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a015b430d3c108a207fd776d2e2196aaf8b1cf8cf93253e3a097ff3085076a1" +dependencies = [ + "num_enum_derive 0.6.1", ] [[package]] @@ -3920,6 +3994,18 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "num_enum_derive" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96667db765a921f7b295ffee8b60472b686a51d4f21c2ee4ffdb94c7013b65a6" +dependencies = [ + "proc-macro-crate 1.3.1", + "proc-macro2", + "quote", + "syn 2.0.38", +] + [[package]] name = "num_threads" version = "0.1.6" @@ -3988,12 +4074,40 @@ dependencies = [ "loom", ] +[[package]] +name = "onig" +version = "6.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" +dependencies = [ + "bitflags 1.3.2", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "oorandom" version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + [[package]] name = "openidconnect" version = "2.5.1" @@ -4165,9 +4279,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "3.9.2" +version = "4.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc" +checksum = "536900a8093134cf9ccf00a27deb3532421099e958d9dd431135d0c7543ca1e8" dependencies = [ "num-traits", ] @@ -4236,7 +4350,7 @@ dependencies = [ "ansi-str", "bytecount", "fnv", - "strip-ansi-escapes", + "strip-ansi-escapes 0.1.1", "unicode-width", ] @@ -4295,17 +4409,6 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" -[[package]] -name = "path" -version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.3.0#113005bcee6cd7b5ea0a53a7db2fc45ba4bc4125" -dependencies = [ - "once_cell", - "regex", - "serde", - "snafu", -] - [[package]] name = "pbkdf2" version = "0.11.0" @@ -4318,6 +4421,12 @@ dependencies = [ "sha2", ] +[[package]] +name = "peeking_take_while" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9ed2178b0575fff8e1b83b58ba6f75e727aafac2e1b6c795169ad3b17eb518" + [[package]] name = "pem" version = "1.1.1" @@ -4342,6 +4451,51 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +[[package]] +name = "pest" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81d78524685f5ef2a3b3bd1cafbc9fcabb036253d9b1463e726a91cd16e2dfc2" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68bd1206e71118b5356dae5ddc61c8b11e28b09ef6a31acbd15ea48a28e0c227" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.38", +] + +[[package]] +name = "pest_meta" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c747191d4ad9e4a4ab9c8798f1e82a39affe7ef9648390b7e5548d18e099de6" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + [[package]] name = "petgraph" version = "0.6.4" @@ -4577,6 +4731,17 @@ dependencies = [ "pnet_sys", ] +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.5.0" @@ -5149,7 +5314,6 @@ dependencies = [ "tracing", "utoipa", "vrl", - "vrl-stdlib", ] [[package]] @@ -5355,7 +5519,6 @@ dependencies = [ "ulid", "utoipa", "vrl", - "vrl-stdlib", ] [[package]] @@ -5853,9 +6016,9 @@ dependencies = [ [[package]] name = "quoted_printable" -version = "0.4.8" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3866219251662ec3b26fc217e3e05bf9c4f84325234dfb96bf0bf840889e49" +checksum = "79ec282e887b434b68c18fe5c121d38e72a5cf35119b59e54ec5b992ea9c8eb0" [[package]] name = "radium" @@ -6000,7 +6163,7 @@ dependencies = [ "cmake", "libc", "libz-sys", - "num_enum", + "num_enum 0.5.11", "openssl-sys", "pkg-config", "sasl2-sys", @@ -6447,6 +6610,15 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -6599,9 +6771,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ "itoa", "ryu", @@ -7212,7 +7384,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "011cbb39cf7c1f62871aea3cc46e5817b0937b49e9447370c93cacbe93a766d8" dependencies = [ - "vte", + "vte 0.10.1", +] + +[[package]] +name = "strip-ansi-escapes" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ff8ef943b384c414f54aefa961dd2bd853add74ec75e7ac74cf91dba62bcfa" +dependencies = [ + "vte 0.11.1", ] [[package]] @@ -7257,9 +7438,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "syslog_loose" -version = "0.18.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fb75f176928530867b2a659e470f9c9ff71904695bab6556f7ad30f9039efd" +checksum = "acf5252d1adec0a489a0225f867c1a7fd445e41674530a396d0629cff0c4b211" dependencies = [ "chrono", "nom", @@ -8108,6 +8289,12 @@ dependencies = [ "serde_yaml 0.8.26", ] +[[package]] +name = "ucd-trie" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" + [[package]] name = "ulid" version = "1.1.0" @@ -8178,6 +8365,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + [[package]] name = "unsafe-libyaml" version = "0.2.9" @@ -8281,6 +8478,7 @@ dependencies = [ "getrandom 0.2.10", "rand 0.8.5", "serde", + "wasm-bindgen", ] [[package]] @@ -8289,23 +8487,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "value" -version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.3.0#113005bcee6cd7b5ea0a53a7db2fc45ba4bc4125" -dependencies = [ - "bytes", - "chrono", - "once_cell", - "ordered-float 3.9.2", - "path", - "regex", - "serde", - "serde_json", - "snafu", - "tracing", -] - [[package]] name = "vcpkg" version = "0.2.15" @@ -8320,113 +8501,52 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "vrl" -version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.3.0#113005bcee6cd7b5ea0a53a7db2fc45ba4bc4125" -dependencies = [ - "value", - "vrl-compiler", - "vrl-diagnostic", -] - -[[package]] -name = "vrl-compiler" -version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.3.0#113005bcee6cd7b5ea0a53a7db2fc45ba4bc4125" -dependencies = [ - "anymap", - "bytes", - "chrono", - "chrono-tz", - "dyn-clone", - "getrandom 0.2.10", - "indoc", - "lalrpop-util", - "ordered-float 3.9.2", - "paste", - "path", - "regex", - "serde", - "snafu", - "thiserror", - "tracing", - "value", - "vrl-diagnostic", - "vrl-parser", -] - -[[package]] -name = "vrl-core" -version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.3.0#113005bcee6cd7b5ea0a53a7db2fc45ba4bc4125" -dependencies = [ - "bytes", - "chrono", - "chrono-tz", - "derivative", - "nom", - "ordered-float 3.9.2", - "path", - "serde", - "serde_json", - "snafu", - "value", - "vrl-diagnostic", -] - -[[package]] -name = "vrl-diagnostic" -version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.3.0#113005bcee6cd7b5ea0a53a7db2fc45ba4bc4125" -dependencies = [ - "codespan-reporting", - "termcolor", -] - -[[package]] -name = "vrl-parser" -version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.3.0#113005bcee6cd7b5ea0a53a7db2fc45ba4bc4125" -dependencies = [ - "lalrpop", - "lalrpop-util", - "ordered-float 3.9.2", - "paste", - "path", - "thiserror", - "vrl-diagnostic", -] - -[[package]] -name = "vrl-stdlib" -version = "0.1.0" -source = "git+https://github.com/vectordotdev/vrl?rev=v0.3.0#113005bcee6cd7b5ea0a53a7db2fc45ba4bc4125" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a93ee342590c4df0ff63961d7d76a347e0c7b6e6c0be4c001317ca1ff11b53" dependencies = [ "aes", + "anymap", "base16", "base64 0.21.5", "bytes", "cbc", "cfb-mode", + "cfg-if", + "chacha20poly1305", "charset", "chrono", + "chrono-tz", "cidr-utils", + "codespan-reporting", + "community-id", + "crypto_secretbox", "csv", "ctr", "data-encoding", "dns-lookup", + "dyn-clone", "flate2", + "grok", "hex", "hmac", "hostname", - "indexmap 1.9.3", + "indexmap 2.1.0", "indoc", + "itertools 0.11.0", + "lalrpop", + "lalrpop-util", "md-5", "nom", "ofb", "once_cell", - "ordered-float 3.9.2", - "path", + "onig", + "ordered-float 4.1.1", + "paste", + "peeking_take_while", "percent-encoding", + "pest", + "pest_derive", "quoted_printable", "rand 0.8.5", "regex", @@ -8438,19 +8558,18 @@ dependencies = [ "sha-1", "sha2", "sha3", - "strip-ansi-escapes", + "snafu", + "strip-ansi-escapes 0.2.0", "syslog_loose", + "termcolor", + "thiserror", "tracing", "uaparser", "url", "utf8-width", "uuid", - "value", - "vrl-compiler", - "vrl-core", - "vrl-diagnostic", "woothee", - "zstd 0.12.4", + "zstd 0.13.0", ] [[package]] @@ -8470,6 +8589,16 @@ dependencies = [ "vte_generate_state_changes", ] +[[package]] +name = "vte" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" +dependencies = [ + "utf8parse", + "vte_generate_state_changes", +] + [[package]] name = "vte_generate_state_changes" version = "0.1.1" @@ -9047,15 +9176,6 @@ dependencies = [ "zstd-safe 5.0.2+zstd.1.5.2", ] -[[package]] -name = "zstd" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" -dependencies = [ - "zstd-safe 6.0.6", -] - [[package]] name = "zstd" version = "0.13.0" @@ -9075,16 +9195,6 @@ dependencies = [ "zstd-sys", ] -[[package]] -name = "zstd-safe" -version = "6.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" -dependencies = [ - "libc", - "zstd-sys", -] - [[package]] name = "zstd-safe" version = "7.0.0" diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index 418bb8fa72e..893052bfa4d 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -187,6 +187,12 @@ ulid = "1.1" username = "0.2" utoipa = "3.5.0" uuid = { version = "1.5", features = ["v4", "serde"] } +vrl = { version = "0.8.1", default-features = false, features = [ + "compiler", + "diagnostic", + "stdlib", + "value", +] } warp = "0.3" whichlang = { git = "https://github.com/quickwit-oss/whichlang", rev = "fe406416" } wiremock = "0.5" @@ -245,168 +251,6 @@ tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "ecb9a89", d # used by reqwest. encoding_rs = "=0.8.32" -# vrl deps, at the end because the feature list for vrl-stdlib is long -vrl = { git = "https://github.com/vectordotdev/vrl", rev = "v0.3.0", default-features = false, features = [ - "compiler", - "diagnostic", - "value", -] } -# opt out of vrl-stdlib parse-grok* default features -vrl-stdlib = { git = "https://github.com/vectordotdev/vrl", rev = "v0.3.0", default-features = false, features = [ - "abs", - "append", - "array", - "assert", - "assert_eq", - "boolean", - "ceil", - "chunks", - "compact", - "contains", - "decode_base16", - "decode_base64", - "decode_gzip", - "decode_mime_q", - "decode_percent", - "decode_zlib", - "decode_zstd", - "decrypt", - "del", - "downcase", - "encode_base16", - "encode_base64", - "encode_gzip", - "encode_json", - "encode_key_value", - "encode_logfmt", - "encode_percent", - "encode_zlib", - "encode_zstd", - "encrypt", - "ends_with", - "exists", - "filter", - "find", - "flatten", - "float", - "floor", - "for_each", - "format_int", - "format_number", - "format_timestamp", - "get", - "get_env_var", - "get_hostname", - "hmac", - "includes", - "integer", - "ip_aton", - "ip_cidr_contains", - "ip_ntoa", - "ip_ntop", - "ip_pton", - "ip_subnet", - "ip_to_ipv6", - "ipv6_to_ipv4", - "is_array", - "is_boolean", - "is_empty", - "is_float", - "is_integer", - "is_ipv4", - "is_ipv6", - "is_json", - "is_null", - "is_nullish", - "is_object", - "is_regex", - "is_string", - "is_timestamp", - "join", - "keys", - "length", - "log", - "map_keys", - "map_values", - "match", - "match_any", - "match_array", - "md5", - "merge", - "mod", - "now", - "object", - "parse_apache_log", - "parse_aws_alb_log", - "parse_aws_cloudwatch_log_subscription_message", - "parse_aws_vpc_flow_log", - "parse_cef", - "parse_common_log", - "parse_csv", - "parse_duration", - "parse_glog", - "parse_int", - "parse_json", - "parse_key_value", - "parse_klog", - "parse_linux_authorization", - "parse_logfmt", - "parse_nginx_log", - "parse_query_string", - "parse_regex", - "parse_regex_all", - "parse_ruby_hash", - "parse_syslog", - "parse_timestamp", - "parse_tokens", - "parse_url", - "parse_user_agent", - "parse_xml", - "push", - "random_bool", - "random_bytes", - "random_float", - "random_int", - "redact", - "remove", - "replace", - "reverse_dns", - "round", - "seahash", - "set", - "sha1", - "sha2", - "sha3", - "slice", - "split", - "starts_with", - "string", - "strip_ansi_escape_codes", - "strip_whitespace", - "strlen", - "tag_types_externally", - "tally", - "tally_value", - "timestamp", - "to_bool", - "to_float", - "to_int", - "to_regex", - "to_string", - "to_syslog_facility", - "to_syslog_level", - "to_syslog_severity", - "to_timestamp", - "to_unix_timestamp", - "truncate", - "type_def", - "unique", - "unnest", - "upcase", - "uuid_v4", - "values" -]} - [patch.crates-io] sasl2-sys = { git = "https://github.com/quickwit-oss/rust-sasl/", rev = "daca921" } diff --git a/quickwit/quickwit-config/Cargo.toml b/quickwit/quickwit-config/Cargo.toml index cf3b2eea720..b39ad94b660 100644 --- a/quickwit/quickwit-config/Cargo.toml +++ b/quickwit/quickwit-config/Cargo.toml @@ -29,8 +29,7 @@ serde_yaml = { workspace = true } toml = { workspace = true } tracing = { workspace = true } utoipa = { workspace = true } -vrl = { workspace = true, optional=true } -vrl-stdlib = { workspace = true, optional=true } +vrl = { workspace = true, optional = true } quickwit-common = { workspace = true } quickwit-doc-mapper = { workspace = true } @@ -42,4 +41,4 @@ tokio = { workspace = true } [features] testsuite = [] -vrl = ["dep:vrl", "vrl-stdlib"] +vrl = ["dep:vrl"] diff --git a/quickwit/quickwit-config/src/source_config/mod.rs b/quickwit/quickwit-config/src/source_config/mod.rs index 0fdcb7945b4..bda2e987f8c 100644 --- a/quickwit/quickwit-config/src/source_config/mod.rs +++ b/quickwit/quickwit-config/src/source_config/mod.rs @@ -510,7 +510,7 @@ impl TransformConfig { // Append "\n." to the script to return the entire document and not only the modified // fields. let vrl_script = self.vrl_script.clone() + "\n."; - let functions = vrl_stdlib::all(); + let functions = vrl::stdlib::all(); let compilation_res = match vrl::compiler::compile(&vrl_script, &functions) { Ok(compilation_res) => compilation_res, @@ -1161,7 +1161,7 @@ mod tests { let transform_config = TransformConfig { vrl_script: r#" . = parse_json!(string!(.message)) - .timestamp = to_unix_timestamp(to_timestamp!(.timestamp)) + .timestamp = to_unix_timestamp(timestamp!(.timestamp)) del(.username) .message = downcase(string!(.message)) "# diff --git a/quickwit/quickwit-indexing/Cargo.toml b/quickwit/quickwit-indexing/Cargo.toml index c4280731836..e1ff51e155e 100644 --- a/quickwit/quickwit-indexing/Cargo.toml +++ b/quickwit/quickwit-indexing/Cargo.toml @@ -50,7 +50,6 @@ tracing = { workspace = true } ulid = { workspace = true } utoipa = { workspace = true } vrl = { workspace = true, optional = true } -vrl-stdlib = { workspace = true, optional = true } quickwit-actors = { workspace = true } quickwit-aws = { workspace = true } @@ -70,18 +69,18 @@ gcp-pubsub = ["dep:google-cloud-pubsub", "dep:google-cloud-default", "dep:google gcp-pubsub-emulator-tests = [] kafka = ["rdkafka", "backoff"] kafka-broker-tests = [] -vendored-kafka = ["kafka", "libz-sys/static", "openssl/vendored", "rdkafka/gssapi-vendored"] -vendored-kafka-macos = ["kafka", "libz-sys/static", "openssl/vendored"] kinesis = ["aws-config", "aws-smithy-client", "aws-sdk-kinesis", "quickwit-aws/kinesis"] kinesis-localstack-tests = [] -vrl = ["dep:vrl", "vrl-stdlib", "quickwit-config/vrl"] pulsar = ["dep:pulsar"] pulsar-broker-tests = [] +vendored-kafka = ["kafka", "libz-sys/static", "openssl/vendored", "rdkafka/gssapi-vendored"] +vendored-kafka-macos = ["kafka", "libz-sys/static", "openssl/vendored"] testsuite = [ "quickwit-actors/testsuite", "quickwit-cluster/testsuite", "quickwit-common/testsuite", ] +vrl = ["dep:vrl", "quickwit-config/vrl"] [dev-dependencies] bytes = { workspace = true } From f570e111b175a0568a38880a503ceffa6cf50535 Mon Sep 17 00:00:00 2001 From: Adrien Guillo Date: Mon, 6 Nov 2023 12:40:02 -0500 Subject: [PATCH 02/27] Make `sea-query` deps optional (#4091) --- quickwit/quickwit-metastore/Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/quickwit/quickwit-metastore/Cargo.toml b/quickwit/quickwit-metastore/Cargo.toml index 26cfa4d9f5c..e2353120b14 100644 --- a/quickwit/quickwit-metastore/Cargo.toml +++ b/quickwit/quickwit-metastore/Cargo.toml @@ -20,12 +20,12 @@ mockall = { workspace = true, optional = true } once_cell = { workspace = true } rand = { workspace = true } regex = { workspace = true } +sea-query = { workspace = true, optional = true } +sea-query-binder = { workspace = true, optional = true } serde = { workspace = true } serde_json = { workspace = true } serde_with = { workspace = true } sqlx = { workspace = true, optional = true } -sea-query = { workspace = true } -sea-query-binder = { workspace = true } tempfile = { workspace = true, optional = true } thiserror = { workspace = true } time = { workspace = true } @@ -60,5 +60,5 @@ quickwit-storage = { workspace = true, features = ["testsuite"] } [features] azure = ["quickwit-storage/azure"] ci-test = [] -postgres = ["quickwit-proto/postgres", "sqlx"] +postgres = ["quickwit-proto/postgres", "sea-query", "sea-query-binder", "sqlx"] testsuite = ["mockall", "tempfile", "quickwit-config/testsuite"] From cdb3bb7f4627c1c6e6967ccc6c6885d2555c1b1f Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Mon, 6 Nov 2023 19:14:08 +0100 Subject: [PATCH 03/27] implement trailing wildcard support (#4059) * Added regression test * improvements on prefix queries allow prefix query to match any term, not the lexicographic first 50 (ES compat) don't preload position when they are not needed don't error when position are missing but not actually required internalize all tokenizer instead of using tantivy default tokenizer * create QueryAst::Wildcard * use net TokenizerManager which tracks if an analyzer does lowercasing * extract prefix from wildcard query * wire WildcardQuery in UserInputQuery * add license headers * change QwTokenizerManager api * handle escaped wildcards * document bool prefix max_expansions * add integration test for wildcard escape --------- Co-authored-by: Paul Masurel --- .../src/default_doc_mapper/default_mapper.rs | 30 ++- .../quickwit-doc-mapper/src/doc_mapper.rs | 2 +- .../quickwit-doc-mapper/src/query_builder.rs | 29 +-- .../quickwit-doc-mapper/src/tag_pruning.rs | 10 +- .../quickwit-indexing/src/actors/indexer.rs | 8 +- .../src/actors/merge_executor.rs | 38 +++- .../quickwit-indexing/src/actors/packager.rs | 10 +- .../benches/multilang_tokenizers_bench.rs | 8 +- .../src/query_ast/bool_query.rs | 2 +- .../src/query_ast/field_presence.rs | 2 +- .../src/query_ast/full_text_query.rs | 13 +- quickwit/quickwit-query/src/query_ast/mod.rs | 15 +- .../src/query_ast/phrase_prefix_query.rs | 31 ++- .../src/query_ast/range_query.rs | 2 +- .../src/query_ast/term_query.rs | 2 +- .../src/query_ast/term_set_query.rs | 2 +- .../src/query_ast/user_input_query.rs | 51 ++++- .../quickwit-query/src/query_ast/utils.rs | 9 +- .../quickwit-query/src/query_ast/visitor.rs | 6 + .../src/query_ast/wildcard_query.rs | 203 ++++++++++++++++++ .../src/tokenizers/chinese_compatible.rs | 12 +- quickwit/quickwit-query/src/tokenizers/mod.rs | 53 +++-- .../src/tokenizers/tokenizer_manager.rs | 103 +++++++++ quickwit/quickwit-search/src/collector.rs | 2 +- quickwit/quickwit-search/src/leaf.rs | 8 +- ...ring.yaml => 0005-query_string_query.yaml} | 48 +++++ .../es_compatibility/gharchive-bulk.json.gz | Bin 33660 -> 33652 bytes 27 files changed, 601 insertions(+), 98 deletions(-) create mode 100644 quickwit/quickwit-query/src/query_ast/wildcard_query.rs create mode 100644 quickwit/quickwit-query/src/tokenizers/tokenizer_manager.rs rename quickwit/rest-api-tests/scenarii/es_compatibility/{0005-query_string.yaml => 0005-query_string_query.yaml} (75%) diff --git a/quickwit/quickwit-doc-mapper/src/default_doc_mapper/default_mapper.rs b/quickwit/quickwit-doc-mapper/src/default_doc_mapper/default_mapper.rs index a8438fea678..cf4bf30a9d9 100644 --- a/quickwit/quickwit-doc-mapper/src/default_doc_mapper/default_mapper.rs +++ b/quickwit/quickwit-doc-mapper/src/default_doc_mapper/default_mapper.rs @@ -25,13 +25,13 @@ use fnv::FnvHashSet; use quickwit_common::PathHasher; use quickwit_query::create_default_quickwit_tokenizer_manager; use quickwit_query::query_ast::QueryAst; +use quickwit_query::tokenizers::TokenizerManager; use serde::{Deserialize, Serialize}; use serde_json::{self, Value as JsonValue}; use tantivy::query::Query; use tantivy::schema::{ Field, FieldType, FieldValue, OwnedValue as TantivyValue, Schema, INDEXED, STORED, }; -use tantivy::tokenizer::TokenizerManager; use tantivy::TantivyDocument as Document; use super::field_mapping_entry::RAW_TOKENIZER_NAME; @@ -178,7 +178,7 @@ impl TryFrom for DefaultDocMapper { ); } if tokenizer_manager - .get(&tokenizer_config_entry.name) + .get_tokenizer(&tokenizer_config_entry.name) .is_some() { bail!( @@ -197,7 +197,12 @@ impl TryFrom for DefaultDocMapper { error ) })?; - tokenizer_manager.register(&tokenizer_config_entry.name, tokenizer); + let does_lowercasing = tokenizer_config_entry + .config + .filters + .iter() + .any(|filter| matches!(filter, crate::TokenFilterType::LowerCaser)); + tokenizer_manager.register(&tokenizer_config_entry.name, tokenizer, does_lowercasing); custom_tokenizer_names.insert(&tokenizer_config_entry.name); } validate_fields_tokenizers(&schema, &tokenizer_manager)?; @@ -329,7 +334,7 @@ fn validate_fields_tokenizers( _ => None, }; if let Some(tokenizer_name) = tokenizer_name_opt { - if tokenizer_manager.get(tokenizer_name).is_none() { + if tokenizer_manager.get_tokenizer(tokenizer_name).is_none() { bail!( "unknown tokenizer `{}` for field `{}`", tokenizer_name, @@ -1857,7 +1862,10 @@ mod tests { } _ => panic!("Expected a text field"), } - assert!(mapper.tokenizer_manager().get("my_tokenizer").is_some()); + assert!(mapper + .tokenizer_manager() + .get_tokenizer("my_tokenizer") + .is_some()); } #[test] @@ -1902,7 +1910,10 @@ mod tests { }"#, ) .unwrap(); - let mut tokenizer = mapper.tokenizer_manager().get("my_tokenizer").unwrap(); + let mut tokenizer = mapper + .tokenizer_manager() + .get_tokenizer("my_tokenizer") + .unwrap(); let mut token_stream = tokenizer.token_stream("HELLO WORLD"); assert_eq!(token_stream.next().unwrap().text, "hel"); assert_eq!(token_stream.next().unwrap().text, "hell"); @@ -1959,8 +1970,11 @@ mod tests { }"#, ) .unwrap(); - let mut default_tokenizer = mapper.tokenizer_manager().get("default").unwrap(); - let mut tokenizer = mapper.tokenizer_manager().get("my_tokenizer").unwrap(); + let mut default_tokenizer = mapper.tokenizer_manager().get_tokenizer("default").unwrap(); + let mut tokenizer = mapper + .tokenizer_manager() + .get_tokenizer("my_tokenizer") + .unwrap(); let text = "I've seen things... seen things you little people wouldn't believe."; let mut default_token_stream = default_tokenizer.token_stream(text); let mut token_stream = tokenizer.token_stream(text); diff --git a/quickwit/quickwit-doc-mapper/src/doc_mapper.rs b/quickwit/quickwit-doc-mapper/src/doc_mapper.rs index bc81198375d..e47101707d6 100644 --- a/quickwit/quickwit-doc-mapper/src/doc_mapper.rs +++ b/quickwit/quickwit-doc-mapper/src/doc_mapper.rs @@ -25,10 +25,10 @@ use std::ops::Bound; use anyhow::Context; use dyn_clone::{clone_trait_object, DynClone}; use quickwit_query::query_ast::QueryAst; +use quickwit_query::tokenizers::TokenizerManager; use serde_json::Value as JsonValue; use tantivy::query::Query; use tantivy::schema::{Field, FieldType, OwnedValue as Value, Schema}; -use tantivy::tokenizer::TokenizerManager; use tantivy::{TantivyDocument as Document, Term}; pub type Partition = u64; diff --git a/quickwit/quickwit-doc-mapper/src/query_builder.rs b/quickwit/quickwit-doc-mapper/src/query_builder.rs index b0532ec0d9e..d2cf5edc3c7 100644 --- a/quickwit/quickwit-doc-mapper/src/query_builder.rs +++ b/quickwit/quickwit-doc-mapper/src/query_builder.rs @@ -22,13 +22,13 @@ use std::convert::Infallible; use std::ops::Bound; use quickwit_query::query_ast::{ - FieldPresenceQuery, FullTextMode, FullTextQuery, PhrasePrefixQuery, QueryAst, QueryAstVisitor, - RangeQuery, TermSetQuery, + FieldPresenceQuery, FullTextQuery, PhrasePrefixQuery, QueryAst, QueryAstVisitor, RangeQuery, + TermSetQuery, WildcardQuery, }; +use quickwit_query::tokenizers::TokenizerManager; use quickwit_query::{find_field_or_hit_dynamic, InvalidQuery}; use tantivy::query::Query; use tantivy::schema::{Field, Schema}; -use tantivy::tokenizer::TokenizerManager; use tantivy::Term; use crate::{QueryParserError, TermRange, WarmupInfo}; @@ -215,16 +215,13 @@ impl<'a, 'b: 'a> QueryAstVisitor<'a> for ExtractPrefixTermRanges<'b> { type Err = InvalidQuery; fn visit_full_text(&mut self, full_text_query: &'a FullTextQuery) -> Result<(), Self::Err> { - if let FullTextMode::BoolPrefix { - operator: _, - max_expansions, - } = &full_text_query.params.mode + if let Some(prefix_term) = + full_text_query.get_prefix_term(self.schema, self.tokenizer_manager) { - if let Some(prefix_term) = - full_text_query.get_last_term(self.schema, self.tokenizer_manager) - { - self.add_prefix_term(prefix_term, *max_expansions, false); - } + // the max_expansion expansion of a bool prefix query is used for the fuzzy part of the + // query, not for the expension to a range request. + // see https://github.com/elastic/elasticsearch/blob/6ad48306d029e6e527c0481e2e9880bd2f06b239/docs/reference/query-dsl/match-bool-prefix-query.asciidoc#parameters + self.add_prefix_term(prefix_term, u32::MAX, false); } Ok(()) } @@ -235,10 +232,16 @@ impl<'a, 'b: 'a> QueryAstVisitor<'a> for ExtractPrefixTermRanges<'b> { ) -> Result<(), Self::Err> { let (_, terms) = phrase_prefix.get_terms(self.schema, self.tokenizer_manager)?; if let Some((_, term)) = terms.last() { - self.add_prefix_term(term.clone(), phrase_prefix.max_expansions, true); + self.add_prefix_term(term.clone(), phrase_prefix.max_expansions, terms.len() > 1); } Ok(()) } + + fn visit_wildcard(&mut self, wildcard_query: &'a WildcardQuery) -> Result<(), Self::Err> { + let (_, term) = wildcard_query.extract_prefix_term(self.schema, self.tokenizer_manager)?; + self.add_prefix_term(term, u32::MAX, false); + Ok(()) + } } fn extract_prefix_term_ranges( diff --git a/quickwit/quickwit-doc-mapper/src/tag_pruning.rs b/quickwit/quickwit-doc-mapper/src/tag_pruning.rs index 78184a0194d..a41ecfd4795 100644 --- a/quickwit/quickwit-doc-mapper/src/tag_pruning.rs +++ b/quickwit/quickwit-doc-mapper/src/tag_pruning.rs @@ -98,13 +98,21 @@ fn extract_unsimplified_tags_filter_ast(query_ast: QueryAst) -> UnsimplifiedTagF } } QueryAst::PhrasePrefix(phrase_prefix_query) => { - // TODO same as Phrase above. + // TODO same as FullText above. UnsimplifiedTagFilterAst::Tag { is_present: true, field: phrase_prefix_query.field, value: phrase_prefix_query.phrase, } } + QueryAst::Wildcard(wildcard_query) => { + // TODO same as FullText above. + UnsimplifiedTagFilterAst::Tag { + is_present: true, + field: wildcard_query.field, + value: wildcard_query.value, + } + } QueryAst::Boost { underlying, .. } => extract_unsimplified_tags_filter_ast(*underlying), QueryAst::UserInput(_user_text_query) => { panic!("Extract unsimplified should only be called on AST without UserInputQuery."); diff --git a/quickwit/quickwit-indexing/src/actors/indexer.rs b/quickwit/quickwit-indexing/src/actors/indexer.rs index 6c6c3b8cbc9..1c7bad7f183 100644 --- a/quickwit/quickwit-indexing/src/actors/indexer.rs +++ b/quickwit/quickwit-indexing/src/actors/indexer.rs @@ -110,7 +110,11 @@ impl IndexerState { .settings(self.index_settings.clone()) .schema(self.schema.clone()) .tokenizers(self.tokenizer_manager.clone()) - .fast_field_tokenizers(get_quickwit_fastfield_normalizer_manager().clone()); + .fast_field_tokenizers( + get_quickwit_fastfield_normalizer_manager() + .tantivy_manager() + .clone(), + ); let io_controls = IoControls::default() .set_progress(ctx.progress().clone()) @@ -534,7 +538,7 @@ impl Indexer { publish_lock: PublishLock::default(), publish_token_opt: None, schema, - tokenizer_manager, + tokenizer_manager: tokenizer_manager.tantivy_manager().clone(), index_settings, max_num_partitions: doc_mapper.max_num_partitions(), cooperative_indexing_permits, diff --git a/quickwit/quickwit-indexing/src/actors/merge_executor.rs b/quickwit/quickwit-indexing/src/actors/merge_executor.rs index ec616fe8192..d4ffb1fdf60 100644 --- a/quickwit/quickwit-indexing/src/actors/merge_executor.rs +++ b/quickwit/quickwit-indexing/src/actors/merge_executor.rs @@ -296,8 +296,10 @@ impl MergeExecutor { merge_scratch_directory: TempDirectory, ctx: &ActorContext, ) -> anyhow::Result { - let (union_index_meta, split_directories) = - open_split_directories(&tantivy_dirs, self.doc_mapper.tokenizer_manager())?; + let (union_index_meta, split_directories) = open_split_directories( + &tantivy_dirs, + self.doc_mapper.tokenizer_manager().tantivy_manager(), + )?; // TODO it would be nice if tantivy could let us run the merge in the current thread. fail_point!("before-merge-split"); let controlled_directory = self @@ -316,7 +318,7 @@ impl MergeExecutor { // splits. let merged_index = open_index( controlled_directory.clone(), - self.doc_mapper.tokenizer_manager(), + self.doc_mapper.tokenizer_manager().tantivy_manager(), )?; ctx.record_progress(); @@ -362,8 +364,10 @@ impl MergeExecutor { num_delete_tasks = delete_tasks.len() ); - let (union_index_meta, split_directories) = - open_split_directories(&tantivy_dirs, self.doc_mapper.tokenizer_manager())?; + let (union_index_meta, split_directories) = open_split_directories( + &tantivy_dirs, + self.doc_mapper.tokenizer_manager().tantivy_manager(), + )?; let controlled_directory = self .merge_split_directories( union_index_meta, @@ -378,8 +382,17 @@ impl MergeExecutor { // This will have the side effect of deleting the directory containing the downloaded split. let mut merged_index = Index::open(controlled_directory.clone())?; ctx.record_progress(); - merged_index.set_tokenizers(self.doc_mapper.tokenizer_manager().clone()); - merged_index.set_fast_field_tokenizers(get_quickwit_fastfield_normalizer_manager().clone()); + merged_index.set_tokenizers( + self.doc_mapper + .tokenizer_manager() + .tantivy_manager() + .clone(), + ); + merged_index.set_fast_field_tokenizers( + get_quickwit_fastfield_normalizer_manager() + .tantivy_manager() + .clone(), + ); ctx.record_progress(); @@ -470,7 +483,10 @@ impl MergeExecutor { ]; directory_stack.extend(split_directories.into_iter()); let union_directory = UnionDirectory::union_of(directory_stack); - let union_index = open_index(union_directory, self.doc_mapper.tokenizer_manager())?; + let union_index = open_index( + union_directory, + self.doc_mapper.tokenizer_manager().tantivy_manager(), + )?; ctx.record_progress(); let _protect_guard = ctx.protect_zone(); @@ -532,7 +548,11 @@ fn open_index>>( ) -> tantivy::Result { let mut index = Index::open(directory)?; index.set_tokenizers(tokenizer_manager.clone()); - index.set_fast_field_tokenizers(get_quickwit_fastfield_normalizer_manager().clone()); + index.set_fast_field_tokenizers( + get_quickwit_fastfield_normalizer_manager() + .tantivy_manager() + .clone(), + ); Ok(index) } diff --git a/quickwit/quickwit-indexing/src/actors/packager.rs b/quickwit/quickwit-indexing/src/actors/packager.rs index 7ba54b82b08..26639e2d6bf 100644 --- a/quickwit/quickwit-indexing/src/actors/packager.rs +++ b/quickwit/quickwit-indexing/src/actors/packager.rs @@ -368,9 +368,15 @@ mod tests { let index_builder = IndexBuilder::new() .settings(IndexSettings::default()) .schema(schema) - .tokenizers(quickwit_query::create_default_quickwit_tokenizer_manager()) + .tokenizers( + quickwit_query::create_default_quickwit_tokenizer_manager() + .tantivy_manager() + .clone(), + ) .fast_field_tokenizers( - quickwit_query::get_quickwit_fastfield_normalizer_manager().clone(), + quickwit_query::get_quickwit_fastfield_normalizer_manager() + .tantivy_manager() + .clone(), ); let index_directory = MmapDirectory::open(split_scratch_directory.path())?; let mut index_writer = diff --git a/quickwit/quickwit-query/benches/multilang_tokenizers_bench.rs b/quickwit/quickwit-query/benches/multilang_tokenizers_bench.rs index 0f4a5b7496f..705cbbb3520 100644 --- a/quickwit/quickwit-query/benches/multilang_tokenizers_bench.rs +++ b/quickwit/quickwit-query/benches/multilang_tokenizers_bench.rs @@ -69,9 +69,11 @@ fn process_tokens(analyzer: &mut TextAnalyzer, text: &str) -> Vec { pub fn tokenizers_throughput_benchmark(c: &mut Criterion) { let mut group = c.benchmark_group("multilang"); let tokenizer_manager = create_default_quickwit_tokenizer_manager(); - let mut default_tokenizer = tokenizer_manager.get("default").unwrap(); - let mut multilang_tokenizer = tokenizer_manager.get("multilang").unwrap(); - let mut chinese_tokenizer = tokenizer_manager.get("chinese_compatible").unwrap(); + let mut default_tokenizer = tokenizer_manager.get_tokenizer("default").unwrap(); + let mut multilang_tokenizer = tokenizer_manager.get_tokenizer("multilang").unwrap(); + let mut chinese_tokenizer = tokenizer_manager + .get_tokenizer("chinese_compatible") + .unwrap(); group .throughput(Throughput::Bytes(ASCII_SHORT.len() as u64)) diff --git a/quickwit/quickwit-query/src/query_ast/bool_query.rs b/quickwit/quickwit-query/src/query_ast/bool_query.rs index 30ec9cac473..9394aaa030a 100644 --- a/quickwit/quickwit-query/src/query_ast/bool_query.rs +++ b/quickwit/quickwit-query/src/query_ast/bool_query.rs @@ -19,10 +19,10 @@ use serde::{Deserialize, Serialize}; use tantivy::schema::Schema as TantivySchema; -use tantivy::tokenizer::TokenizerManager; use super::{BuildTantivyAst, TantivyQueryAst}; use crate::query_ast::QueryAst; +use crate::tokenizers::TokenizerManager; use crate::InvalidQuery; /// # Unsupported features diff --git a/quickwit/quickwit-query/src/query_ast/field_presence.rs b/quickwit/quickwit-query/src/query_ast/field_presence.rs index be68c6a8d38..0f5eaabd070 100644 --- a/quickwit/quickwit-query/src/query_ast/field_presence.rs +++ b/quickwit/quickwit-query/src/query_ast/field_presence.rs @@ -21,11 +21,11 @@ use quickwit_common::shared_consts::FIELD_PRESENCE_FIELD_NAME; use quickwit_common::PathHasher; use serde::{Deserialize, Serialize}; use tantivy::schema::{Field, IndexRecordOption, Schema as TantivySchema}; -use tantivy::tokenizer::TokenizerManager; use tantivy::Term; use crate::query_ast::tantivy_query_ast::TantivyQueryAst; use crate::query_ast::{BuildTantivyAst, QueryAst}; +use crate::tokenizers::TokenizerManager; use crate::{find_field_or_hit_dynamic, InvalidQuery}; #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] diff --git a/quickwit/quickwit-query/src/query_ast/full_text_query.rs b/quickwit/quickwit-query/src/query_ast/full_text_query.rs index be2e74dba43..a3ca2929caa 100644 --- a/quickwit/quickwit-query/src/query_ast/full_text_query.rs +++ b/quickwit/quickwit-query/src/query_ast/full_text_query.rs @@ -28,12 +28,13 @@ use tantivy::schema::{ Field, FieldType, IndexRecordOption, JsonObjectOptions, Schema as TantivySchema, TextFieldIndexing, }; -use tantivy::tokenizer::{TextAnalyzer, TokenStream, TokenizerManager}; +use tantivy::tokenizer::{TextAnalyzer, TokenStream}; use tantivy::Term; use crate::query_ast::tantivy_query_ast::{TantivyBoolQuery, TantivyQueryAst}; use crate::query_ast::utils::full_text_query; use crate::query_ast::{BuildTantivyAst, QueryAst}; +use crate::tokenizers::TokenizerManager; use crate::{find_field_or_hit_dynamic, BooleanOperand, InvalidQuery, MatchAllOrNone}; #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Clone)] @@ -59,7 +60,7 @@ impl FullTextParams { .as_deref() .unwrap_or(text_field_indexing.tokenizer()); tokenizer_manager - .get(tokenizer_name) + .get_tokenizer(tokenizer_name) .with_context(|| format!("no tokenizer named `{}` is registered", tokenizer_name)) } @@ -182,6 +183,8 @@ pub enum FullTextMode { }, BoolPrefix { operator: BooleanOperand, + // max_expansions correspond to the fuzzy stop of query evalution. It's not the same as the + // max_expansions of a PhrasePrefixQuery, where it's used for the range expansion. max_expansions: u32, }, // Act as Phrase with slop 0 if the field has positions, @@ -255,11 +258,15 @@ impl FullTextQuery { /// /// This strange method is used to identify which term range should be warmed up for /// phrase prefix queries. - pub fn get_last_term( + pub fn get_prefix_term( &self, schema: &TantivySchema, tokenizer_manager: &TokenizerManager, ) -> Option { + if !matches!(self.params.mode, FullTextMode::BoolPrefix { .. }) { + return None; + }; + let (field, field_entry, json_path) = find_field_or_hit_dynamic(&self.field, schema).ok()?; let field_type: &FieldType = field_entry.field_type(); diff --git a/quickwit/quickwit-query/src/query_ast/mod.rs b/quickwit/quickwit-query/src/query_ast/mod.rs index 50696d6a1aa..e815248b16f 100644 --- a/quickwit/quickwit-query/src/query_ast/mod.rs +++ b/quickwit/quickwit-query/src/query_ast/mod.rs @@ -20,7 +20,8 @@ use serde::{Deserialize, Serialize}; use tantivy::query::BoostQuery as TantivyBoostQuery; use tantivy::schema::Schema as TantivySchema; -use tantivy::tokenizer::TokenizerManager; + +use crate::tokenizers::TokenizerManager; mod bool_query; mod field_presence; @@ -33,6 +34,7 @@ mod term_set_query; mod user_input_query; pub(crate) mod utils; mod visitor; +mod wildcard_query; pub use bool_query::BoolQuery; pub use field_presence::FieldPresenceQuery; @@ -44,6 +46,7 @@ pub use term_query::TermQuery; pub use term_set_query::TermSetQuery; pub use user_input_query::UserInputQuery; pub use visitor::QueryAstVisitor; +pub use wildcard_query::WildcardQuery; use crate::{BooleanOperand, InvalidQuery, NotNaNf32}; @@ -59,6 +62,7 @@ pub enum QueryAst { PhrasePrefix(PhrasePrefixQuery), Range(RangeQuery), UserInput(UserInputQuery), + Wildcard(WildcardQuery), MatchAll, MatchNone, Boost { @@ -98,7 +102,8 @@ impl QueryAst { | ast @ QueryAst::MatchAll | ast @ QueryAst::MatchNone | ast @ QueryAst::FieldPresence(_) - | ast @ QueryAst::Range(_) => Ok(ast), + | ast @ QueryAst::Range(_) + | ast @ QueryAst::Wildcard(_) => Ok(ast), QueryAst::UserInput(user_text_query) => { user_text_query.parse_user_query(default_search_fields) } @@ -236,6 +241,12 @@ impl BuildTantivyAst for QueryAst { search_fields, with_validation, ), + QueryAst::Wildcard(wildcard) => wildcard.build_tantivy_ast_call( + schema, + tokenizer_manager, + search_fields, + with_validation, + ), } } } diff --git a/quickwit/quickwit-query/src/query_ast/phrase_prefix_query.rs b/quickwit/quickwit-query/src/query_ast/phrase_prefix_query.rs index ace1f2b1ad8..c7d67ea0617 100644 --- a/quickwit/quickwit-query/src/query_ast/phrase_prefix_query.rs +++ b/quickwit/quickwit-query/src/query_ast/phrase_prefix_query.rs @@ -20,11 +20,11 @@ use serde::{Deserialize, Serialize}; use tantivy::query::PhrasePrefixQuery as TantivyPhrasePrefixQuery; use tantivy::schema::{Field, FieldType, Schema as TantivySchema}; -use tantivy::tokenizer::TokenizerManager; use tantivy::Term; use crate::query_ast::tantivy_query_ast::TantivyQueryAst; use crate::query_ast::{BuildTantivyAst, FullTextParams, QueryAst}; +use crate::tokenizers::TokenizerManager; use crate::{find_field_or_hit_dynamic, InvalidQuery}; /// The PhraseQuery node is meant to be tokenized and searched. @@ -57,20 +57,19 @@ impl PhrasePrefixQuery { field_entry.name() )) })?; - if !text_field_indexing.index_option().has_positions() { - return Err(InvalidQuery::SchemaError( - "trying to run a phrase prefix query on a field which does not have \ - positions indexed" - .to_string(), - )); - } - let terms = self.params.tokenize_text_into_terms( field, &self.phrase, text_field_indexing, tokenizer_manager, )?; + if !text_field_indexing.index_option().has_positions() && terms.len() > 1 { + return Err(InvalidQuery::SchemaError( + "trying to run a phrase prefix query on a field which does not have \ + positions indexed" + .to_string(), + )); + } Ok((field, terms)) } FieldType::JsonObject(json_options) => { @@ -81,13 +80,6 @@ impl PhrasePrefixQuery { field_entry.name() )) })?; - if !text_field_indexing.index_option().has_positions() { - return Err(InvalidQuery::SchemaError( - "trying to run a PhrasePrefix query on a field which does not have \ - positions indexed" - .to_string(), - )); - } let terms = self.params.tokenize_text_into_terms_json( field, json_path, @@ -95,6 +87,13 @@ impl PhrasePrefixQuery { json_options, tokenizer_manager, )?; + if !text_field_indexing.index_option().has_positions() && terms.len() > 1 { + return Err(InvalidQuery::SchemaError( + "trying to run a PhrasePrefix query on a field which does not have \ + positions indexed" + .to_string(), + )); + } Ok((field, terms)) } _ => Err(InvalidQuery::SchemaError( diff --git a/quickwit/quickwit-query/src/query_ast/range_query.rs b/quickwit/quickwit-query/src/query_ast/range_query.rs index a053675821a..ef5d654af8a 100644 --- a/quickwit/quickwit-query/src/query_ast/range_query.rs +++ b/quickwit/quickwit-query/src/query_ast/range_query.rs @@ -24,12 +24,12 @@ use tantivy::query::{ FastFieldRangeWeight as TantivyFastFieldRangeQuery, RangeQuery as TantivyRangeQuery, }; use tantivy::schema::Schema as TantivySchema; -use tantivy::tokenizer::TokenizerManager; use super::QueryAst; use crate::json_literal::InterpretUserInput; use crate::query_ast::tantivy_query_ast::{TantivyBoolQuery, TantivyQueryAst}; use crate::query_ast::BuildTantivyAst; +use crate::tokenizers::TokenizerManager; use crate::{InvalidQuery, JsonLiteral}; #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] diff --git a/quickwit/quickwit-query/src/query_ast/term_query.rs b/quickwit/quickwit-query/src/query_ast/term_query.rs index 0f62da9544d..cb1129fc69f 100644 --- a/quickwit/quickwit-query/src/query_ast/term_query.rs +++ b/quickwit/quickwit-query/src/query_ast/term_query.rs @@ -21,10 +21,10 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; use tantivy::schema::Schema as TantivySchema; -use tantivy::tokenizer::TokenizerManager; use super::{BuildTantivyAst, QueryAst}; use crate::query_ast::{FullTextParams, TantivyQueryAst}; +use crate::tokenizers::TokenizerManager; use crate::{BooleanOperand, InvalidQuery}; /// The TermQuery acts exactly like a FullTextQuery with diff --git a/quickwit/quickwit-query/src/query_ast/term_set_query.rs b/quickwit/quickwit-query/src/query_ast/term_set_query.rs index d27597e37bc..b9c93ba36e5 100644 --- a/quickwit/quickwit-query/src/query_ast/term_set_query.rs +++ b/quickwit/quickwit-query/src/query_ast/term_set_query.rs @@ -21,10 +21,10 @@ use std::collections::{BTreeSet, HashMap, HashSet}; use serde::{Deserialize, Serialize}; use tantivy::schema::Schema as TantivySchema; -use tantivy::tokenizer::TokenizerManager; use tantivy::Term; use crate::query_ast::{BuildTantivyAst, QueryAst, TantivyQueryAst, TermQuery}; +use crate::tokenizers::TokenizerManager; use crate::InvalidQuery; /// TermSetQuery matches the same document set as if it was a union of diff --git a/quickwit/quickwit-query/src/query_ast/user_input_query.rs b/quickwit/quickwit-query/src/query_ast/user_input_query.rs index cab0d9cea9a..910e37ae254 100644 --- a/quickwit/quickwit-query/src/query_ast/user_input_query.rs +++ b/quickwit/quickwit-query/src/query_ast/user_input_query.rs @@ -26,13 +26,13 @@ use tantivy::query_grammar::{ Delimiter, Occur, UserInputAst, UserInputBound, UserInputLeaf, UserInputLiteral, }; use tantivy::schema::Schema as TantivySchema; -use tantivy::tokenizer::TokenizerManager; use crate::not_nan_f32::NotNaNf32; use crate::query_ast::tantivy_query_ast::TantivyQueryAst; use crate::query_ast::{ self, BuildTantivyAst, FieldPresenceQuery, FullTextMode, FullTextParams, QueryAst, }; +use crate::tokenizers::TokenizerManager; use crate::{BooleanOperand, InvalidQuery, JsonLiteral}; const DEFAULT_PHRASE_QUERY_MAX_EXPANSION: u32 = 50; @@ -182,6 +182,31 @@ fn convert_user_input_ast_to_query_ast( } } +fn is_wildcard(phrase: &str) -> bool { + use std::ops::ControlFlow; + enum State { + Normal, + Escaped, + } + + phrase + .chars() + .try_fold(State::Normal, |state, c| match state { + State::Escaped => ControlFlow::Continue(State::Normal), + State::Normal => { + if c == '*' || c == '?' { + // we are in a wildcard query + ControlFlow::Break(()) + } else if c == '\\' { + ControlFlow::Continue(State::Escaped) + } else { + ControlFlow::Continue(State::Normal) + } + } + }) + .is_break() +} + fn convert_user_input_literal( user_input_literal: UserInputLiteral, default_search_fields: &[String], @@ -216,24 +241,32 @@ fn convert_user_input_literal( mode, zero_terms_query: crate::MatchAllOrNone::MatchNone, }; + let wildcard = delimiter == Delimiter::None && is_wildcard(&phrase); let mut phrase_queries: Vec = field_names .into_iter() .map(|field_name| { if prefix { - return query_ast::PhrasePrefixQuery { + query_ast::PhrasePrefixQuery { field: field_name, phrase: phrase.clone(), params: full_text_params.clone(), max_expansions: DEFAULT_PHRASE_QUERY_MAX_EXPANSION, } - .into(); - } - query_ast::FullTextQuery { - field: field_name, - text: phrase.clone(), - params: full_text_params.clone(), + .into() + } else if wildcard { + query_ast::WildcardQuery { + field: field_name, + value: phrase.clone(), + } + .into() + } else { + query_ast::FullTextQuery { + field: field_name, + text: phrase.clone(), + params: full_text_params.clone(), + } + .into() } - .into() }) .collect(); if phrase_queries.is_empty() { diff --git a/quickwit/quickwit-query/src/query_ast/utils.rs b/quickwit/quickwit-query/src/query_ast/utils.rs index 8033424c158..b6be400f393 100644 --- a/quickwit/quickwit-query/src/query_ast/utils.rs +++ b/quickwit/quickwit-query/src/query_ast/utils.rs @@ -23,11 +23,12 @@ use tantivy::schema::{ Field, FieldEntry, FieldType, IndexRecordOption, JsonObjectOptions, Schema as TantivySchema, Type, }; -use tantivy::{tokenizer, Term}; +use tantivy::Term; use crate::json_literal::InterpretUserInput; use crate::query_ast::full_text_query::FullTextParams; use crate::query_ast::tantivy_query_ast::{TantivyBoolQuery, TantivyQueryAst}; +use crate::tokenizers::TokenizerManager; use crate::InvalidQuery; const DYNAMIC_FIELD_NAME: &str = "_dynamic"; @@ -75,7 +76,7 @@ pub(crate) fn full_text_query( text_query: &str, full_text_params: &FullTextParams, schema: &TantivySchema, - tokenizer_manager: &tokenizer::TokenizerManager, + tokenizer_manager: &TokenizerManager, ) -> Result { let (field, field_entry, path) = find_field_or_hit_dynamic(full_path, schema)?; compute_query_with_field( @@ -108,7 +109,7 @@ fn compute_query_with_field( json_path: &str, value: &str, full_text_params: &FullTextParams, - tokenizer_manager: &tokenizer::TokenizerManager, + tokenizer_manager: &TokenizerManager, ) -> Result { let field_type = field_entry.field_type(); match field_type { @@ -182,7 +183,7 @@ fn compute_tantivy_ast_query_for_json( text: &str, full_text_params: &FullTextParams, json_options: &JsonObjectOptions, - tokenizer_manager: &tokenizer::TokenizerManager, + tokenizer_manager: &TokenizerManager, ) -> Result { let mut bool_query = TantivyBoolQuery::default(); let mut term = Term::with_capacity(100); diff --git a/quickwit/quickwit-query/src/query_ast/visitor.rs b/quickwit/quickwit-query/src/query_ast/visitor.rs index 398f8d9852d..8c60153ac9d 100644 --- a/quickwit/quickwit-query/src/query_ast/visitor.rs +++ b/quickwit/quickwit-query/src/query_ast/visitor.rs @@ -22,6 +22,7 @@ use crate::query_ast::field_presence::FieldPresenceQuery; use crate::query_ast::user_input_query::UserInputQuery; use crate::query_ast::{ BoolQuery, FullTextQuery, PhrasePrefixQuery, QueryAst, RangeQuery, TermQuery, TermSetQuery, + WildcardQuery, }; /// Simple trait to implement a Visitor over the QueryAst. @@ -43,6 +44,7 @@ pub trait QueryAstVisitor<'a> { QueryAst::Boost { underlying, boost } => self.visit_boost(underlying, *boost), QueryAst::UserInput(user_text_query) => self.visit_user_text(user_text_query), QueryAst::FieldPresence(exists) => self.visit_exists(exists), + QueryAst::Wildcard(wildcard) => self.visit_wildcard(wildcard), } } @@ -105,4 +107,8 @@ pub trait QueryAstVisitor<'a> { fn visit_exists(&mut self, _exists_query: &'a FieldPresenceQuery) -> Result<(), Self::Err> { Ok(()) } + + fn visit_wildcard(&mut self, _wildcard_query: &'a WildcardQuery) -> Result<(), Self::Err> { + Ok(()) + } } diff --git a/quickwit/quickwit-query/src/query_ast/wildcard_query.rs b/quickwit/quickwit-query/src/query_ast/wildcard_query.rs new file mode 100644 index 00000000000..6e69d242fbe --- /dev/null +++ b/quickwit/quickwit-query/src/query_ast/wildcard_query.rs @@ -0,0 +1,203 @@ +// Copyright (C) 2023 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use anyhow::{anyhow, bail, Context}; +use serde::{Deserialize, Serialize}; +use tantivy::json_utils::JsonTermWriter; +use tantivy::schema::{Field, FieldType, Schema as TantivySchema}; +use tantivy::Term; + +use super::{BuildTantivyAst, QueryAst}; +use crate::query_ast::TantivyQueryAst; +use crate::tokenizers::TokenizerManager; +use crate::{find_field_or_hit_dynamic, InvalidQuery}; + +/// A Wildcard query allows to match 'bond' with a query like 'b*d'. +/// +/// At the moment, only wildcard at end of term is supported. +#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)] +pub struct WildcardQuery { + pub field: String, + pub value: String, +} + +impl From for QueryAst { + fn from(wildcard_query: WildcardQuery) -> Self { + Self::Wildcard(wildcard_query) + } +} + +impl WildcardQuery { + #[cfg(test)] + pub fn from_field_value(field: impl ToString, value: impl ToString) -> Self { + Self { + field: field.to_string(), + value: value.to_string(), + } + } +} + +fn extract_unique_token(mut tokens: Vec) -> anyhow::Result { + let term = tokens + .pop() + .with_context(|| "wildcard query generated no term")?; + if !tokens.is_empty() { + anyhow::bail!("wildcard query generated more than one term"); + } + Ok(term) +} + +fn unescape_with_final_wildcard(phrase: &str) -> anyhow::Result { + enum State { + Normal, + Escaped, + } + + // we keep this state outside of scan because we want to query if after + let mut saw_wildcard = false; + let saw_wildcard = &mut saw_wildcard; + + let phrase = phrase + .chars() + .scan(State::Normal, |state, c| { + if *saw_wildcard { + return Some(Some(Err(anyhow!( + "Wildcard iquery contains wildcard in non final position" + )))); + } + match state { + State::Escaped => { + *state = State::Normal; + Some(Some(Ok(c))) + } + State::Normal => { + if c == '*' { + *saw_wildcard = true; + Some(None) + } else if c == '\\' { + *state = State::Escaped; + Some(None) + } else if c == '?' { + Some(Some(Err(anyhow!("Wildcard query contains `?`")))) + } else { + Some(Some(Ok(c))) + } + } + } + }) + // we have an iterator of Option> + .flatten() + // we have an iterator of Result + .collect::>()?; + if !*saw_wildcard { + bail!("Wildcard query doesn't contain a wildcard"); + } + Ok(phrase) +} + +impl WildcardQuery { + // TODO this method will probably disappear once we support the full semantic of + // wildcard queries + pub fn extract_prefix_term( + &self, + schema: &TantivySchema, + tokenizer_manager: &TokenizerManager, + ) -> Result<(Field, Term), InvalidQuery> { + let (field, field_entry, json_path) = find_field_or_hit_dynamic(&self.field, schema)?; + let field_type = field_entry.field_type(); + + let prefix = unescape_with_final_wildcard(&self.value)?; + + match field_type { + FieldType::Str(ref text_options) => { + let text_field_indexing = text_options.get_indexing_options().ok_or_else(|| { + InvalidQuery::SchemaError(format!( + "field {} is not full-text searchable", + field_entry.name() + )) + })?; + let tokenizer_name = text_field_indexing.tokenizer(); + let mut normalizer = tokenizer_manager + .get_normalizer(tokenizer_name) + .with_context(|| { + format!("no tokenizer named `{}` is registered", tokenizer_name) + })?; + let mut token_stream = normalizer.token_stream(&prefix); + let mut tokens = Vec::new(); + token_stream.process(&mut |token| { + let term: Term = Term::from_field_text(field, &token.text); + tokens.push(term); + }); + let term = extract_unique_token(tokens)?; + Ok((field, term)) + } + FieldType::JsonObject(json_options) => { + let text_field_indexing = + json_options.get_text_indexing_options().ok_or_else(|| { + InvalidQuery::SchemaError(format!( + "field {} is not full-text searchable", + field_entry.name() + )) + })?; + let tokenizer_name = text_field_indexing.tokenizer(); + let mut normalizer = tokenizer_manager + .get_normalizer(tokenizer_name) + .with_context(|| { + format!("no tokenizer named `{}` is registered", tokenizer_name) + })?; + let mut token_stream = normalizer.token_stream(&prefix); + let mut tokens = Vec::new(); + let mut term = Term::with_capacity(100); + let mut json_term_writer = JsonTermWriter::from_field_and_json_path( + field, + json_path, + json_options.is_expand_dots_enabled(), + &mut term, + ); + + token_stream.process(&mut |token| { + json_term_writer.set_str(&token.text); + tokens.push(json_term_writer.term().clone()); + }); + let term = extract_unique_token(tokens)?; + Ok((field, term)) + } + _ => Err(InvalidQuery::SchemaError( + "trying to run a Wildcard query on a non-text field".to_string(), + )), + } + } +} + +impl BuildTantivyAst for WildcardQuery { + fn build_tantivy_ast_impl( + &self, + schema: &TantivySchema, + tokenizer_manager: &TokenizerManager, + _search_fields: &[String], + _with_validation: bool, + ) -> Result { + let (_, term) = self.extract_prefix_term(schema, tokenizer_manager)?; + + let mut phrase_prefix_query = + tantivy::query::PhrasePrefixQuery::new_with_offset(vec![(0, term)]); + phrase_prefix_query.set_max_expansions(u32::MAX); + Ok(phrase_prefix_query.into()) + } +} diff --git a/quickwit/quickwit-query/src/tokenizers/chinese_compatible.rs b/quickwit/quickwit-query/src/tokenizers/chinese_compatible.rs index 1f326081e11..785d3c4a33f 100644 --- a/quickwit/quickwit-query/src/tokenizers/chinese_compatible.rs +++ b/quickwit/quickwit-query/src/tokenizers/chinese_compatible.rs @@ -135,7 +135,9 @@ mod tests { fn test_chinese_tokenizer() { let text = "Hello world, 你好世界, bonjour monde"; let tokenizer_manager = crate::create_default_quickwit_tokenizer_manager(); - let mut tokenizer = tokenizer_manager.get("chinese_compatible").unwrap(); + let mut tokenizer = tokenizer_manager + .get_tokenizer("chinese_compatible") + .unwrap(); let mut text_stream = tokenizer.token_stream(text); let mut res = Vec::new(); @@ -210,7 +212,9 @@ mod tests { fn test_chinese_tokenizer_no_space() { let text = "Hello你好bonjour"; let tokenizer_manager = crate::create_default_quickwit_tokenizer_manager(); - let mut tokenizer = tokenizer_manager.get("chinese_compatible").unwrap(); + let mut tokenizer = tokenizer_manager + .get_tokenizer("chinese_compatible") + .unwrap(); let mut text_stream = tokenizer.token_stream(text); let mut res = Vec::new(); @@ -256,8 +260,8 @@ mod tests { #[test] fn test_proptest_ascii_default_chinese_equal(text in "[ -~]{0,64}") { let tokenizer_manager = crate::create_default_quickwit_tokenizer_manager(); - let mut cn_tok = tokenizer_manager.get("chinese_compatible").unwrap(); - let mut default_tok = tokenizer_manager.get("default").unwrap(); + let mut cn_tok = tokenizer_manager.get_tokenizer("chinese_compatible").unwrap(); + let mut default_tok = tokenizer_manager.get_tokenizer("default").unwrap(); let mut text_stream = cn_tok.token_stream(&text); diff --git a/quickwit/quickwit-query/src/tokenizers/mod.rs b/quickwit/quickwit-query/src/tokenizers/mod.rs index 07a27c86689..4b70eb6cce2 100644 --- a/quickwit/quickwit-query/src/tokenizers/mod.rs +++ b/quickwit/quickwit-query/src/tokenizers/mod.rs @@ -21,32 +21,57 @@ mod chinese_compatible; mod code_tokenizer; #[cfg(feature = "multilang")] mod multilang; +mod tokenizer_manager; use once_cell::sync::Lazy; use tantivy::tokenizer::{ - AsciiFoldingFilter, LowerCaser, RawTokenizer, RemoveLongFilter, TextAnalyzer, TokenizerManager, + AsciiFoldingFilter, Language, LowerCaser, RawTokenizer, RemoveLongFilter, SimpleTokenizer, + Stemmer, TextAnalyzer, WhitespaceTokenizer, }; use self::chinese_compatible::ChineseTokenizer; pub use self::code_tokenizer::CodeTokenizer; #[cfg(feature = "multilang")] pub use self::multilang::MultiLangTokenizer; +pub use self::tokenizer_manager::TokenizerManager; pub const DEFAULT_REMOVE_TOKEN_LENGTH: usize = 255; /// Quickwit's tokenizer/analyzer manager. pub fn create_default_quickwit_tokenizer_manager() -> TokenizerManager { - let tokenizer_manager = TokenizerManager::default(); + let tokenizer_manager = TokenizerManager::new(); let raw_tokenizer = TextAnalyzer::builder(RawTokenizer::default()) .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH)) .build(); - tokenizer_manager.register("raw", raw_tokenizer); + tokenizer_manager.register("raw", raw_tokenizer, false); + + let lower_case_tokenizer = TextAnalyzer::builder(RawTokenizer::default()) + .filter(LowerCaser) + .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH)) + .build(); + tokenizer_manager.register("lowercase", lower_case_tokenizer, true); + + let default_tokenizer = TextAnalyzer::builder(SimpleTokenizer::default()) + .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH)) + .filter(LowerCaser) + .build(); + tokenizer_manager.register("default", default_tokenizer, true); + + let en_stem_tokenizer = TextAnalyzer::builder(SimpleTokenizer::default()) + .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH)) + .filter(LowerCaser) + .filter(Stemmer::new(Language::English)) + .build(); + tokenizer_manager.register("en_stem", en_stem_tokenizer, true); + + tokenizer_manager.register("whitespace", WhitespaceTokenizer::default(), false); + let chinese_tokenizer = TextAnalyzer::builder(ChineseTokenizer) .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH)) .filter(LowerCaser) .build(); - tokenizer_manager.register("chinese_compatible", chinese_tokenizer); + tokenizer_manager.register("chinese_compatible", chinese_tokenizer, false); tokenizer_manager.register( "source_code_default", TextAnalyzer::builder(CodeTokenizer::default()) @@ -54,6 +79,7 @@ pub fn create_default_quickwit_tokenizer_manager() -> TokenizerManager { .filter(LowerCaser) .filter(AsciiFoldingFilter) .build(), + true, ); #[cfg(feature = "multilang")] tokenizer_manager.register( @@ -62,6 +88,7 @@ pub fn create_default_quickwit_tokenizer_manager() -> TokenizerManager { .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH)) .filter(LowerCaser) .build(), + true, ); tokenizer_manager } @@ -75,8 +102,8 @@ fn create_quickwit_fastfield_normalizer_manager() -> TokenizerManager { .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH)) .build(); let tokenizer_manager = TokenizerManager::new(); - tokenizer_manager.register("raw", raw_tokenizer); - tokenizer_manager.register("lowercase", lower_case_tokenizer); + tokenizer_manager.register("raw", raw_tokenizer, false); + tokenizer_manager.register("lowercase", lower_case_tokenizer, true); tokenizer_manager } @@ -92,9 +119,11 @@ mod tests { #[test] fn test_tokenizers_in_manager() { let tokenizer_manager = super::create_default_quickwit_tokenizer_manager(); - tokenizer_manager.get("chinese_compatible").unwrap(); - tokenizer_manager.get("default").unwrap(); - tokenizer_manager.get("raw").unwrap(); + tokenizer_manager + .get_tokenizer("chinese_compatible") + .unwrap(); + tokenizer_manager.get_tokenizer("default").unwrap(); + tokenizer_manager.get_tokenizer("raw").unwrap(); } #[test] @@ -109,11 +138,11 @@ mod tests { it, no one shall find it. I just need some more chars, now you may \ not pass."; - let mut tokenizer = tokenizer_manager.get("raw").unwrap(); + let mut tokenizer = tokenizer_manager.get_tokenizer("raw").unwrap(); let mut haiku_stream = tokenizer.token_stream(my_haiku); assert!(haiku_stream.advance()); assert!(!haiku_stream.advance()); - let mut other_tokenizer = tokenizer_manager.get("raw").unwrap(); + let mut other_tokenizer = tokenizer_manager.get_tokenizer("raw").unwrap(); let mut other_stream = other_tokenizer.token_stream(my_long_text); assert!(other_stream.advance()); assert!(!other_stream.advance()); @@ -122,7 +151,7 @@ mod tests { #[test] fn test_code_tokenizer_in_tokenizer_manager() { let mut code_tokenizer = super::create_default_quickwit_tokenizer_manager() - .get("source_code_default") + .get_tokenizer("source_code_default") .unwrap(); let mut token_stream = code_tokenizer.token_stream("PigCaféFactory2"); let mut tokens = Vec::new(); diff --git a/quickwit/quickwit-query/src/tokenizers/tokenizer_manager.rs b/quickwit/quickwit-query/src/tokenizers/tokenizer_manager.rs new file mode 100644 index 00000000000..7169d5ed740 --- /dev/null +++ b/quickwit/quickwit-query/src/tokenizers/tokenizer_manager.rs @@ -0,0 +1,103 @@ +// Copyright (C) 2023 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; + +use tantivy::tokenizer::{ + LowerCaser, RawTokenizer, RemoveLongFilter, TextAnalyzer, + TokenizerManager as TantivyTokenizerManager, +}; + +use crate::DEFAULT_REMOVE_TOKEN_LENGTH; + +const RAW_TOKENIZER_NAME: &str = "raw"; +const LOWERCASE_TOKENIZER_NAME: &str = "lowercase"; + +#[derive(Clone)] +pub struct TokenizerManager { + inner: TantivyTokenizerManager, + is_lowercaser: Arc>>, +} + +impl TokenizerManager { + /// Creates an empty tokenizer manager. + pub fn new() -> Self { + let this = Self { + inner: TantivyTokenizerManager::new(), + is_lowercaser: Arc::new(RwLock::new(HashMap::new())), + }; + + // in practice these will almost always be overriden in + // create_default_quickwit_tokenizer_manager() + let raw_tokenizer = TextAnalyzer::builder(RawTokenizer::default()) + .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH)) + .build(); + this.register(RAW_TOKENIZER_NAME, raw_tokenizer, false); + let lower_case_tokenizer = TextAnalyzer::builder(RawTokenizer::default()) + .filter(LowerCaser) + .filter(RemoveLongFilter::limit(DEFAULT_REMOVE_TOKEN_LENGTH)) + .build(); + this.register(LOWERCASE_TOKENIZER_NAME, lower_case_tokenizer, true); + + this + } + + /// Registers a new tokenizer associated with a given name. + pub fn register(&self, tokenizer_name: &str, tokenizer: T, does_lowercasing: bool) + where TextAnalyzer: From { + self.inner.register(tokenizer_name, tokenizer); + self.is_lowercaser + .write() + .unwrap() + .insert(tokenizer_name.to_string(), does_lowercasing); + } + + /// Accessing a tokenizer given its name. + pub fn get_tokenizer(&self, tokenizer_name: &str) -> Option { + self.inner.get(tokenizer_name) + } + + /// Query whether a given tokenizer does lowercasing + pub fn get_normalizer(&self, tokenizer_name: &str) -> Option { + let use_lowercaser = self + .is_lowercaser + .read() + .unwrap() + .get(tokenizer_name) + .copied()?; + let analyzer = if use_lowercaser { + LOWERCASE_TOKENIZER_NAME + } else { + RAW_TOKENIZER_NAME + }; + self.get_tokenizer(analyzer) + } + + /// Get the inner TokenizerManager + pub fn tantivy_manager(&self) -> &TantivyTokenizerManager { + &self.inner + } +} + +impl Default for TokenizerManager { + fn default() -> Self { + Self::new() + } +} diff --git a/quickwit/quickwit-search/src/collector.rs b/quickwit/quickwit-search/src/collector.rs index d91fbf1b8e0..b91033f16fd 100644 --- a/quickwit/quickwit-search/src/collector.rs +++ b/quickwit/quickwit-search/src/collector.rs @@ -1169,7 +1169,7 @@ mod tests { fn max_num_partitions(&self) -> std::num::NonZeroU32 { unimplemented!() } - fn tokenizer_manager(&self) -> &tantivy::tokenizer::TokenizerManager { + fn tokenizer_manager(&self) -> &quickwit_query::tokenizers::TokenizerManager { unimplemented!() } diff --git a/quickwit/quickwit-search/src/leaf.rs b/quickwit/quickwit-search/src/leaf.rs index c11e86636fb..b6d3c85b4b6 100644 --- a/quickwit/quickwit-search/src/leaf.rs +++ b/quickwit/quickwit-search/src/leaf.rs @@ -33,13 +33,13 @@ use quickwit_proto::search::{ SearchRequest, SortOrder, SortValue, SplitIdAndFooterOffsets, SplitSearchError, }; use quickwit_query::query_ast::QueryAst; +use quickwit_query::tokenizers::TokenizerManager; use quickwit_storage::{ wrap_storage_with_cache, BundleStorage, MemorySizedCache, OwnedBytes, SplitCache, Storage, }; use tantivy::directory::FileSlice; use tantivy::fastfield::FastFieldReaders; use tantivy::schema::{Field, FieldType}; -use tantivy::tokenizer::TokenizerManager; use tantivy::{Index, ReloadPolicy, Searcher, Term}; use tracing::*; @@ -130,10 +130,12 @@ pub(crate) async fn open_index_with_caches( }; let mut index = Index::open(hot_directory)?; if let Some(tokenizer_manager) = tokenizer_manager { - index.set_tokenizers(tokenizer_manager.clone()); + index.set_tokenizers(tokenizer_manager.tantivy_manager().clone()); } index.set_fast_field_tokenizers( - quickwit_query::get_quickwit_fastfield_normalizer_manager().clone(), + quickwit_query::get_quickwit_fastfield_normalizer_manager() + .tantivy_manager() + .clone(), ); Ok(index) } diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/0005-query_string.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/0005-query_string_query.yaml similarity index 75% rename from quickwit/rest-api-tests/scenarii/es_compatibility/0005-query_string.yaml rename to quickwit/rest-api-tests/scenarii/es_compatibility/0005-query_string_query.yaml index cb3caf4fcf2..099ee2bc5d2 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/0005-query_string.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/0005-query_string_query.yaml @@ -151,3 +151,51 @@ json: lenient: true query: "to AND the" status_code: 400 +--- +# trailing wildcard +json: + query: + query_string: + default_field: payload.description + lenient: true + query: "Jour* AND unix" +expected: + hits: + total: + value: 2 +--- +# trailing wildcard +json: + query: + query_string: + default_field: payload.description + lenient: true + query: "jour* AND unix" +expected: + hits: + total: + value: 2 +--- +# trailing wildcard +json: + query: + query_string: + default_field: payload.description + lenient: true + query: "jour*" +expected: + hits: + total: + value: 3 +--- +# trailing wildcard +json: + query: + query_string: + default_field: payload.description + lenient: true + query: "jour\\*" +expected: + hits: + total: + value: 1 diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/gharchive-bulk.json.gz b/quickwit/rest-api-tests/scenarii/es_compatibility/gharchive-bulk.json.gz index 1cb0a99376e3aa6d9cf29bf4424cfc9a38e1ddc7..459830d8abb97201031acc85f9b6bc8807fb8b2e 100644 GIT binary patch delta 28402 zcmV({K+?bbh64140)HQi2nb%HLuCL1WMOn+E^2dcZUF2&Yj5L5lHc!F5ZVv9JB+P< zchkTjGdp=;f@E)yOm>02ER23i98;oJr0jT%`R`Xv>Oo1gP0^0?WrAQVQLO4>y{o#K zccX0X{O_ZSyV2$P(`Yi)OFPZ3{pj=O|F|1v4z3Z-(o|qX(0}OFsJdNvxb}AGb>+Xk z_VWrZYFib{h$r5Ur682^SEIa`WOKOpo0XpX@AcC7CEPn%>T6x;9Hcum#UqtCBKOTQ>K z)KY|r5=6Zk&42aG?^8RT74u0k)APruj0pFr+(E)}3w@gx+Qk)@J^&WL^>>|TPFGnG zal_TiHDAqA&+Qbh&2(A$<>mVO#riw^;CyM9*`fx$7`;j6h4aa(%JQtbO{yX(v)Llg z!fi5xAM!-cUGlM5E&n?KYV^d<8i_4dndDSus=JhP*VYPs^SMz)r*yVO-w z0)+V)B4-GHKxGB$X-M>e2pkV_PlrGZ4nI$lO`w%L?~Xs>jjnYgiQQWy3@)Iie|hH7c`D zKoJJr$$!c!o7?Jgd!{hzt7q{m{A9ffo^XS)63nF}@Ss!4Q)?*kMmk54 zWC$sMFg_H^_~=SIg`Hz0gg{IR*HNl4#w2)6g0V7X82eNL3_R#8tIE;E|J{wssXlE% zy{e{-*7%vuA}a;6ifM_lMurBDB+=Ai=7?lY7=Oc3rH#>Czn-n~Ducz(Yfb1BS!C?Y zmn9g@*5SG2o2*>q`t}>plJ}q|ewl<7a1^Xj*A`f=7K>t8CBTZrYdcN;s!RU?6#hJU zlb1!ZHmc+oUx6qlWqo73r#$~{{@eU-9)1HWc13dSm$%WHvs^EeY6>?eH=rHK0+x+m zz<+({+PuqYaRV0sxTvy_+1jm?=cApxHx~UI`ZivRY}{K=pa1nTu7&n+Eh=S-Qn_<2 z%gjy}dX?W|I?%7|V;JOA)>Iuk6(LDVT27_4);ODkoX)D^On=nB_j4xqgFoU##A!}s zGCf;>*_1msqP%9#66K5vPC6k4HWH^2;eU`~kG)bxIIGgO8_}4l)LY>cSAtvTsjx_E zL`2FAL4gFp9pRzVjhwLHu^W-xq2LS$P=UM!Jr~APqHxOKKSR9L2qR%%m&Q)=!sz^b z?yH?6c@I2J-cJ2gU+sGl;8CrP$+Vi~(awR9`3eHKuBs(`Nv?`vW_zz>x8~ZLA%8Dq zuC$d+zAMX>e+z_yNpzYmRfGvFc4qsrD9XzD>t$AJ*$}%uKrzU0*K{2ju2hh2BDRYx zGe`y&UUEWn*bw#MIvFEXxilw*=capl_hygcO5 zFm&Sv+1)s@==0i!fwRwreuEA$`+w_a#M>Z?o()X+0A~x|eV7)jJYREj?O+)LWdTiD z#XDmniNG$hDvx^eVd|4DETsB3$xX4mDx;@}!8npp=k6fmjb(u7-R&pu=kwe38sMq0 zWMpXJ)IfiI{Xatk@W12`U*-O)1RE#MXjBuK-M0N~xYsDEO$6mN-| zIsyNDGD$ViUHu1`7txf!$0c~x6A`V!fT)`}&~sO-ZjEncwx!|8wTTx`c9AX% zVK{>G2p*dqy0X(JpsC?wxHW~ljc)JrRxZyIT{l7=OF|hHNIaQ$e}AblukoI>(AL+V zCls#3%Le&zC_2cn_7b@#rCY;e(E&*hGx^}=lcG@@c6d1I!19!Mw1@6V0cj7@^F^e6 zv`-C5d)Q8mNqfMj2cDUtT0o;hkPJ)-v7h z8xT&ZpslWNHE@--Y+C?#glz*64+rSQc z)P|Dd=97ZD1TncSV(V4LGe6NYJ+E2YBa9pZJ=kbsh;AB9?5NSidUUI$)yg5oQmZ8o zLTSbju}XVxMCvf*LJ*^|+iNos5-dTKrzsO!2}uc~Od!pvO@9rb(-cu1NN1agDWx^xu?{ZyEK_CgZ z?uj4RDn1bHwtv&qn{iYl&xmcn}R!fXUzn!vYd zQd*tIo_pM1^Ht(9aKDk4&&;{rY}j!Q3mm+39K254c0r6_Og%F;U~44kx-y8WU|yJW zKYO>bKCzIpWMx#kPf*h&MM(Vb3n)IGpLMaa}y^Zn%4#rXZ?8%VkS>zP^En=dcbDdzp=4vYH33R z#iW5b&bdxl@Yrut&%yRQ2^=eHoRdUq%)PZ*$A3-6z&$d`O8$Bh0rtK255U~yNAC|N z0~!|%sX-3ck^r`Jn}niP(*a)j5n*n|z8U4}A>hZ5JYucec^)#% zT7M~3L_3lSl7cIOl*utH2(pqv?xJZ5M&RL3+gc&R2?`r#FhqTAQ|q~>QZh&ekk%%( z2nAwlm}6_@k}wglR7wfvJq9~N)?gRF{JGRjb11b5@7{<9ZFU$elo5##GCJ@$#OJW5 zkZ=V}up&Z2O5A0w-(`8W0EQ=j-)P8jjei7c5VFatDU4nv8ZH%e)T_sGK%;41UCAV0V>$pX|{A)9*_fqt!9Dsi5ttRb2fO9D3NW`55}tTzRou0&fe(VP3b zwD<%Kd0Bz|=m_P~gM-cVEf$W`FY_ z|Gn2)BZ6bnYBlXh9M)UggCFE@8;XW?*U~=SwOY?eYs^beSNVLhT2w>FNB1!dD+n5@ zy6XWZcAd7?TI1ikphx0}JT2=pxt{K#hz*u{0_tjFmxnt>3>4YO6O98kkU^QK z%98K(*(is)IA~_#K&T0pGZP$<0e?}b(ov8?31PABj3Os2cY3^*$&-@BYrTkQM2p~j2XDto?|E{1uF1EKAPEBik03uSi<_o~6UP_41; z<*lBVg(-%nc5V29!`kY2QoFV`&UX|4+hxY9iNCA|Y^KGcr_pu8c1`_nGk>usjQjsk ztibE$P0*{RKKXT?{az0Nf1g#!X8A-{$=i4=;5#3CN#*m@?|6{jwC4T)5x3F1-L;|$ zp`GHm!rlP(vKV?`WDmhGUtN>bQ4*vaih^B9@Mc;4qD?lhN`F0mzl0C@{3kuBI|n`V z=YH^eru*VvYn<+T8R_*IsegttQ=}a4q&G8uZZyIJW!C~9B)K&`ohN-#7H1FEqT4Jq zE*Ip?-mo*^8tW&vZqP>}Zn}xM{3zn`9+g|rT=U>~+y+oc3PBX8B4U=~Ks*El>{1_` z*o(Oi?sWj^aq3b^c`%wtr#xUXRv>_PGPT_&F(gxI+{)nr(qu3Op?@?`B9z2d2<{bj z7)YVEGv+Gcl}@SjI#9<1hD_I|R8Zl)wa989glYKjx(>OWisP z{u){RDf_e^e%~*PkKR^sJ7%@0J>c7zwI}}zN%OopWzkgR1A=m=ba(CXzHHf*Q~?qV z7iiGbYhuOqI}S8iZ%Q`%KFp%}FSW;+d11Vq3meyN?TPui&RD7MgzM6GJ(+p{`2nsq(DBnBA(q}eZi$6kqP<`QTdS2(+N41u zW3d#FA_y%7lK`#%M6t{~tm|r#n4+529IUmy(et-YsZ<}~XMbPj?D&}y3+~2)Z?Zx% z!<|)Db1)C7Emzb2eoh_={U%YM<{Ey&UZzi<#u)k-2dA_}^pDS53{`L>i zw7!pjdaW}k>*1(J)}!K1PCM)QSd`wD)1f;~Tj*g2$Qo+8(p4#Bn(p?SnsvtCzk7E= zx^o2eaHfNwxPPNWRLa5o7%2j$Y3evd$ax=9i6|9@@EB^j+?(RzIJJnT))0iK@?dFc zJY$qXIA+*rqnOberLHs6Ibp#L1B@M2%6U9Um6|{i7#tMrMragwnS>Btr^*5EwonxftnkHjT2B{| zJYm6(Nq-$3m=)MLYau(=#A)N5M}Zoqh>e(Z$|!TMYaX7*zWHuWCtgb4fHKUsn{zuu z{Q&Bfl^4l7$bmB*k8>wxmF2G9OK6@V`6_mNtqDG!JXcgZERB#*jtNDOh1>wt17q&P zS4lBXn*LAnmGjq`^=Hmk_0atp{punK+3%3RWPf&ORnw&?yP~EqhX?tAz1;|rD)7(; z#ye~yVgowJHXAwy?LqK9eG!ZZ=Tx@#UEWTYS^15R^O!>qtsbBlIG5BAcF(&|_L^H( zyG*}b>QJ3Fo74&Uh*vssN8Dq1278E6~}>GVq{EO_jf2<6&Ap@2zXkggz3h)LrVN6OQ9 zFx&b#LFm-$VpZiuakZa0)QXV&=;wM_?RzC%7dmhM_U5N=hdGKa*=-#~??-;RqllR} z5$(aQHf6&+#sP+b_S4Wc%yp<<;6g^(E=Yl$`}dia2Y5SH04{p z+^n=+^3-9rhY&pMKnQwTTEmo+DW(`IP77%<=0v32J7Jv-Rv<#Of{K*wjm9_~5OY?r zz!4O&6v`0kgptHxCX8cBl4u{Q(|w$0En<4rMeQ}Y24whjsOeD>4*2pLj%h`qui&wp+vOztsH zAXv3(e^?-!6pRc#mDq+KXrDDjU1xX{Bbe;=I^7lE7{|lN^I6Xd;{fKKjj8x^{nm-? ziwp)GE8bshbDDVcYW+CCBNmS&J%4;~vja(u2i2Zv8MbbEkibJljB zs}G+j-@H0vJ)iJy@eUMa8MO7kSUo@g2&yzXKC0W+z&#azk~$7Qr`x9IVOh8DJn4k) z7I66S+(Q&kIh(t~^`wKjJ2cOCDz}C6sYh~ms7^hPyMsLaFzycac~0WCX@5EW81DM` z?1w2N@FB~60)OX4wLa_PHt%#>BK_sMQnG}h6lrU4taW_q zWo*1?pAu_yD&h;aEUWA4UVpE|$ojp{r#qT#l8{>X*O|YmY3=)tpa*cSRbU_KV4NF_ zLprBFkA8V|uC%Hr^EDv{EhV}xDZd2bT! zyXO5U?Wl86N6NEwv`N!<-y8ioVUaJP?MrC;659Tp(6;~bN&;j-?hTW*BWRO-=|SLX3Y1Y+(U7<_cg9ic|9A5xdx4XsVIhWtOnioE&9Jc z^BB`Q>D=h`E|Gl-(toWoZ%T!Mc(&ko1J#Dux2vl z|2+W7=Fud(37}`zA7oeAQmfw&j|AQW@J@S-uJrxxZ8%2`&wo+(;G9K09L`ZUaJxs%s%+%@EfxsKXyyDO(OC53^Nq+P7W5*IoL4Sg52=yt6L z1MWe5{A#->r2bMYmtWe2v|Gr_U{=={C+1m!PE8?j&R+JsVN56w%Lnd%%6cHMQ& zu$5)^?r*Ed@9KcV#tz&6tS%6damtC}xg;<{8GjU1C?ia#sfJvkDs17{7F&%} z8*wl~EOk||!fF9m)DcFUhM76eWMA`5Z7mPB-Sp^V3^z4Ul+~ujxSYx602phpIrtaT2BXnvrVT* z9)Dh6yXg^mw9)hk?@|`iBUo>9=@EUXt#sTwA7dyzLJYH#9)U!fNRQBB?4!ri=4l)~ zB9F6-j(`Q5MMtp0Y@%aK1sg=Cf<1z)p+{ihrqCf*;dao0c8M{99y1nY0UZYmFn4(jWup zSd3k`b@TY8U&gd~g!(D$ngeAVZqz)2I%m-=O_6}TM(_@yJA$Vzn(#HG_6v6ioU&-b zJIwAVI%Co7GkMOOIr{B0w#@MnP8l)>M|3YMCT8e&X~hIHt(d5>VuB+pCi1ajzJFaV z3J3D>YO9yV{r{n|S$b=MTjY{j+%;_kWpdgMX(hjP8m$I!(BycuP35+cP;G?oZ-WTu zxc77dISP2VMgP`Gn2Y9kQ89ct#Rek!M?C-b+aWr>k(lyVk-HR~O3IoK-X8LLq*bHE zRTEdAe;D?H1S9hbI&aKHM%H-%2Y*Rg>iRcszgJ}%ymi?J?%UqIyI4qy9q;tG$1g`99EySd)eYiB^0JQtAjmh|$*CdM`E~^8H5HHRHTf=NPM2)-lnAi#-NS zPueMabfUy7CxBwiyHySJ`(MOnvk0!%*+&Q~`dD!_)oKNG^5&bdZhsbqT7PfE4|@5i zXZGTrCjV5;SH5wXLDh7-DsU@N*c~_#MX~~I0)~V6`YNwa0MIm;<3#pF~*0`l)YS_@75lmrQGx+KKFAH_A?T%%C()oW{Rs8M4 zX1!YMSMwIqDS*psgI-+)o_|4rI@OaJR=nSOSyG`(9wxgbKS9+L(s}KH5K1vx!0=?*Jk~Gp8TBy=?kN~2Wx$FSZV7o4H)P; z^vicLWBmCQPLw;pg1DSFzs8Go z=2tMjzWl=)^>yV}beHkuA9(oGj{FIE?4Raet@j(LtL_Xhn}6FtVwI@baQU5nl4@x^AdDwt!)dDijJyf2h20*yZh+AwEq#mTUb0;R4Jb89v)N4Ztk?qKOm zuySUN(-FEeW^){8Hismql4HXN<_Z(a4RXza3==39xn$He`vhsP#|Z=gxIR`{o?4Jh z5(Y5Ja(^jPYG04eNt$7d<#?@fj{!$IsA`aDo*INGP|(@Rb(UhIh5g)qDmf%5;~5__ zn^zOxvzjwXlHG%AfW57|VTYN_hqKn0%x6FD=Q5d-83LE#l<;obVzF9pi-ivLhjz$= z#IELNXgaqd2MKAbejf(1uV2mV5YQ(%BW#fkOMfx~l|q?DIYy8XlLMLaoFK+>lOhgP zDhxxWSJ^uS5L;Q(1ZWKGif0hw6y%vD!VDOtNRCrl_Qf3vo5z474TT5=5MT+I$`wt8 zQkepTE6yNG3CNg`lmRJTJPG^y*X!b@(7RUNzFvw&k^EH8e7}SGqHvu_%IKdkiR;Z; z+kf+H?&E$gVVmU^hA_>#HTM5*&!P7s)GypphdG8TZpI4i158jGc~FmY`&aucul~Ll zyWQN_l~Kwwg4cBNLIc4=GXyVJf(e$wlyL~G>J>SpSQD9Hk#dR*;*{dNFL)9wrI^7$ zYnU2|?4Pla$fyh{4ut#Eb?zD8 zVzt-x^@CnMChPsqzDaw0DF1l*mo@vzKbK15a}MkoU>LAYQ^G~@P)lS`SHhM&?ATuM zGd#AZ8K-tBwtevGftnEyyv*$yath##ier&oLtOZXRZ$uc=(>psZqLb?$+3o3X&K@6 zKW4Y@o{ED~BKs1sy}vMMgn=04Qhx(Rk)+b(l&F-;va|9~#d!=ka*$V45T+!x!zl=f zgA#U0=Ri>VF{hG9E)W`RR(MzqJ*lkv45&B7o1*?POd3zjVB0ig?xTM$q0D9oC89V( z-COk2%2)-zS%h{d?9vB`WzEq*EE<=)}r8bID4pg}5#M>wdk~?=8@_j(;MU6X$pu zH^vep zq9n8@Zw0xwhw?I5dMKM70}f@A=o}G|AxlOT#oXS#FwZE)IZTOLt&DHX!kN+n-d8MH|H+VS{BvT_zN2lVRa@BU`bMuh{fxOh{% z)JAV~A@%m|O;IO8Z68(nl(#j`_O|-dwoa02ST4U*^8S-3_F{SWPk&GM-yfdhTNh}* z+y0@;j2rRW@-B4!qq<9e)X2VkoaPGgjA5j#N&)s^2^01PLwE|5vez3V!}Qx}L1=hKG;(Bqwl-VO#M=c!~^j1q%7;uL?tXHaICnO5kp3ml}Va8e7%`4e1_nXIS`&(!) z#vVYJBCdEEUiuie$A4qc=?=SKF)M$7GGj$j0cX%l$B#uG1CAVv zfXPS(h=fE)YM32Nv)tKKjiQDVDpDY=9)8h%nK#AyrnGj|vw!r;z1yQ!&UNQlndCyg z+dn4%ytP+9wO?xWFJWqAVm2H7G5a-qo+A+3EezYcC!FUnb(jh5&w86%TCa$S zu5pbTKd#&;AH;sUu-fXOo>vC!Wh>>O&6g|ch+y&_LuZy&_p4rE>}`c*&mGxHRa!MT|e zpmEum6QJSwnbTZ`XJ}4pwU`{uDMq8RG*5#DDm>}7|TW;Qxo^R$j1 zx1h|=*F3Sp!ZS9fUq^J#<^*&|*5(A%W%4#Bz{4^(PcDI=+|3EZ$n4Fj$+=Aa<^=ku z$baCR+#JGlI42-yRi(z@HK2Dey~8K^>{X`r^bVctvzOL|tM8|?ILDbim&bXM>P#l* z=@C!ma!y`t#1Q{dbYeIo4k9yixm%tG0XGWv1`K>1e|2p<$s=qMS9odF{Z0Itj5L=c$>P;J0!wQU~{pf zOk;CADlsOr^Y|AurOg;D+QfE#3~$rhdB^~h++yZGS;yJLdFk)qr*ej^6DZ~O*{b?f+z7Uk~oTz0;0AV2u~IQNkMVBu~f z|FHtyNdXTP?q&*%?hA82`3)A~mhyh82zQn5?h)n2@_ev3_g2tgA#N| z=iHG&9G-Mj`kEjo z3?1nFy*~%a%eZ)2KK!#(T7TsXy!v&+UcYXFfvZt?)Uz9-pswA$0#L1E+sh{&P#kI& zZk%^LsBl!;&f|}6m;JVyozdzvj5Eqr(!y118Rg~T8o*OG@N3s(FLtM~ziI0sD( zg}tAcwY?HOE@l>t>Jm&9mZ=unRp*T52YfixAPl@Ng5wZ_3!jh8Y<~h$wi)d6y+spP zjm|i|n@zr-x-J)HAfs(r7vy>x9}2gK;zP0(tH-5IcHvB$9)Q*3EirexJC>PxG8_4KQ|A6GZ>mLXMw*3R?h4%fH z6aS6BQb$*J|NTF0<$oo2A-@{bU7A%0r8+O*=@Kw7yF=+6n(9C_I0JM_z*M8uRc3xn z)Vla$m$;ikl3PZS$BonZQdb&fQOM<9x$2#+fY@TAlkzlDX`8)B>Qz(R-y|J^n5)|+=Ffjbk}&vo*obN3RzO8&IBBW~BL4$0GExqq}w-E^+m;hRl!o#Y4; zt4w*IM*c(jV~uv*0h?XZ>!hYX06y#OXTANbw?C)eo^yYiqhD)vx_*}|64&J@u`4dQ z*E^>gld>h6R>}Q-q4dqo&6~}eqFA>o&mG^adUSel=PD)JuOv!XU0bYo$!_z(wLU0` z+&An_k=#0E>wkWD5Wng~>D9W#*sYU;<@u}Re-Em)gmixsyTo}eb`Cz*C5Ki2>9t8j z^2XhyER7w`ep=Y^)tjsCmax^^o2$>#tIYm$b-uhWrG#cF=#=->qPX7L>-9sRcIyFz z*}*EFhV7GhN>k3;_6O}yxAkWDwz3M&yRc1@pRVzXBY(@Q%S>ki388Vy83T$SB7vlY zP)xZrgexd{%3UeVrp7PgAV+{OEtSau%p_(RNhw5GW|YdM$~Z&uPERmzp7Kv0DsAXKF$wd2{JF_ua)C0^u5-AG>9*PFa57s5z;m!)0rRw^oH6Dz;3 z_C@kXoqs0@OzKq+lmB@AZQ`WiDyZh~k(jI;|W7k?R+8oAQZT!@rI!!T5sVr3AYG6D(7NGp#&e8@NJ`#^DQh{L3@;$=)40f)Gi z#(!^K|MJJhX1ja2&evrF$nu9;*Up&86ZB`KGO{P30VWmN5RFA{h(tixC$lOjv+F*! z0{}!Ar8MtKWoFfRrnylFWs1pM0bt(+l{wQWr`#~j^IYrk&C_GRp;WR|pbY0waV0XP zkkt(tv14IrGGXk-12fkbQN4Ir?}nX&>VLJ+FO6UBo-6xc?lfxeEm{{_OP!c~QJR^* zOvE;M()Rvu|L+d2YvsB~5AG2A!>d2PDPBKpv?W`rWKood*jFoii2Y7%AM&+5 zwBCe?_w=f0M11z^jBnHd0p*;b&f)f$zc8=N2uh$2rs6Ni2U7~_+0*@VXMl@z27eeR zv8Ic0Sbs;DnLOvf5Tvly8D=SzjHFD!)Sc^1o^s%h_dO*xzna0w(PDlP-Vr6nkuW$% z++H)V;0Pdq(_Gkdzf){9!^jG=&eInk#KU^sS7G0)C#zt7eh{m^8vC(W>=t5KpZ{-? zm-hPot5s~<5B5#164&_AEugpMZGS&8R#CZAOH%b6N?a_mQ-3!%w-o^n(^sA$C(SWs zcEm~35z=GEN3LKQ)L3I^IRAh4u7o#o96SFN#aSR5q;2ayDhYPZ?qsvGS&Vx&*&sj< zB~rH9vQ*LGZjaIbJ$$6j(oM>qMN)ug+O8^oWKrZhzW4a36IZic+onQOhkq-UG@tq? zNX14-&C0Rc=juJ%zt9#^Sro|GQK~B~l)tr%4X8pe29Iu|RJ@mx? z^LW0t<{~KSMHpIFnf@WYMajQWLqqtU%szsc!OKTAV(`*sV!3VE;!a=J$BY!k(+7SR z>N@o?-F!h)IJKs52b#iNbAL@HTmo(S2rNh{?j7uDff@MVXIez3f*5OR;A=gu2zxGp zyNl>@@O7PjlKP+2)TPd-=6fpf^uV{O8-|9OLJhI)5I=z1paW_hhq`O4CS0bbhpOZH zA$9m&VvK7FlXhrZhNoH7+jIf-XK z#8FYiM1lACO!H7C;eSs6kn$r6$ji;|+R19?H&i``xVmdt2K}*I-=>BNc0H502BJFD zM4F|$6Q<6q%?{AL;bLsVaqzHcD6;APbS#uCM|c7Jn5b^_3q`kfkp*!>gyO zong}t1088$s3K}NnwCyYJ46CRpOA0*)YPYzCDm)=H-x{+4F5Zehf))rf7wdhO57X1 zJddjtbe#`?jT-wo4t!Fw`u^@Gvsq+O0LiufR2eD7Vdk$BKgS9i(PO^mq-*ZBQkY+m`_FWX3z-Z|f z3bCcB8VRZUZkWHKI7sscQgJaaGHa`+ptHr-n{L0VBKj0zs@Oqp@|CB6=}EGGgj*Zq1cXr>znDN+JB6Xkyf`94LApRxHn<(+tM-Xz9s!L|J$8oW0dUS7l4o6~k zz8?}jP=6g{kWi(*n(ye2rXii+PzzMu2yN3a0&9%d=`M-E-MPw5f{rXIz^7fgDR_ui zguj`2nysC_b{bx5qjs*(T|A*+P)Tcfoef)%^p`M0$-Urq?Y0-%tksAHn^n^+!=`3t zvkID?uD>fu`~z9HK?kjNJ+5$yUT12uVCwoK3xCrvQpXl%U;~k(3p02tk1x#57n6-9 zHPcMU_^6LCXl8qY0jDHSASta-s6X^X*Prq+jS>ZA)bP9*Cc_qYgo-WsZQH+ zbX673h~=#%GGb51v@l^OpWc)k+iNVIAUg&llO@~7lS-BCL+zI@dx9&QG~0*UnmOBV zy?;G@w%>lA9NN<{NgRL>6STbfwO`)0b^`w+NMq1<7u3J@bm~8M4p>ZGZQUNRcpg=Z zJ}GkSC}qx);O|JCy$e17@T_UbUOmR;Em$52 zRsxhLe2m->$rZmWluvggX{;1TlBcc;ohv0O;3r24b;N)zRD`BpUo^Nt5Hk@x-GzDwW5wq?rn$M#7ESg_MH1B-g>HS@lD}Q$& zA_*0bAtPn0tjtQza>7KZil)J7iToJdrUetVetCZ%6|Z%wNI$Lpke@k~O`J+?Zl5WY zGo^B-RL+#jnNm4ZDrdX>Y`357_A{mOt5Pbj;yxH_GAKONQ60#fu7%{gG($%a6oNvn z+g?bnRwEioGxt18b=z~5O;6WvV}Eo!K3)1zhA&ZkyhH2!h&h1y+zHdzmOJrvOg^*P zc2({KQUj!#uA#b4V7QKl9D*F%!@3rls)~I}vsKf>;Quwb6PDrGwr?1&XSu|(Rl`QA zjZHg5I)%`nPytqTUAq{|o!HRfFn2=JY~7&UAXi5|qQfvl&-ZN4@u_JapnniSLYD7_ zUws|Z^^Vio6sdN7^inCF5cRwXWIKN>Z&9Y`baw_MhZ8YWq!duN38q0PtzzamH(3>{ zC+L;3DvVA=k;SYE({pswYxi(;i(q(S8@OpKmixqpGE(soR&2X>kLB@53TpA?m+6sY zT(@5HS1-O*(pj}fgR)2KZxWKl(i8I&+EyW{j>`I7R*x1KPzF`e2tFO*ypBLWo zT-P$S4g+D^Ed}u-wFR;$DBuJ)f;tJ8AM-T1jH9Fg4PRRc|D>o_>XkXdNnmOok{D+s zVp@-NNtKwcnxdz7d20#R;$%qkQ+o2@O%6u4t+ zp>1_JdX2Yx9lhrJI36b>+NZYVGEX30-BNw8>Icg_=#IGG!eBJ(K!0?X->wqJAF30> zJE|src>tGAEV%&RovnyG>6}z86_@R0h3`v zULv*jimNX{$Nq-8x>0ba1SOSsNfAFaSk=+QVQ*Io3#}b9zJE`ZEI*P!0XIA6huyO> zNoY@Rm?xbk6f+8^X1`Xs6cFa#Y9{18{X8=n;4Pd6C2UQCp;F+;3P~cLnzFx8DKhM% zu{=~Zed7QR2$o5T$>a54^4d5@_0{PQQlNA!%F+^QD$s`;vv2r_vM4Qcr94tJxV9Dw zEh?a4_I1&rx{&{32%6X}1D1Rpnjm0?O@rBWromxAi+v#9Ohz?#; zHpoy}N&b3=BR|cuGEW6Jc&E8od>lqI+}(!jIHuMz!8cJ5#S!|K(!WNrpTgnfl$y)O zl`)*@<_mk=k=x_QgQ~B(g#tS?h-vtGVCdL%fN9h*u>;+8e3dZ{ZLuKV@IA*Qdgx(- zY=2BF7h{AxO(Qz7EXTomfI>VG!CBwoaE}v&9AxOG<7g%cbX_+sWCohAdmh#Tmk!3$ zwE!7od;IH!YHFl>Q>}*HM(4U;YzIz0xi%)V_A3i*!k(dEo2KE|oz2sZPRu&!wHK@} z%Z4Om5}2LP?_27Uj*1J{*7S}+@bHxUB7Z0bFh3LglS)l}ps zV8_zEn4m{$*Rn{?s?|2rzAq@?ep;S3#6_xRs7|N0*+}mpFm?4X=}e4`ka;p)|FoL2 z&U28MJ#pW(8pgb5T777fU&VJu{ zX8ZV|_1@U*g|cpe#jiN~<<&a21u|OhD*Cfwmu7Fk6rl zQ$ZT@SZS8ZQIh%I44}8jr9oj`o&s;lO`0C+sl??1y?RGlqBjw;=eCE++Ll~e9Q7Zd zlzKH8N0hCKicQW109-%&h#E*&S69!#N&qw?&Lu|6+k+0`zNI+QbBzF1Ls7x3<`9Nh z-}ojm$_Aod1JdfeyQ)}{PZ z;`j|+WJMH2blp-fjBA6nxr|e)pr5i>owAictP!5CE4y|D?=upy(k^xHz3Ln+>!7JT z{U*OV{fO$kSKWiI9}MHaDq+dTO@CYAgTPk;O_1NU&Aw_I9p)MZF@Jsb8-1R-zsFOB zv%*=rOCQLQaeO=rs=9AObu7E5#DlbiWCiuj#V5$H>AzGf{u<;Cz2x7>xqK_AR9KH z3z+JpxjcK5`dmbl%Kl$Q4@46Fmtj}t_^+AEDwbzL(dIU>UvJa;|L~MWL9TpROzd2NEGK<7r)$sn-iI;LxO0^c7|o+iOlDQY*i&Be09_;`kSBbM%}UTYEf zJ}UC`5sKNvB{}O1fzw7SBEM|}y6veO@U6dZXCQ-EZ)Z+=Q!X_@;##INoOwml`QRp9 z)69vmx_>!AkkL2|l$6qGQ1;V1Eu^gKX^^&RpQhn<1=KX%M+bFA^3w6)ksaH2_x{9R zX9qeGg(P>>J0w{7{sB>Kp_>ZiIEB^M7gE&C%uofOMdFfVfwTc>ugQygWb< zjVupf5>45J5R3z3r0NE$ZJAf67@J>8lq2TTSOERuIoc?`6x&z#oh~|z6Z1F(u0BmCn zcZ&YeE)DIy<{b?Epn6j=T-}8*ucjUhb0LO%I?Sc;?ZoF7BDYh$%mr;n4E{D=dvQ@m zYc0b(2WwwBSZl2c);gMJX^z$jza`WeM1P+(;lM$sy%4KZjc6)d>oqJ`(-WKzD~pGf z%-gWpO9g9x&Dg7V(K)0Cy^TM8Q**kp*9?6|L7GKDN;gc5j^NmeM`Nea#1;k{?AW@{ zE!S?>#YDH%do0*x&@0x+JVbB32t3Z-aj%zp1nlV7OFe<@3({JbJEEx$Tz;!}&VPbM zPi0!<<-KV7NDn}a=dK1dA(A?wfVQtrxAl;sm5Xl)BjY-&7T#L{~k zrnn;?;grn%Q%nSx$O(j4#GMUmczVz>UB@%qQ+qexq$SQGlwaSL@$oW(yO#YjUbi%+ z8ZTv!)Hw-Ic}Ad`*F?8f1uv#+Sbt#5HFd*;KIUofIXI3sU%_jk{mP)esAF;-HxMcIL{prnr4gAyfF6S0z0DJws5l1Fl~pu5{fW^qg1 zOVJ(B;C2Da9#@((`=x4Tv3Bpi=Z|v*uZ=CDj z-~8wGTd4n+m&aq870P3T6uU^RE2XVR2SGxNS){NM7)4si3+jZk?2)e6xLT$+ycO9k z-e`6UBlHF@R5b1fRz`xqzJDqkXC?Eyq^4>Xm@R|HhC|EBCcpL9&B=cU2WfEU zO@H3>=S~0Dvgx-a&CTxgKBM)o)f-BgLK$Zdg zn-kz>prljbX0Z24iksK<_0X4w6_c_kiIuN z4N{4G3+q=VrEg)-(#d_xx3}mGh$RLtE)Ag+ z!5LJ^WWl)ywolsNOb;)SKsc*p#8U}pV3(2#%}b3l>gEB8x@cpp`^kT2^r@smLwm0q z2g_SdDx4nhQhMQH%Y{V4<(}tK4rkr2vE)O|bVYIxFMf|6(G63I7a}=`olngqAsU;L z5Dmd3L`XXX!QG|U8~Xmx#>uzA!w&NIv8K{BjSq@e;k65~Hnb@|m?INsN zs%wn!>rEFm&!!u$ZHa&KY^<>a&(JY2C5Nap!}yqdD$;Iwfg9oS%2|c{iVT^K0zpZKU}07T$po zuSOt|pSXb!pUNoMN`_D2NHgJ6zEfM23I;o`M#mzNgS6dG9G$ z8Pa?5VQ*x-r*P!x?m1Lpu6qttjOd;nrZCIBnA#*naZjO&^V>7y6eqW*g`5ntJvme< zT6+OifYYA-_ELoQ+&YzEv*(5@O=T}geOVrRZlQ^j*fTRC&0tSqi_q6oSo?6-Q|MyE z_54B*Vy$PBUd4YY>&Yjy4_`e6{}N>Nw7L*ys;4lQXler}XgIWP9W47`snhk#jrW>) zFr4bJ;|Wd8nQZ;N<{VV-VA=~b^~7kGIO_Rk3k3B-pL6W=vq%TB|0n4_mZ#8H0&D9;x;i9`7Jj?>ZNQUs1& zec|5o2;TC8YYwG->C|AB8C824>`S-Q=uV(*9{-oYA&0%@bEc1dx)|CNUpm*2?)W7l z{vB$IUKV@%p@%PD>Z3Bv4rG5KKPJ7CGw{NP`oLhB+$VHzUo^*T{HM{{wS(uccBYXE z9)xkkIl6y?VZzRgN%bdvlV6;SKZKw(>DGOnsuh-V(XSc*Eg$^oNnlGgHPiCiqrh)T ziqq^Z{eB7K)kHYuQw$+yyBgWZmBXs4>t3gZ#qsz^p3>|(h?a&|Rj=qYm^k%YJ(r7Z zMdLfbwK&lXW0SEm4k1+@egZN`1!}@iV67|!KRJItb@g}C)2;S^FMY2)!}}IRF&xI@ z){^qa@*OFZBMR<8_cFpYrX~Q?eojAQGc z`maSW+ONZA-G?GJR?^sWdB;|aB+tXf#zdXn9Far^K}u?G_`h!gJOB_N2^OffW}_cG zmRem|Sy`1weVLWGJZIA!f1SL|v%^G36@T&n*t@dcMvf%?S8976dKY7gc^?KAD^It3 zW)>cgVYgu{&_E+ksoX`DhKJN@!~cCFswlBYmZW0o@IEZywj{FhsK|^fGvbS}R|+?6 z$C~SL>1=(L=Ch_!_hbPZWMJ^jU5y_wY_}=%J+AE#-xe$`W`Aws z!cs>x9*8lm0@p6)1WOgQqegPMbI(9?;$V*$@X~w~jQHRMo=f63(!>M@{v~s+-?Wu9 z7|>p@=Nx#$GH1rX#WM`pMJ8gk<*-#H??0gwnG2$0(#8i2tq)X@ltg6OM43QEQpJ+; z@^lWn7D1QG`Pp6q()aL9@n-YYE`Qt2(^Y{9M9X4UZqFn}FyS4GdCQT5Y0!z2 zSu(iSBY5E6XJvy;vEML!X0oL-7p|KP2mgx&5NM>#e~4~_iO_K{Kq5qz34Z~rikk>> zAqheiy0Tsx0p1!I=;98C>v`#R=@wAY`Sj{|sYXLES_M1QFf#%^z)Hseg^X?|8u-u>{`Z-#m9J?-vy+RbfrJv;E(*$@N( zW;J)&vFuXB&bezKLf$V|ih3E!s%k1wq4D}@wDEcm%?1ccjfSyoL{^##7M=A}Mc}av1~{0Nmir7l&HMiN;v)t;F`oKIpj`FFbh6vCn+6lgULvSylFd*9h-?9aHE#SShY>k z`JCqEY+At^x0OgHEghShL4(%tz78H>%pB`71K`Knr^TxHC6%$d1CDwe7v8gi+xz~Z zFMs|7s1_PLM$vI?oB18hLQMqeP1BK=(#=H1gpV>ZB+tkw;KnY3#mddfYAkN=ZR2sV zvh7@YN*8F@opuXe-wv}G?PeC`!2G_u8s9G{>?2d?A=tf28XYJWQ=v?O771`hoBIQd)YDcgi>;9Zbz| zU5&-D*fbsIm^RZmw3rQQN8Tx?1Y^K*TqakY{JNYp$D)%(n$@d;O8UMI9^kzrvwv(` zNVEwS5u2p>xtoVH+mxHW$@D;oK`=`rxS(2qLl`Z|j3y@}^C=PGi02{f0Ay3}UInKz zM@gfiL2;FgqB4OJvY3SdP9{8XN2bT{0eiTSalxaf7BeUaXu%XIvx?K~vct1Jt`m5P z`gC;##7PlM+C}+!mn)ksvcr=IPk&Kl*b4jJ(&D_1Vo$$qhBzPevr5*M7 z3zlr`=TeX%whM0s+I%_IyS-^Uj#1m0^qnAaN#)kJU5S;PF0WErIattvnArP`b~zGA z(k9kn#ZFj=7fFj_YzPFGp+Z~TG1V|%G@O4LxkTI6<4l-#Ha+m=+Hk`<)qj`TaT@9m zU8B7=_Xqmewpp7)<-!}s*`fMs;{TMTBr4@tOm-yZi^=teB+j8 zi08&Q1!t*qyRljTb#b0!owrW=aT=g-3 zyip6_U~vV8k-4Mz*bxKvU=f;-O_VVz?wIzTIwG_SEGHdPG>S*=gH*jmkG9)kM=~M; zSy3*FdY%ejo)rrq^ae`2Uctv1%*nI)@}&5wF0RhgytuB{#iESOh2{C8UZ0)A&r~ed ztL0Zktmm&+7|wH1VSicpgikLpEGnK9@2Ug_>Y|<%=kU!}@aYm5CglbW{#P(H>-F;b zqp zaOR;a_(Fo84=MS;h0?T>mXQ}JEz9OCaExr}G#^Rm)tvMcji+eqa2Ffj3LOMre*p za6(#4 zlzK14%R?Xn1SQ~0bs%OU$5ZyP5XIZp<1U5T*&Oh)<$qL*PW|r9FfiOnUVr&=GD|1x zfw)lpP4DMvdL#tYp}!=80ZAVXx6&%74LCg^2jU5#EQ2$oyoS$%=&~~5-WZx)P!w}} znTtNU5Cj+m3N{pGLsJgC)yv994A>Vn$&^bjN6)y@iV=*22v+kHqX+-Vcq3Hu@pOmy zyKA!-4SycaQ(RXmJ_O8g}DV~AIX>9_M@0`Gozjh8OjtP+c@Uik)~gUabdT?18jF>c;F!% zB7e)U1Hj#sB03(FagiqOyzrFhjCiFO`mlqe6H<^FA)yp(ofI8%LGXdYLV^~ICUfhZ zOLxH`4j%CTG+=*ciMnO7hP~Sh@s$|jK3dXs9)ES<=Z`&+zim9O8V2cc8>zNN>Q1dd zpWIoMJUS)Q&hL|Uori=@-_yYZy!OZgsei;WXATQp0Eg(Hy#w|M8Wb2YCMQrSl7r^j zQE}Grln4v3si;FBBEmTniBT_|4idxbHYo=d z#tukvR?cS;qQY_+jx+#fB3V$$US?*p^Z}HT>pyBbV!#(aYQk-4iserhm#y>GMG-lNjbqJ7I!Vbvc`CaG*!l=Gf_BoIGl0 zb;zL1XfC%5+8JsZDmNdf)DN3Iz@YYV34dV4`3On`qj3`fhY$X8Ml!U1;e|4lF=rNb zRJj3{KuBhTQW_6|!AztqMq*R2#t3G(@lG^w{ynN(z1D#R9Jol^bj2e!(#x(ZR(*8^ z2%8!gU4W|C_0)L*55%6+I0VZRu zWU6xz@((u$9Twj&(rlK-qKqGOGVYwRo!j9#rD2*FMmmRJ-QlKZSE~sw3P^$puF${) zuLpa%Ukk8D9-erk65^P^+Yu^U3c^V6Hl%Y55yqL$SfzB=Od}~}?uerS_dQq=EYe_Q z;*5fbuw3}yBfsO!@_&c{d(1Q=2v)2hjAF(+=2Da7wNi?CML6?;GR2)eEz8TDRFVyq zi<`Z%BPXzIj0^DX=9eGb3TQNMZtF3u^z!zTCi1`)ut-f5fm@ZnO2s8?zR{>|53T?9 zGH@v2UWn0E2esi%GsibAPD=l+paid*bRt{dFW0a(SjuiBw}0<+Fz#-!ol{Q*3Sz00 zTMWi;*B7u6OE>u|n_{`Xx{M!Vx-3KbSe>PhXKb@%!#I9$Z@ZiyTI%AjPdLbQ)9rF< zC8wO@qMom2)gCE3{U6nHxtdClrY=wfj^zj|q6639UDA|c1MA>b1Yu&L{!{z>Ws_OGwnGJhFBg5GVkv(X;4B`*JyKgG7ZpOQO|=DMotE9hZ=#`h00y=`~f2RN9c z`ha%7P3slbB*7RM$FWVDHqbVqs64iD+kfb{>BXYFbm98hwishm(S)$nN(tU6y;6^Qk6o^E~zy?Ea8OUu$N`wW!> zuO3ua0#7g7FeVCdAV#u3Bh-3&R^=;vh@Yd5x)@hcD=Z$7tQS@ z1_B{)>whQE`l)Klsh4N#7QMBN3xX$>;3r3sbkxVVljYqGMt`0xx3d~`v@Ca$;;ZV%^shDe`~8Jx z!9wGzbHscGEH^)!oR6zkzq$0?>tkGz*>+|-TG)CO=f}dT1E+pq}msh_5fwJmF2kd_D7M0dU-yRrPVvQh%UTG-L`>>~}fk z8-II(8Ns-a^dV(Fxa+3pSmyjBeVt%WuNs*f&Yz~orYn5=<#L_wS0zVKwdt)TYiA8H z`YLVK_Z6}U?CH4CGQef!s={xV0#Q)T^SUao z>U9;1a#i@|)V`4rg~LpY6-b}rR&-$9MHCqY!Uxy0gq@!KS5cH;{Y7XH7NAcHFnw(p63LcRG34NaX{J)aIE`)yvHzt@%7li?hQIK5lI3_7p98 zr+kpvIK=KWk!KN-(%EtN#Hg4e2CBA5>Pt+KNw?Pd3>R!wZ_V|M{rY5*expBc4z8MQ zh1u~RZ;l$CWu(n!+a2nON4h;L`G2idVZU!&oF{#$8%5cDs}`mvJ>4KT9ZpHKCK)Ms zaLOnw;z$GmC+&aRd)D5_Z6yDS&if$~oZY5KQ53y6WF}c$GP&fkNV1p1?gn0p4_k4& ziVv~g#VW-_q_vPjgsF^UQhSU+(toPdPPZlP zMMY|7i`89u&A4zs@A#rjr0YKTQOqC3{87yRzQz1rJM$U4(xH*p&lL5C;>atGyyD0! ze!slJ&j4Kmr7PfY=gY}f+je{iOpbJ}=7mStf>LMveNMChkl^P57RHWQkDkG0m6LV!?j)%I& z_;g+g6Cl!=z8-Om3gJ`vn?F11YQ&?XJU36Lc{(0afI9aWpU%OADu1Nx@^aL9;}D2& zZ3N698g&FjSdXxrk9m6h0_Ohol=wfM`TSIlfT=nwyr4bW5H-e9{kc;A{Gg`SZv`bv z&!|DtW$9{UKHVT~)U9;XMKm}!6uPm8xGURsj_oTe@vYI?*kd3)Y>qR!*Q!F;7C41? z8`~9E%D3IVIB%kjdw+(DLH-?+8p@r!Q}glSVkN%wr&NLBJDE(L(9eb=dAIL)Ybns7 zye>l9qal8(5IU^8yYwcO!vz>HxiFGD-WqC*)#(~rZE7E1@ADnkgWN`6`{74?#Wl+I zK(ls^nq3!Yp={I4bZ zFO~tn^*3>G!6*n9*A14}4UUT*IP?^b(P0L6HJkfQY24p%zs_&%U6HJPQr~(KbWJnT z`3kdR@j~YVW`8hjd{*gGmEn(A)#9=s`zpH$KWTVO>%)iaCTss)6yvF^{K7V`21C-u ziyFnMix?Z`HV*Fy z7E}LOMqq3lD=nCI+8W8Q&w*qtNn>nzz)hO!SSV}RF%<%H^r{}cs=s@$>M;UyvGx0*#AA_eT3n*^{V%+LRuebby=~v%B&i7oc@>1-oLM| zLDm%R|5Pd72QQdF{lR5*iClx!n4Q!YDAULG4aeeaggefE3u)~x2j$F|tZ*n_#1WR( z1YuPE9mepzzJ}!jdWTEL&w3Zh;DhSwwaRV1^>nbirg04W8MpfM{kNUvWM~V;8hBVi z<6xe^rD97V>!oUS zz_+Do^)VWMyWrQ&71Qp%$u5X%*W~`FLAO0E%N<5M#NMD)^$6_NQWc2!Q+ZSS0mON5 zH;4B-pFRfpEGU|D2S3cd0`k214jWd_iet5r%}}58H6l9b*bfg()|1)4d-Y?z!jJ{j z@aW?j{2@Wd6461i#uCSPA;t>fp^?QBPv3xI3D!Sp9YJ3>rpIeQ9g{nwvbjeCe+=XRr6DPY8`q7y1~4SAS>A1ViJ5~=k~KjaQo=X zWn5H$)iLfp6P*Isq};-{w_q%N>A@!pyWHT0udvV=a^A`27oP`Dh%jjV_xcBz^&FPF z`9+}t1^2+kVZ|QLD?}RH@4$lp-3R{>e5bHZ$J?2k!i$?vu&~;zwdn(2eYgC4k`+(9 z`f)P#5EH+7muQs20aSMYd>Y{U{QQqQhiXuNdca~>+zKyjaGI~~w?W^xKY0K7AN!X> z8>H#QWJA;YYENC&Fcd7Iw9)t8%=BuW4ORFW=wUfI#cZ});80FOiA@jdzAAp4^V^$u zcioh{HHc^Qy7F!GRP5TAE4{;O+ue1!-6#%H-=0A^NMm~jS4So7+5Pe5R*4XiAf(ZM z%Xr-AW#9kT|Nc4fE`~!}H6Zi>D%jeD_V3Ty zqrFtI26DJMwuR#mm8^yAAhoQ8XFS!cjr36Utc9zuiq-<{ucozt$5GYRW^;hL)}K=r zpywuA=+&aEe_q-1?ay>RMR9dP2)0{)K6yW;Vu&@mw}S&D!QcSl7d*URq-(L*=QP9b z^SQ-F^9ociWPYF``pTPFIu>ihKl(~y0qljHfC@RCV!f~+bEv_=16Ov5)y4ALJ6J|= zmBX{AWo3D+g%qT!P!U1tgK-*|x(eESHJeolap^q2v9kgmi=Y*cdt!gc7S{KFpwAu5 zRWDa+3MMdMDjQqF&jo0kGnnL?*(3Z5Bjx};w+qyEe}jnyxAxSJUDrpe20z8BX-jNk z54&wZ14VAWSgvrWDeS*_^r-!8OpY2bu4WRnJi!MDDFjfYbx78D%Nm+(bmkS(ucp~qM93ug1uKhx{pY^~56IZT;Ph>Op-N`F@D7A4 zmCe2g-b{gC;`a-cnW7-M3hppWk*^kky1Vg1SrgpZxwCqS<;!6}&y!8&D^<>{cTk}B z6FD;=`=as&yjl2_Td_g5wkX!%R_s_{T<>&OEroNh+8wOsKRtf>)~Orl7{t-|k_nx#_od2uf(vz^@+Z9I!m%o9fuvWf$NF+BfkknL=PEr;P?q{<9CZa1u(z zBGx)}gzz|3v9>hfB$1YYb#9{+-nxtZOKTeW2_{HNBA!?m8by*s$FbE!MTTgholGeF z5}A|J9W9)UwZb@Hjh01@QUX7SvJU=~B8`o4M5{!S-HrTzxKNBKI7zYO1lU;;F=iuU z62{=|2o`Z^v~l`%v6HyFjbZT?7{r&wAz16+)P7F`dNA#HZ*|+H&Wmjt;wkJE%3?*~ z{lwB~A!4p#9;HmjDM^*FVUqG_cSF42+x~Q3Y_mU+b}WulSceM3kSUhN&ZdO9R7Xr| z9fb-csE%HL#Qt<%Z0r0;Yr@!48tKrcVQMMJJfVQXDZ{0Ov6LwjFJpf?FSgZi5_iu^ z8itXv5fvJCH&r3{w%2Hxg6s4F`!$WGVO-#aLMWWa; z*j&Q2coF;4d9ls@fRt21YUK=eQwN4FQtD!672_#?3kC3XkfjlM8T8e8u`T`_n-rEO z1WiK<3_PJyI5>MT1tlQ?lGIcu`&*#Hv7YP0<$ z6-+ChI>%Ka3{WAlGz2@2IKtwWVSjaAY)k)tQ7-00X_7c#Q7*;y!8&rGf{k=U2r0RR zv#4Ig{&Ze!OaI}*>6oh6(L_hsn;w{Y6bT)2rKI$^>yo>du|J&`+oCV9IeyB zTTE<6kb}X0qRGH)k9NLXT<7}cEW5pbddTNf(=FPu7sCg_9&r`+2)BYBVTX_h(2>%h zHY=5AI2)AOQ1VEGNfH9~#8M+cm?gFy@&N7>MesvQo)X~TV1n4#8LV+>Z3qXB7|OWW z9rEZg;5JEuc_R!VQ3w_yO_Vi`12t-m-n;}<4@!u1M)KG8oz9j>l9S)=PS)Ok)H*rh zQ70h_FGEQPVZ(@W-|K6%r>^V%RMg2%dgMk&oX1i$M7nTF-%JNLUT%R8BhF$zTeK-? z)ULxgUoDnrcv<=!f9pnz8!)5hN3Y|q(w->rX7rGDPmY)#y&3ZEXDYJg87hM*AP{XNm4gL_C zED;?Uuc+zxj@1NtpTffp9S^gisc=hZY zcP9GNlOBow`*q${wm&_8>R(6w>!^QyO8slBQUE3YDsG=+g7x3AP+#EfD){s7Cq*VZ zW_Dxdet;$zK8AVbi|Qi4;?8Pcg%2~-F~uEo`?Uh?uc&AKPWgoW`s4DR*JHl=4F0k2 z0NjtQig7(qJt8KTue7$-3`_0yhpX~3@Q4vfc;gMJd0cPCf(P$^ybV5xwUrb4qIC20 z^8-aa{6JAG64AOMua?(pDh9IC8hFr}i>YjRfH34%`+pX9b8FB2&u-42Ho&N!pBCc! zwwnw^FR#xtI1cjoJco0Tx92%z9X&m5aIb-sJLE}@6DEaBT9@PD%f`v`#^41PykA;e zt6c2x=6w67)iRrZmIx^i$xv(2!yceV)DzjiAN6Csbch0KxCXHXe~A82B05NWC~=IZ zJ5&e{)f`GZef5SCtiRS!0*<3I)U)^(6(#D;F`HVz=X5yFT*Fdw&B_EJQc}@!A-7n| zJ)4jT7Lr--xN=ge4f&DB9C^%<$Bf8h*6IgirK8eyRDHyM=oatUQMp2#5%x2s7bLo% z$+Kmxsr03<*#+}utT*F2oM+E4%p$udf&xYM#G=O7>tMu9u6*`F~C>0 zsZG@i=P)H~zmIGuR7d~VZrDxV*w1M8;Z^8aQ0`z*RFFnf(W(2~-@opJKBm0Cr)`IB zyXj{8uoH5B=TdjN*rGlzw&#tDmU}JQR)}i@tZmvIVvG$JSM^l(Z(#jcFL6~s4Hs8y z@P~-264628s>Cs#xT+8yDy~XAeZ^G?)?Zwefa8d(dKUls@)_T&IfS*qq(n4l^O zRIon#4u}q`L19G(Pw;1DYobZ;b{RaVMPRJXj4i8wU!ecXo5g2kwIOV1fsE3B+h8lo zN2E4@-Ega^_nm0~JFRLq1ozn-3n2wp*k@3yxp@v1OfNi^RnFHvJ=H=`Q=t;icQP!+k7>j0V(*xCP*89nPu-hp?qBxvw`7*lKj5H z%47d&QFI_b@r_gF)zDrd_dW0NpmaZZ(k&T(m(q4JuCe#&+%x7OXQ_c6oPvwf*?o#U zkymB%ih<7WZo8eD*7F}oua$?T*HQ;tZcQq)Dj+l_@panuz*{YLi|Y zY7-?%stqMBOhTz`nmEE^7ipD5gj>U%rut-8dToyZx6^B75=M+sYYdMg0e>V3jg@tO z#7YgrJ4zJ=eEjuwzC@7ql=q*q+kCd3UYh~S2TCcXc`_)uws}>}MfI+0?4L?Ih_*zl ziq9`-x9lV?5we7aEy47gLD76WkhU7=L9|uQW;?q*j-?clY;}~FKTgm8X;+Jmq}4qC zOauj0SB#*1ns?LX5`rYYt5Wq|l*wm*H$z`b1w81j;6%1dJ`0&}5jA10)T*ngO>@w~ z&Rf5qp$^pP9qmJ~pC2+(x2)RYhqMDflv;-}RMP4=N=2%uAtY5my4=Q0$4<~xCQkA; zKeW^_s;o`mU!Zt^NQu@o3Y((YtinmZ#=gw!3w|PMf z4PX2&zbf4ri(y=G&l{}SWap)~?@M)lJ)dy5ezEDt->uF|A0$-P3>46gBf@0Eaawa4 ZW~i(q7^<@4q`blM|6G>05@U!00|2Qcu3-QG delta 28410 zcmV(?K-a(Yh64PC0)HQi2ndYQU1R_QXJ}z^V`yo1Wi4WLY-=uRb8l_{>^y64<3^I- z?^h7o54i)zR=>Mx;ELU%vdu-G3-^aE)-5rUD~^Mz2QI z?ZU&gw@a@p|J}8pS8!3=s#r!m@qR1?p`5=O<;5hM!@ciTdhWm1OXHVt?_{a3b)}b= zHEy`DTITR!T2+hkVm#K1?0k|{)0H{5#caGP{jwbIV_3rOY8Z_fI zRo{-jyc#Y2qJP*>OA#hY5cO&_*E7FQ?RZwqC&f(9AEPoN+@o>_3Ck_?ZC+>>S6uo4 zSOC{Qbe=g~WktjdS1;FmHA_9WQ@A$MW#yNb>+cuq@9=~3rCnx=8uViHCYcw`C#x#U zv+6dfiloeDi#!Xr$qatT6FqlHX{Uas6Cg)V{Hzh#Vt-Xm0sng0$gi5zmB}+Zx~P^b z|7v7wA-hXmMJ+&>pCNLF@JEC$>i_>Ws&(WvB_cZ|`T#U^st^}Mh*a%V$mjm5S{C!{ zN}m}$P>YU`3{s>uZAXekQbkgk(yb!xFE_T;GCu=pEQ`#YRlclnKdpKk&Ay67!Cp2@ z#F8VLvVT{jGW!e^VNjl|tg^YSF1J@2jf8*8@ziTqj<<>yzrs(}tKbPY7%RbCN&*i$ zl{~eE5^tn)6iJ4V0tn+nv5b$dv{TqQMnVY0q;MUj3S&%y*CZG#Q--lmCBVRg&a$c; zUHtFesGRE47SyY1+Gq{vcNSSGm{m+mj5RVecz+~`rVcYlBy++TmMU$G=KA$)l~)-o zeqL)rr^q5>XTB`KXto~DCGWCwk?Y%cpd}wbPy8|oE8r-YqpnS`TrC#GvPysziPv_T z{8g9!BPjfN@+L2fWNlQ*FTMg%Ov?Jkcu#r$+x)lr-#z>WX6%aO+AnXTIcK?ECe;*f zPJeDdJCX$~8^3`6(2adhiyOEAz(tjX%+`LTJRj}ky|L)$(6{kgWaHj~`tt9WaV@lm zYf&jvl**lJS!Q;+(5w6w(}8|vAHyK0vZm_TsR&6@(sC-TwZ_>T=mMCXbaDUPXDX@_^l?aCvd+e1m!daEJ-H66arQQms zxDwntPlZKVBO+2}2nr+!?g$T^ZsddokKKsm4h3g8fC}U-=(#YS5`|L+{~6+~Mi>eE zx-@o@7e?pjb6@Qo$p_$Z@^QOs3T=k9H1>%vTV>byY3lOLA2VGk@EA zCA&4(-VAvmbEU0x@_kvZ{97OtOrq0lsUl2Zu`}CGMNwAHUoW#-%ZAwP0g6F}yQb^N zaHWEL6R}-nnL$?g3A}6kdzQ`1g|8p4pG&?6c;9BVUFU>rVFB(evNirTe35aDqa2eo ziYwQ~=j9=fhM^lb$nM6GMPJr741b(`F7z98fZ1O^Bi;sC^lV_l2RK{!?!&ZL<@uVM zYX{31C<|!HD&83rNd$J0Re98#k5iv)VIkMQNp6beRT(`+491a+I(G*dZ!7~u?`}VP zKcC;O*8oq2B_l&Crw01l>;E1aAonSHm~H#_h9b|48^H0f?2eWhZvnSRAb&v`h6Dib z#zhsYrFcu!)Cu_KlS!(9?&?3lyoja*J}$wlo``4-21MP=fu6fsb!&Vpv#kwJCT|ZI zkQK{`o@bx+AY=esjv9}6GHeah(`b*O8>qYSVwqiQd;6uP!g>fn?s6E$1MpqY`n9a< zq%y1grFOGud_m`>|FZFukbjN*yOwelZ!Z=;7W;W&ufRDsw}MHQWj2|6xM2>C0aAr5 z2fHrj+Y4Ja789+GIE>FI{z&Kr#}`=qD;ng&Jrb({PhOPkTbfm(SGrvK{{tsiJuzTJ zRGWD5WEbhO5QZZ-kKnP{p({Im0-72=hFep(+vxVbY~}Ji(RCx_v413#QGvvhdH0tZ z^BV723vGSG3kIU-($N!3(JEm(``e_2oqZ z72fGpV=dF|#sT4!3V+(_`ZgZuWJ`~d9`AsMWxWkC->wVS(<)omqY;HE=7OfXr(F~? zNco3(wGHgBM{OuMZayifOAwRWBDP*-Jo6Jh)AO3OJ;KN#(1VR8hUliz#Eu$GtVg$6 zTCE&XEVWwlAe3ed5v#Q4Mx+i?E(9?eyS+9OA;A(vd73hzm4A?wFv(d@0RDuKeIgckSFuRK{BwcU1w}N z{Vmt!6awlj3fvw^L(QZ3Uy?OHG)wWy50;ZaJBAkXNCLH)4xUp1Nrl{g7Xeq20 zFU&^pr3rkSCZ*MR?77DcHeV$!1NR$w`OKW_&4wN4u)x7f$HD8wZ5PA{#?&)o1GYwj zt}BD63g(45_p^5^>k|tpOIAju`wTTrQiR0+zJTK6`F~m0n-4qIo!~59)OKT*m)v;0 z0Jhnwo=<$U@$=*O=ftcks1D2IKR--OJ-$QMCm=uyfWB)HQTK;WxBZhs{dk8*gIb)-SXkL3IopH06D3!k`AtfNi(^ zrwsnYd4B;adgM|}FvVMYdY^m<#SDU-9a<%}5e$-jLsCah1=)p^cq{qcO07jV1Z71( zMR*L!BRLm;+Jt~}LYkC1MLqUNASaD8Mg(n`pvakEq~J95?VO7tW{FdTIAI75%D@H2 zL&|l4R7)EoC?*Zean5zZg2#TFdJeYdN#IynZ!@?vrYkxU~Jo<9!^UW+*4*@@h zHmxPIcrBX^T?=jdJvIe^V=Fg>OnnS5gc=tv;XtTp$p^Qj` zkkNt1DL#iig@h|;f)x=GQsORa{aco23x8mE@{f&%9M?#o1|gfQn!@N+qTy0eN4p^^;Sjl7oLP=F-uzP2;kbrAjr!-Io%_?2Y&@V z%zyWlOm8+1^51)%H6l1Btya^H#9_U)J@`Qmx1nfQcP;JHU90tsw8p&jbd}F1t3@?* zd~_efu!5kWs=FRwV%KSFtu>zB^qdqu1bQTn$kVbulk4d&ir8SOC!nq-c6qpC#6Xda zJkdB%0~wTwsx0|lpN(>;i-Tq+4u6E2U^z3v5g8DLDjfwWln@r{&M0!ig2zq?hrkpj zHee?a)`SybsZvg0ZgmQd$$+zh^1aI`wZ*>Q8*03Gt+MBf>0+3-I1t)yxU%o_zEF0@ zaIYGi3)LFiUf$|?S(svIYS)GzIIOLXC$(#9<9s*rzg=d$n)u6lz-C%3dVd;SH*D9~ z|27kQ!o2?v#R|M`-UPjB>XTpR+3)ob@b_7jY?e=SmAsAT0{-S>FR6T<`W+9_o7TMl zKjSufx4TwUA+%E*SJ)fCUKT@7jO-y8=BsOxI!c0+Ls76R3EnKLU$n{QRq3zCAC~YT zpZ}yMb?2am{@f3K&vakhYk!T?eJ>-uJ|opoW{Q;Ko%CkL&y7ZSpzK=UgCw`6r}L;! z%Hr(7T6CL*#^r*X*&B8STx0#j)(!ed#7#F5mmfu3-lK90nrj{$kJ|tWNg;>=RYc5k z9EgX2fL-c?6MHe&!MzS3Jx*OpDGx>y>68ad#tH=RPNuf|B!*-vjelD?JV2TZ#vqgi zN`#Ww3cw+slnN@mw-#9qgfI;sUe_Vln5%Pb&sSHw z!28n*RDChk$zLO@KWCr!!|#V>@yXjNZpW+^wFi6~v-aeFAZeaAr!1O^d_Yj{l|1X{-j=Gim4JhO@r{vcb{qbqd5s>Q9N^UQ*6uEYWH!W z?G@a1(eU+E^2;<^)Roj4YT^t|wCuAV0#j20DKFBgE31$1SmNOSBh^ zU~9E9N}DuDWGt2fQUsx;U=pD9pD31@hjm>o5>r&unuE2rH-CEm_9>O>BmC^koE<+? zV!_>b@J&`oX1KG;Y7XWhwdIPs=%pBy>`&z9ce6!x`-|6&g8vbG>&=9g9`^U_FVDBP z^>ZbQ#^3%Sn%4L6Pp@?bWj!4A$a+-V$!TXjpNi7kayoR!X$w8<09iv#SGp>NOw-+d zQ?t(a`}glpNPl;ZpdQY2@Dq2Gh)OwlA0tKJG)*0+2s!UVDiNi^5FSGr~m!9i@%f_dS)ZRyxlowZv&wQ;?VYx$;hUtGCr$q+`NsL{%=$Cut9t1EjDLP}k%a7bNMJHMw5sV+lwDENm&1en zz}{|zNELYK1LGaG5wQUsWSb2ggZ3bJpS}o2gmWre`z~*%%dC9o<2>fjL#qcU2F@il zgx&Kpl)dJb)h^R-mpW9Z%_en1KH`;5+>tmwWbWf8nsOMb7OGQ@8Q8VsR=^b%Y)p_S z4Sx>FwqF9zAd%CSS!^M1Fw_c_LOyMEoQjsnLIxVgLOT7@2@4+kB|^D&P$*y$7^EwR z6JpXh#gXzf9?Z5rP7pfvx>!|tQC#h34z(gAKl-^|R{LH_*M-j8zrXqEyJ3!^OLkjF z(fg5~?kHj=PDFdKt4-N3k8yxup#3y-4S#bTs+a4K;G1~MNya1nKzT9HZM47#k1|Gr zI$Q=y1x@)@FE=Y~mppZt?I8pYI}n1Nmew%kWQr+U19`EZFg?n*c?=@*or^A#0}rYJUx4 zn1`%AwUQYxgE8=6uWJoBp98&%{IVGpQMh%&*`@5 zd05u%J5M^Hy9FG6JogaAQ_kk@a6Rc@?heiKoyu+DeCmf-TGHy1LgZF|vN|^XZNznIvD2$7HQHrZk2UQ0XD)HlSm~v9{V^MJd$Kp2kCPh_X2Rn@XfK z=NMsEcix*s`>uIEN;~RY)RFQm9c|L|-S0C97$W3d{CBPJ@wEop#P?|wj~VlQDECm@?R||aR9?>pVy;1>Un+{B z9IHXMeT)9D&pgJoPJcQ#dc8|zpMrF&%$rhSAf7Gw-FV{rb^L^<=0Z zb?t&oUYLOxV!mF3-;E)SZv1`q|Lk4qbL2L5|5sSrR5__w5^j+C;_BwEke8h=HcwM07u(-6y9SU(Cu;kAA8o4xVM0#0NuJf2KBk5kF$8JEh|Or zVNjUV#N$SV@_%$3v3x4Tf3>&vhx!FPip?yG_vrIG;@gk8u;-DRo12f$Y~Wx^yHX&R z6|LJdDi=@~%Etq>JyWd2t~|9G$C|RISmzQ6Dq}@U;9?RgYN`H&btzbT;!utiwH+rc>@CI|B z{&-uTDUZyvl z2f$c+&A}((X{@Qq{4F&#;bqJ;4HuDi*5eqDaXafE@?&6ReHPAV;d~a(c;WOjpbq3R z)_;0B0Gw?)J@WAS+D(thqm8CVc$c!69>IE>OONP7ZKdPh`4~g#5n`B?^av!jY4Lt%2H-!$l3b%s}v`dT;^q8?I z3+OmlfcbOqh*7rBk@Xd0_#9y_+Uhy3Q-5Pko+B$I+}=5S>}X@>5oCy^^9bTHX3itf zFdOISlm;0%$71Zlt((U${W7M_Bh*h}*BmJ0aHHlC)H#c0X^I5wHG+2t-4Q%((S)xd zwO_bH;FLuZ-eGn}(HV5GODiUrX~jg16@L>P zSuv5174z+KQ8R8rP_ z@b-|`Bdr=Ou9~>|{KK#pBp8`j(0_SjE;6#t12{<1QrEw6`@JgD;H}F(aNqXs-Niyu z?0Bb-$B9~ylt92bNx$z_%lCms#F{iLNVMW(kWxngLX5W7)_bw}kncCjt{LZ@I>%VG zvW|%^Tun?-Q7&OSm|(Z`Cbsa7kflYckgjCK37 zDAal*e$dNDJ+l}0H2J4$zVeOB463HvRe@WH!tTI{D2nw5S8zm>eL1(TwNlMbB4oAE zUZm=4*iwRXB%Nf4uuhHcS3A92Eef#`@r9b?OukKQfP<+`vR4`j1+21-M^q&U{Y-CL3exmgV?@nR`iT@H9~i$Y66O@ zngF=YBp10o2c;R1nrT2$YABbOVxl$And=zVE4l(uN^=glly>5Uv45v0#R+7oO0$gQ zT0)>;mIHG&RyDz6z+u&d93h=+q%g{ab~o2@QR;C2stLoHYl*)7C+N<1ng~W^!-tES=+vw0r z@CVwk;)o?9Bef9-!+%-Y_TnlNkicH~dUWB)Y&YJa`oNL_Vjc-hNPtn)OT`0>f_p~=jG~0b28@DTAVOdiH!MzI z6e==WU=%YZW?+13JtGH3LGGjC2PPqdqX;Gu!(s`h84Qjn7=?_BD;R|gk1m+vFg(U! zLT|-H8jLd;6@PDV3Nj$-VB(>pVh<+QV@w3XB%{%B2&eRSY&62;nhK9en0gV>kqM)~ zA@K>LK$nS97=;dtRX8#2K@kh1c#&}n<5O~(=!H@2PZ7f~q4>ii8AbtT#4~g)DfDDK z1GyvQ4)PQ64E0yonRtdVR?kH>oS-=q+i+^MQxOgmmw%Bq&Y|*+)FBP7DVKjdLk01V zj%EOp(G1{RG(%%b?ON_KeR(0Bz84FL({nxIY>xb_4_c8ef?@?hkt-R$r)jbY*>;Js1(XH$}xhBm>kHQ z=L9j9n-p=VQehY}y~^GxfY{2KCO~6gS3H9dry$QP5oW+BMRJ_dvM=sX*gOUtX(&V} zfB;LtRIX?$l*$w!TyX|jNiRjQ0biINHSqk#f@SN+LbA# zhC&(*(Wu~YYK#y_j* z^q9X-|2nT9605zguOIaCFR2+-!8sfr7tcucrK-W!7aC=V9 zOpZ0QO3Mhh|1rCL_f#B|64{r4?SK7+K_d*rD3=;AiX@dLr$nV(mYtP{D$ZlTk%PRV zf-ohi9Zo?=9F(w2ItPNfWNCR>ms$%_6izVV6EgENhMiV#!EGK_^z$nM+n$EyQj4Uw`-OwSI4b zwsjQ2oH)nR7-wV?#>pl{_6Ujq%|H&2*32+yudBI$xmE9WJG4J5fC%c!rXeZBR2j+a z4H9uAxG|O(5hbBLc`L}ZJ(QQh(nHzw7;q??MCXWz3|TU&DCYL=g?UCX&S6U2Du*H^ zT9X&f%H(MB_4|Ii!j}uV-hbHlWKZd0R7#U(5^mc`cvIBP)bB?Bd|R&SKd5^KGZx+y@fp>d#i?F;8cG9XH7Ec^pp*cCu+DW?_1cSq#2L-)MZu~+`yFY;j#>Ewlo>0M3OIva zI({tj7;xlR1WZOUKqMqWQp4Y7{k`P>}*@_3(@C%YVEn);FcKtDdD-?%f`> za;`ha$|M)^-TpE8=dHc^sr^!`e+g3~6SLXqkJ+!`^Bj>N=-pdR_f>O!FB$6eReM;g z`-Kjb&z3q&A649pS!5`-XM{FXf4zDnl!0sV^16bYvWhkwzao@{>1)R~vgKGW&h2F`^~1`JY;ak{5l5bDgHZ(-QpJ>fiusl!ZY zf7aXF(t1Tqbd77&_;KY%`5^Y&h1FIM^}I4*FIy=OZN6MlM+B4i7&^1Gx?lAQD{m>o zW6K8^F#+Zsu&5~W4$kF5&AXtn@#Y;Y-=Omjsej~ohky7o;paVWf}6#Sho5vR`@*Sm z8r>ZnAN6mW`bwvx+sUu=d%W4JiQO=nv^T)a-j(frRhdk4W^XG@+tUa#3a|{7m){Ha zcnjN|;(EYEj2YY`E@;g0Xv0tTpoh&Tw8xz1a839ySGmH@9h>r$Fi!0?LK{x0%uso$ zP#`yim487m$35s;`Mhg3zv_0={8}wm_EZEP2{Vdp@U$Ff$GbiP_pPPB45d5+)C&!qbe>sleoq;(46P<%OfqSVe z%n5k!Jj@B4&`it;xC`WBP9TS6V@`lY=3`Ev#(!jFPEE6CPUZy6eNlWG_3! zG_%p!nx}RAxCLc?zUGM)7M`&={W_v^HYcD%vNk86E|a%80UnmQd2$H^K^%g@1&2 zyG#7wv?oWq)qngYA;R6}{aWMQaDGDux##?bxRBegy?uy#QX-HU@1ocj~* zMX!fA>vjx5JLirJ;_#%K($@s>Y2BBtV=~rziM+j+D8hRg)>bz}58l#wZGU$Iuq$Y< zw!C4-7#h;HH*ly+wDk>}rwg?G4H|fONc3+D9Pz|~ZG%Ivxw8|@x^0Dzng}-um4mLr z`{G?;7133%ukgo_QVYFG3{jO6!W~{MH>Yjh?f1M~S2bCCpyX?2s?2J~YVVqG;yVYr)*2TRnD9Y*Z-Ejwqn`}T`|`+ZFpvEla-qIUcPRNR(-K)K+a-#|uh`UfQMUH?EB zulg z{wC?@qwPC)m{}Kz*;w`bwBEcc3EY{$ey)=zox7L#Rr0639e;7VUUf*G7R#k&>ZWte z4&Q8=>m)~*SY^rsHS!$ekD=D>e^zx zOLm(NuJu7ll$gZNb^O0U)>#%`S)EYDvh|9eoaC8YbC*d@+$v2*ab zE;+3FPp?fPk~i)qWohhi_S3?SuijjBw}h?U-duf_US;;5tMlc3DJ3*ZL8rX07RB|} zUaublwObD$%nnxZG;E*5Q<`$#wm)cxx~(_Ex0O|J-hYK{n*4N)UmRIpU1mBHNC=Hn z&KOVx5eXzEgks92AzVSpQ|?M>HZ^_`2RQPGU)zTV_jxe!L$yDaT` zw^C6tn}1mOeYG!=Kk7V5U{bGonEc1^M|$GZfa0zzlNOEQflW+rRApqIcJJGZ044E$cf$dqzxA zoS1T*d(g--jtu%h~`jzZ_&Eg zTI$5?i_*;eWg@o8leYJN`+s+ET`PAsU9Hu^d~k=@A71_WP4W6+qb=D|C5xgw#J*bD zL+p2A`;f2gq4g$Ayr)-1BjU4PXMCd$2q@eI%;Y%-h9HHt&M-@%WF%z*rtVyC@{|L2yzeQo`PB?Yju!Ka z@Qx@kj)cKE;`W+>1xEk@oaVxw`<-H=8Aeu^b)LTXARgB1z6$$ZJy`|w^MhFR)!2{4 zVz&^>`uu;BytLQvU#()>ez0$9m4CR#k8T0IEpPjYv5Lx_T9T^oP~u{do%*}ExvdCr zn7;B1Icbh5vm;KLj*uQRK5_-ipvD?Q!#UFF|Fd@`ypiMB`L8I>0@)yKTlY~(uyb}N zo1M*K+_T9B0eUErvelNQiVk;sjQ;Q8BXyQ;QuZv80zA`pRq-Q>BH!`7$A3r7c5Ryq zO&zXS(tPTpAQc-SH7mz%pR4z5|3X_xWlC7*G zK*2X@dVl$+EWL`;0L2&m=cQDOqbIzIIA{9dp)AN}NU$Dl>xL29r zZ)1WIs*B5`5=TW56MqHX<1@`enS?(9K+2COATKw+YbUFn-%#}&;_9wp8T7|;eVZC8 z*!4`}8i?vp6KR(2PMA8cGQV%5&-{MtV-&OHsSrq0f2lcKJt6HJ5*3rsLZ0TDt_h?I z@_p4Js_J1kAiC+;fksir@5+6hHJYV)ex%a}r>Yb>+9+XZfPXARUc15yXjoK~)K`8$ zL6*MM46mN9c7{zq40NQ0p^B*8Xj(co?GOnNeL}wJQ&XQ>mQ=5e-w^&PGyLx?9!gDg z{$(q1D{*i5@;t7dl+JF)u8Wx|?%S4T+Aeh@!%(vv!^6nb1KlDa6&mC0%&W}fj|4t~ zmERP65rW_Hn}4J7+hZ0b#oxc7uP2>13gx%O9mGBb?>Sr{4%W_CJINLfe9s9HLYnP( z*mqHA0;8o{D8!bgY9yrYyJ7x{;vmf*NX5mx$gHiNg3cCSZ@T@eis(~>sbUAY$yc5N zrYFh%5pEfVAa;=t->e#e5fDak{9*!O?G(OJJU~}hIDbmiHK)CGZV0ANgZP&)Z=%Fo zA3?}bszx<*lvIq>%rxC63_~#H7+$G8Db5$_EEj`4c zsaX~whU*(fNR@MlO{z;=x`9L0bxv+BH+{Of{lG(^W&70RB#vewY}$k>9mmD0>CwII zjk`#UQ>5DU(MzRxLe%pnknQ}jyhWL!)7=@698Sbgky1e6CYT1H zw2GPM++f2d9j@2Hyi%u#0Ne+%6C}d|L!fd88<`J|--z75B@ZC` z={ib3akxVJ4_DS&x0!h3(<~kO4h&BJay!jZootPX6HT#kvq9!iY zCP=xV=5-{oa!j#B@Qj(s}UTf6#B6mN3XDS9o>xH&tJkevE>9R&k4;&>Vh; zW|X*O1x$twd5P58E3Uo-9s3*V>PEqx5|mWlB}M$yU{yyGhkw0YDJ-;h%=kW4viwK_ z1>EeMA9l~mB%wXMVV-oFP|PTtn*CbkQb3q{tC^7Z^z+PQfVXfOl(01khDw1WDxW>R7~8SYO%q@I?cZDb`=1l`54}oe2mR@btgx|8`@-z z7t{5RWIP&iTdWz%{B$=T+hl+7R^paduf2@TRRKv^TNTEBc2*52YhhJ5TWzbV*>Kb@TPBk2q#!^6YWR_88KM)Pt)z#>+b*!Xl6CVOTi1L~Cq{s&)q!o>32@+O8k}8qRNL}R z)73CiyMOGi>RLt^*qVWKGe9QRJY?9GVQZe}V^7yLkE%K1PE@m8-{HvdSEL#t!7g@f zTMdv!G$(XI?D_Ox90norRTa5-^kiJS2zq2LH&v<8h)G=*kCl2>{-;Dx9XeDV%D9M* z2uklq5M-hDQbvk0OR9J18yU&zuhbY$;7BZ#`+w(=l_=+>qM@8LG#2B8#}`Ihc53a6 zZl{ACAv$Pz!QvK$BN0SfU%1ZRDR!#z$Aa*&~$j-#0*&~@FkkQr#c z?s-@XTsjy}*8*gW?eVV@s;QCkO|=?&8=dQZu^l-1Qrl{`HZ{j%3 zDtsVRR#TCmfE`QsVuBv2UCSalt5(}g`@W!n`)PUF5ErSMp*o$~W+T0az|_^lq%$!# zLgvYI{nKj7I?q93_QZYDY8dmLY4xE^ekq#>VXw5)TE-`3Fj5{VL)gzB<#4jzD1SrQ z>Wfw@bE_xHn(gC<)_Y^K7s|Q?7Qf=`msji97R02Pb=vTc&0CjLo!u$#)0AQtto+J5 zsI_*%ud3dl(55=(H)V?gT=NUF-(1|2CkYg2{S5!F_CP>47g%_RDg+7G9?Wy^Pw1rp zwo@FPH3#uTO>_wu*1X{&*9Wl2(SIruvfdN*wa;53Fae=BDrG3Nogid`XRc3CsqfQoXn6mqjCDVT=_9lVeGdMSp%NE5=%s zRw}+4Sz_h^X{r*DH7sO<}yyHf_}f97oQ-*rvFl{_-l|mlz-sc zt##$-OYLs1d@XqKQ*O3fBcK`%}?*%{ePvFA6eZi%T9TnlYlUs<}ty?OS zttSo!fNa=&E?}yY=JM=K>T?lID*JyKJrGIwUxrygaQPW`D(d^~F;PRMtg%P{gBM~af? z5c_fR6`m|9&tbN*^1Rh{TAsJthnKI%uuq~r2+s?dG<$W?h<8_{ow+Az_Pm#-3J`r= ziMfD)le)Yly5b|O{PTA_(HK4fZ$ScYkkYjlu#}Vhz<&~jVJ?5+0Bk1tuXGaeT}lnf zD2|Q<5=!`KhM*BYSFp7==d}r%0-YOGCxgJ6>zJ4p%M<`|wm*lK71Wp^Ri2Sw@=(eYBz_=P*O^#LD^65w2-o@r$O4PeVT^b6;RW3A05;g$xFwF zM|N!A-TM=NogL^%6q4Lg?~q{S`v*j|g>EX4>%@Wua72Y2_o;H2AJ9Efp3*W?$Y(11 zBSPEp^D?}xpTg0Z6!Z;{Rw&^ohWIP6PpDwI+J7E*9{ED6L(ZpdH%FTX0Mdcx0peaU z<^k~L@bUmbG_pJZ+99Ysz!Zxo4`3ui$pbu@DDvO1!3r31bLP?1>g0A!yq?*LXT!aMPJg@U^SEXla;aQgQN=?-vTBAPoQ zu*Czp1F(%T+$s7;yEL@-ns+etgX&GiaCH~LyqbD2%!L^4=`feVw-cXVh}=%~G8eQR zG5Fhf?ZrhMt+fpE9ISoiV6C+(SnFt>rGGhEC;XOBXAphXgaZei_Cl;uHKM6-t=F(% zO;2z>tSlZ@GH=6XFBPo)HDj;dMdy$n^fvzVP0i`XUNiI=1!)!qDcvwJI)Y;>9*vzw z6I&Q;uw&~&w_Lkj7Zcr5@3CN)L9bXN^ANrDBJenS$Gu+a5wN3QFZBerFGy=$?th4; zI&k@|-Z={vJ(X#Zm-nLSBRv2yp1T^dMD z?ppTCc-_*NYP^&^Qs*Q<>O*VGLY`k1G^=ioTnd2V&FOJ7SfYt?Gmtx^%*~*SCC|+ON~Y1xAj>4w z%{~5&sdY0@a!Gazz{2Tv3%Gw`iFZqb6;8pM0hUX~n*kP2%bOpqc!J)nM3hO@n;WoH z(%v$#K>FV7G)N`#Ev#Rel)i;QODFd&>k8R4zlGH&p71w6HPWenGw>owfHQFWqyx@$ zAeI=oxHN=P1ZPkslLhA<*gk24Gd;XS0^zKZ5l6D}aaem&-=iPkX&9AYW zw~^x0TX+XXyc&T-e&Pl`d@7@0D;Yk8Bh7?Q0q;eFPvLLoz^8wJL<#VPBoZ0+dkR{N z`kq3QsG?l#|^<{bNxrHW9V$aNoG=n{bEka*U zVeP|RPoawu*Ykf1L5Q`UO?nlltS6t)K792Q{7aD4)9ON;sh+}IqNxp_pyANEb+GJ% zrB2r`H{NUJ!Ema>jwduVXR`J8nsZRSgK00&)Dxp!;;84FEfCZTea^Ad(;m-(ilP{t!nApgdpXBo5)< zJ5EQ7OA$DB^@V%SBY4XXt~r$UrBj1hW>oEIurJ+GqdS4NdHi1nhaC2r&zU~<>0)S8 zeCb?6y5pCK_;;u+dRgr4haSFssgKGuJCOZ}{FwAk&cF*F>H~via-Yz>ebF4V@t;O( z*AAY)+L?bwDtHja5$EU*h6y_}Ce@$xO@47Q{t$xFq+9oOs#aLeMZaeJw|wxUCxI>1 z)J)53j{?6XDNeJu^!p`@R}pm2*v69A~%R9DWBzYb-HYV!q=7=OZ2vSme z!~cB~-~oUDNw7e*H5>ilvDE6y%F3!d>dUOev%5q`6@M``rLY^D7QX%;dso)m$dRP~ zN^Q?W?_x|b@58`i<>_|M%);X_>^6)A8ffGxmAlB&@Q_+<_`h#N6(ttQl2j}m-iHO; zmPA$_6`65mMtsrEq{pXa(rUg%18GK?&emsXK5HsA4)%xvFU?26h!0-ixg>5QO-yj$ zUoz+VOv@B-j_Do^~6W+0yw;VZ` z2Aw!ru7!+3Yj`?XiP6d4*zAmT%mxn@Ql!)~`UjA$;m|f~0J*1W@GN8-OoNW%7lF~N zq=^~N6RgylC4+lCf(Pz>RyNoa`whcqCR;jl;kwyy@V{69fkw*whv+t#2ptClBtm4F z5Pz_$xQQSak|0!}E9<2Z;H`myF79x+o|kTyZc*0jWwD%LR>~t(*V&r7vceSc9jlv@^8^&+;hLeHTQhLT`QH8ASYyo5cf zk`U2@Q6~-fM2S&hGH7p&&SC7&1$bVnE%6X}R?h21lTVR0F*DD^r-KQxqZSgHyy!Vm z77$#-09!|$!Iu`yg%I7H2kg(R7QjG5+O1d5Ot!%^=%m>t1G~^TA``gVMn^80b$?(7 zD=n>3IguEn^nV1S<_6XwY@k7J4H1-e&qKb!wCH5-@IcWx1`m-aZ34-}Q0MR}NMnQ! z*U<*%JThhUZRZCQ$s)uFK9nW0qS(9ECmOgwQtmh{-N4 zaK%+{;EPgnhs&rcji7)tEPn%dwjns; zn;2J0h^842s-elrfD=eK}mMtqnHxH9KjA!{NON?Ei><#GlWQ|6CsIUP_WJt z?93!RWuUDb9z2JgSeYbsNSG?16u3SPoFIbWBm($NfW8Zo2gKOu6;4MC*pn_01KPi^ zg((BegfRy~(SXw*9JdIW!Xzau$i8%e&ALAsmeKxFZH<7TZ}>7i#(&75s5GTHx1F#w zaGodQlLEG_$AzV}v+0RMR$x+fK>ucLwVJt4X*qKjFlz1VN#w&=t-t95{PxHUSWPwY zLJ04b^dxAKxYs@#lpYkp&wR4ll)%L+;cBv*4q4lG4F8n2cn|o3`WFv6;vL zH)?5&Rof(;&uL!HrWL$#TZv@S(y^%-G-wU)>)-*#JQ9ogP=68T9El)A(Wtf5Sx4y+ zXxl3deyL?b3X`>H(wnylJdQA60MLNi2Vz4+Xk|tN7>QQ<1fMI5L1n$HdBlJ{Ij}5Q za1c^Zp2K#|&Nu>0>Od+Of{A<i|;HbxO z;XNz3z3(6T;(t$oYN5en6dl*LncvYY)I^ZpG#z;<-ArUm_$VVo@{EiEZtNmhtlX@u z#^UzgHXau%+s>t@bb*H5X}93@?J%3sZe~#q%tzz*NJf7dF(_d>`l8MM<1B!daS1b;yiuNCmDjnbrEF>%CzJ$Zi6 zH2AANZ~>rKJaX zr;Ibw!PE@b)mR*hP1A9XX)}#Oi`k%d(@1hX`P3#tV;gwc}BXmUa_pAr#{cpkzI zKsE*MRd6bElr$^+EI_cV9CaQE(IB4yYN<^&6i`n+ncuI7`2^A-w6_zRBnCSl~~E?@+zg3g9ROk ziM`)wmm`5BZDJi(?1Y7Qk+e9*hCpx`Dzw!dQw{S)!}+I?OSEl0&V*@a(*s|w4L7V) zeSfJPr=kAPHQH-)uh5j_(TE@{gE`FtWs%Vgh5(l~QAI>FV2^;Mrd9Mg3Evfx;I=4j z!Ksl^vj~rALIRZrynW>r*b<&QrFH+I!w~~MB09GbPUrQqlMHtlnfn-DW_Ovk$9qHH z?s)mfMsb%IBL6_q>{Rmebqb&Au~&RIjekexm)X8*eUuZWIw_~C`g-cCby}~=YM|`v zws>N}H*RT$cy5eSaF#l^8>xnLCP)9Wh`J7NH5*L>Z&vj%n|yBSO2ta?&wHqj=;#NYz{P zXuBPDBqJh_73H$1=c(}JS+M{@Z=l5M6?~k*oIIN^Pl})F;_5uji|cw_EXvqiSe`HH z_1QW6OvPfoT7Ff;dj5Ka;XD@=mVbp$`1At9qT)&Mu1a8_F6vov4&QtQpDux6Qf}bj ze+5&sUN5gtinmP{e!aME8v_ks4D9@3y=p(YEU`Ws-j@QiR^{esr-&lgQ!-vJl? z+epv!?v1z&XCAtOFC_T+kdhBvC`~(Q8F`V?vTV)*$H~a5{&`6o0}n(~H{ZYc+g6@{8>B4-10G@26gA2^ zBE99x+sL`6AVlEXxO20UBnw(fX~FU2L0fBL%Fuulgf#S-T7(kIvR9rQF<{RdbiSGzMiE;8IYZqtHmY|0wy?L!M_4aYvX5M(xM(4 zQhBf+wo%VRsrOR6JOm;@Py)VG2Vy32JY^pXQM_$E?oz0o%>gf4PJgxN)bHL51H+x< z^_MRvvvjf^hzr%<^nRYEM?ydy`b#1hko3`TE3IVJyMs-?tIw7|JuyKN4&%pr+CRXQM;>DATk8RwZDowM zz*Slc1mngyHmVTrjHJwTHpvLN6E{JLVx{Lq+J7MAwE*Ld5UsLNx|b!7 z7_jGZRwt~ek*O!z(2%l=CVcnqf;{N{61;dc}VE=Jsmv2YmYpTN`EYK=CIHOaEK1tJ7AxnL4grt zasrhiIcTmO6=w}kiLd~hiaG=$BAhdk81>TWAVK_qlNU7U<+URQJmfXvEX&SwhM1Bf z1gGFFqBSrh27Z)hSi=k)+S3jZce(AGc|E_LfjZhdma5MlmqvHb3T_cLhQ9hJR@|?v z16Eikh<~p$r*UyeyB&-zsDqOl6^$f@T;~1%sndOu;<06PEKk-aj9GQu$fUfXL`WS- zA}SbTlX75T?0^(!<$M+)DlC`bNCRLdk_DCQWo9NzA3!O&{-dTN27K|OCft^$SYDMY zL>E3QUSq`$V9{kge~sr!um8RM<3Cj@CR0$kDx>_8aEMe_~0*RBtz>L zUMN!;b7o;jl^bvggk&}-rST9L%tXp!BsK+Wj9`Wv??eOV-=oS^PSPiDvXhR;5LwcA zg?~o_7VQFulY>PZQvZd!r-4S8mcl9(o@cKDg;~H1gp?@?jxl#6n$uiX!TDYI*>`1C zUZmoU^C^_`bGN6>-(My7sqpFCU6#O?ah;GSCpmo%slj+&eubq%-~9Q{#jK7A{+L}< z*YH62{CTo{=)?TO{2LG(u|T9}{0r82vVXjsPs;dpakKWg%xGgw;fr?7FA(%KtfnES zDKOsr(W9ihUdAdCcWLpsM0VVvoVRZ4fwG?G&0jyM`{ z--9K=A`Mn1&M1fo%Y_d<@;lBfkAE1j$4oPVV8sf;C}zB4E;UJBE2WrMgfkx~Q{36p zvb@|$CD~B9xY-*!astc7xB%a7e)++zfJXD?wjRSuFK<6-A`e^vi_}CBxK-(^R9wR5 z8;$Dr(E4vL1BVjsg&19RP#fMfb9~d{r1akkO7OZ#C$jbZat&*PrR+v>`+rUcbd$fbDVFQ2%lI*-%QB>o)mi#@#x_efjN=FQw#)gUr7r&Z zgo8{s-7cqAa>_X_>iKF`?UAz6|4}`ctEmKO>Hv|eFN5`5x!Bmm-89@$yb=C|et&<%URh$RqlNA|O61yj13!)$(Q zHg?n9+TT`RId*b8ZQC<2H!r`qF@51V*Q-`?9NV;M18ozE%3~Y1jemZdUM$K>7p{MA z<`5D{>G*L}hkLEJV*Tf~OxLnprH?Mokl`|z$_WKi4WBl_s`GVKfmko+>E?IPi{~xB zv@G4S&rmt=>OplS@bt1he!c%0U&m|s_WOF3zAoOMm&-Pk5PY}h{H2~PTVLQD@e4t3 z*NcmK(cE5QAP@q#et!b3pQ@&udU>{P(OcWNAb4U4e&SWVx#?Vf{bBK8exJPj=TDRG z8g$NEvDsh%-pENM6oNv^OqAssjk~CFj!Y8G@qU=4Xp)!yk;5eCy_JfF!ZZM}3StS>EknJF8Jg%W@|vzN&sq z|5}5;-(P4JEHthgkSv`1&Hv6W%1u*E5e60LP74RUa2E^#@u- zL#8mrewS0ev41C+5sV8-A5!LnyKZ`pWzJ91*9rFYs*$xv|sAqAEY7#`X$WHxbWYt{mEECDzV9J3*VIF@NeS z5C!EtudCv!URSXwSA}m*?HdVEILyRYf%GYEMF-YhM3GS-d~iKW*y-7S6-5cwUxWr> z0s6E6V}B63!hzvOtkV*+J(ekfy$4h(UDZT?r;~?`R6f8+ZJr5Lz1%#~n$NSeI6M5{ zvJ|=BVLWM%rw)-Jza%q}#KS-+x*a_WQ=gdD549(+zUd z;gm#cl97T3r;NfPjzkb}(j3?S+upPGMs6edS9IPFnc(a;MT(;6#UV4v;*!ZFmqn7j z9CkPGQheBo+bv%|?C~1&->-^#_QO4rS~H6n9UvaNTUC6B^)6OLS_>&en94XNwZ|AF zt$#}GbX(G1RHSycSlyM^j0^YkjxWkYy6%G?#r#psAI1FdTg>mZGoP_59U6K4Oi_O* zj=bW?D~`P4_sc8%4A3=Dx&jV&zMO2eZO4be;LA7X@7oGPkSu4Sb=st{m)|qtT*;?Jfw*134fMU zi`@@YiXQd+Lp-X-C~H`s*8apnmsZ`5H<-tUtU1p|+k-GO5I&W^`Lm<0Mm##obMthXr{f_7sB@3;=^RX`LVwCG zFGrm>4uJ^QM!*cBQAa?8^$5%Pn5V}tVD3*(iT~r7&rjtDn5whF3)-U%QDZFCpDXpx z4{Cb-R#2k!j2a|emaazT(+$!_-AYGYM1ylfp&NUMyRvQP*uJt7-x{rrJqFUl<~XB! ztty0Vfm4XLv0ZVceB14d^CsH3XMeaD{cJdr zcl(aFmI58h>msy08seu4p~JenOK)O1Tz~mO96P%zWMgVa+<(Xd9V}4Y z+QLAcS~aF{uMj?+muAdHpeH&D=@|`wrsRT?i%>oj=n2mlJQH3Jdcotr`t!87-t`E( z;K@bA|5~#DVj1vTe-jrMjDm1+-C%j$;JE04Lr>ut9cFM>v$@}t#{CWV>-^T<70KEs z^{ppC*EA!YuP{3nFLXX&27klGXO%ux8UBb>EiMbPudHHt&t4-97x8JG{%+(+@z_Fg|e0%BNZ@5uj3`Ym{rl<~WKH4zPnF_*@PY}{A6!0^-3f}%Nh@WbpYAkUlcuwnJAI940k4E0H0Bcg+j{qVqKJ(>Nx zS3lM(3|T-8k3O!!9};vd5gin3EOCq%VyqAz8d)sy^bIJMVEyBNh$Y}S;lp|s{iB8j z$2z(A+pixg?$W6)K_j6*R<(t)_QW^YY4G+lEc5VK_V@!mAW#A73c)q9q@us*+dv~H zLEfS&?vl;0tOt<-_=E3u#qjLfze2H9y6t)}dFR8_e4XvVzSmCc!s$ zZa-TDw~xMD#zj?s9pm0J(J6pU$}N0*3&zry9(=N}%MEV$3JaYf=bdbR@p zcgxQwS@FcHA16}}G4Y#siAE_LKy?Servbjt&;Ph{s0O8f2P}rgt?#jcIH(mTAi-CdX4jp88n?HQDVG`44ObyU)x-5+0W zl?V|DLK?k)jK_^$_WghT@1Fx-e$Z!klv>^n_3Jjzb^ZV<9<6f2Si&OgVmP!_14195 zf~`$x|Ng8!+DjE{Acw1CTR09;$y&$`Qp;L+##7DONDo!dTDbbEXf4qGYFZ0;993;? zHV3F{{W)a;dTz3XUMR+h(FNI|L!6%mv^7^i`$tDwzSvssl8m(KGWJ1gL^2wL&DC-#SI zVSNvO`rN@>^>U@AU;+cCvavP%T!6MYgGs)bJ;J{*Vh-SQyFhLCH<(y(Yft^yb$zsI z@Kdasw!|j(u-gVSP~_%|L;6MTR$t{tL+GT)+jpvd}z z|EtZDL=cR_)PXiMEd55OG#vI!KmP zNX8RnHO51wScRys2&>@w%dZM@9O2c>qrc=DxkmWwxiXoWf*HO5uIdYd&+-85Tm^hm zEy2w8tUGv^=g1B#BRu$RwE%9I0q+AI4wm@$YMPxzgiL~8u(J5qfBsAGfQ%gsPTv-P zs$|9s??AXx+3btp%@p`0e!ozeDGHLS;10tS`DziUyBj~0HNl;oJFAyiz8nVhJlSNv zQsvBg2L*aRkuw9bFDh@qn}z@AHZvJc`?GS&D!BDMO;EBG!U<>QrmM^>9%svI4^$Qq zp9DX`3z{v1DlU&etw)UoIgVC;a5c?;(xN&qtPqCC!6*PgHTC5>QO~k2z#{dqXbiK$E~Hi~I6lEY%oO#9e{7WL%!50eq=M^*%+A+%{}!05pT!bAG5|S$EegZK zlwiu<=>y{}Zn8W7c{QUif}eedsqE^ybV)Dsd=h*K!uipv>r*&j>S}In@vdPX7S}id zT!Y21b&yw1AUpG+YCFf5gRc-?AJPb_7XSz17GSWvB94DM`BV;yF3mN=B#K1bD*T?Q zSvs}w3I~d)=l*(%2YBv`Qq|-N=uB3&og%lN3u% zfSn}~V>U7-VGQ1mU=f!_8>e3vJBhp77#44VL3~*pg0&7#?e{dG2h)!CR<~X1yx688 zp2A+CELIfWPb{4lBIYXQQOb0jl2jQRCMl10H^lqB?N8^$Hv1E4$Kp7Jb*L~5nPO?| zY)Y6*b;P9BQK&$I>gYv(>`&*#w$6{VCX6kmkq&Jdrj~-t6ACDtGF)01OPMn9GWMtQ zVp|O-ardmGVHg=3QK4aXQxyUavpms;@=(b*VeUojPv^xp`$H|KEG1S&QHnV=Jfzmz z#3VdcB#JGA%_U5W7qLH`7u)O)NJ%B6R?c8IbztZsr7mVxF`lx2Pyk;CSsIa-L0_F0 z+v3l$Nnv?H&@`mLz!NHkgR>V?P!bX#NlkS^UI=}4UTlj$!zY$2qGZa$n53|~B9=-= zX-s6S(#SBbxlUfh{&Ze!vpyL= zeSRElAW_^~64>4B+7kv5tsRQG%C2qEt9IcJ@W=Pv^z9=u0Oe zqKQk0NC_t-HY}td`#{5tO_)#$($A>mW$>TQi|yamepqmWb_s3}&IMs%6W!4B{Gs}P zV1Q!-#lX(C#l&_5IT-vWnhebLXy?ntb*^vDvfHbFhkQOY-J%_PF?=BG5m#Z4a4YB$ zb_jU@9VrcJvr>tMvq7m1C67dyBq3l=EHx5@Sz_BE58zHw1V6OoDFF@+CWwul!5Wv= zhH&7Bp^Tf|A&(vdZj&UKH^LASg1>H4Ir;7G zWbI9Vt&<}jbrQ1hGL(c6HjF6uy}m|!>bmYvMV;)VM{ab)c`QXkqzk9?&2(_%~?K+I})na*um!;3~w{En!0W)fT^g8Y;?TG?!Mh|KCB?_*>W2j#by$B2JFnRL)`Ptg&MLU!8D|?94oNwyQ5}?XR%012>8wF{Xx3Sc zsc+g@4cI^LtOgw?@vJqIPjkhsq0X<>>}KAif-=zkqrs`59{K>hOv9xH^0Qe%myFD;f}{VA~o9ET3;S@fsEJ+}hbPTjLA^~2-^ z9e!ODCHdvt{vT)rT~pe3=mL%UG+d1^nR5dmCkS_f`r%Sv4}tf9Cr#J({yF};^((!R z<^SP}SI^FIXQDqn>5=HaU*~OQ`_rR;{&m#9j{4W9)W60m1yJ&@;`TWvSpOXh^#$Ip zfUBVuy-N^5J)u+(mUxGFCLj~J1JH{Ot%$Mt3`c<|1D+u(y( zTREXGN;gkGKTy=e4-~~B5v?ooYI&`uVjw%Mfd{R*n97z12t#hQ|7USGxAxrs?B@Jw z1B~kVX(6s}yU9@W^7=f3;~;Za2J(2zUQ9ss8hbW+iYY=Pjhv*L_ zqJy-D632MDLxu2A&7s88S8pi6`fCj(;5a%%J&S))QKH@)v#AAqPKWc%H7q69tV|Fh zB^50fa*MUxvk94CA({1#D<`GekRN%>k;fc)%!oW@t$r|8Ix1a9)klnfZtfs|w+v;;O{cS6r1~{l!%YIF7igXYsEupYgp~F8{Qe zrD~3a396z%1?#i#fatIq6jo&L1bZFii6vxefQh(COh^|(LMzG`5_~9%c?DYNIURDsdXqrC9RI5RHTX;LQ(~!%Wceb z>;z3^;v{eLLrWc_%Gw0}1&Rl#6qH<%SWS||Sz!fdEcTbcw)tT!g;fc2T!)N@jyj?e z3YtlOnx+OU*en^z0PNj^Zz=^b0F|5MKL?(>#RHiViSV)zEtPe^9gtB7n^?k-RivbK|*ECKmqMI hB1|?Mr!}WxhRQmEp(;C0${Q^I59euuL}Z8p0{|6Egs}hs From 5c7487a2b332dd460e5d46db9fabdd0d1896a735 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Massot?= Date: Mon, 6 Nov 2023 22:03:03 +0100 Subject: [PATCH 04/27] Reduce load per node to 2000. (#4093) --- quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs b/quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs index 5832c4a624c..01fc6a42f99 100644 --- a/quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs +++ b/quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs @@ -39,7 +39,7 @@ use crate::indexing_scheduler::scheduling::{build_physical_indexing_plan, Load}; use crate::{IndexerNodeInfo, IndexerPool}; const PIPELINE_FULL_LOAD: Load = 1_000u32; -const LOAD_PER_NODE: Load = 4_000u32; +const LOAD_PER_NODE: Load = 2_000u32; pub(crate) const MIN_DURATION_BETWEEN_SCHEDULING: Duration = if cfg!(any(test, feature = "testsuite")) { From 6580fc923ed5c5c14a00e5b3884e687875a10bc8 Mon Sep 17 00:00:00 2001 From: cxumol Date: Mon, 6 Nov 2023 15:37:00 -0800 Subject: [PATCH 05/27] Update monitoring.md (#4087) --- docs/operating/monitoring.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/operating/monitoring.md b/docs/operating/monitoring.md index 0734d8dd839..afd3d111dcb 100644 --- a/docs/operating/monitoring.md +++ b/docs/operating/monitoring.md @@ -5,9 +5,9 @@ sidebar_position: 2 You can monitor your Quickwit cluster with Grafana. -We provide two Grafana dashboards to help you monitor: +We provide three Grafana dashboards to help you monitor: - [indexers performance](https://github.com/quickwit-oss/quickwit/blob/main/monitoring/grafana/dashboards/indexers.json) -- [indexers performance](https://github.com/quickwit-oss/quickwit/blob/main/monitoring/grafana/dashboards/searchers.json) +- [searchers performance](https://github.com/quickwit-oss/quickwit/blob/main/monitoring/grafana/dashboards/searchers.json) - [metastore queries](https://github.com/quickwit-oss/quickwit/blob/main/monitoring/grafana/dashboards/metastore.json) Both dashboards relies on a prometheus datasource fed with [Quickwit metrics](../reference/metrics.md). From 0b226530b5c3af54b06b9e852f8fcc50e19368aa Mon Sep 17 00:00:00 2001 From: Adrien Guillo Date: Mon, 6 Nov 2023 20:27:21 -0500 Subject: [PATCH 06/27] Abort fetch stream tasks upon dropping multi fetch stream (#4094) --- quickwit/quickwit-ingest/src/ingest_v2/fetch.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs b/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs index cdf00f0202f..919a12498c6 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs @@ -338,6 +338,12 @@ impl MultiFetchStream { } } +impl Drop for MultiFetchStream { + fn drop(&mut self) { + self.reset(); + } +} + /// Chooses the ingester to stream records from, preferring "local" ingesters. fn select_preferred_and_failover_ingesters( self_node_id: &NodeId, From 4bd835a67f7fb48648d97288c757718bd2e82b84 Mon Sep 17 00:00:00 2001 From: Adrien Guillo Date: Tue, 7 Nov 2023 07:49:53 -0500 Subject: [PATCH 07/27] Unpin serde dependency version (#4090) Precompiled binary was removed in 1.0.84 --- quickwit/Cargo.lock | 28 ++++++++++++++++++---------- quickwit/Cargo.toml | 3 ++- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 2d4cc4b9184..a22a874b5d0 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -1837,6 +1837,7 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" dependencies = [ + "powerfmt", "serde", ] @@ -4759,6 +4760,12 @@ dependencies = [ "serde", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -6741,9 +6748,9 @@ checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" [[package]] name = "serde" -version = "1.0.171" +version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" +checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" dependencies = [ "serde_derive", ] @@ -6760,9 +6767,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.171" +version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" +checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" dependencies = [ "proc-macro2", "quote", @@ -7753,14 +7760,15 @@ dependencies = [ [[package]] name = "time" -version = "0.3.26" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a79d09ac6b08c1ab3906a2f7cc2e81a0e27c7ae89c63812df75e52bef0751e07" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" dependencies = [ "deranged", "itoa", "libc", "num_threads", + "powerfmt", "serde", "time-core", "time-macros", @@ -7768,9 +7776,9 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-fmt" @@ -7784,9 +7792,9 @@ dependencies = [ [[package]] name = "time-macros" -version = "0.2.12" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75c65469ed6b3a4809d987a41eb1dc918e9bc1d92211cbad7ae82931846f7451" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" dependencies = [ "time-core", ] diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index 893052bfa4d..3226ddeedce 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -144,7 +144,8 @@ reqwest = { version = "0.11", default-features = false, features = [ rust-embed = "6.8.1" sea-query = { version = "0" } sea-query-binder = { version = "0", features = ["sqlx-postgres", "runtime-tokio-rustls",] } -serde = { version = "= 1.0.171", features = ["derive", "rc"] } +# ^1.0.184 due to serde-rs/serde#2538 +serde = { version = "1.0.184", features = ["derive", "rc"] } serde_json = "1.0" serde_qs = { version = "0.12", features = ["warp"] } serde_with = "3.4.0" From 49d2a3971f2b0994e211e7448cd87f78643a05d2 Mon Sep 17 00:00:00 2001 From: Adrien Guillo Date: Tue, 7 Nov 2023 08:35:57 -0500 Subject: [PATCH 08/27] Honor `auto.offset.reset` parameter in Kafka source (#4095) --- docs/configuration/source-config.md | 5 +- .../src/source/kafka_source.rs | 124 ++++++++++++------ quickwit/quickwit-indexing/src/source/mod.rs | 8 +- 3 files changed, 93 insertions(+), 44 deletions(-) diff --git a/docs/configuration/source-config.md b/docs/configuration/source-config.md index 46fea0b58f8..48e517406c6 100644 --- a/docs/configuration/source-config.md +++ b/docs/configuration/source-config.md @@ -64,11 +64,14 @@ The Kafka source consumes a `topic` using the client library [librdkafka](https: - `bootstrap.servers` Comma-separated list of host and port pairs that are the addresses of a subset of the Kafka brokers in the Kafka cluster. +- `auto.offset.reset` +Defines the behavior of the source when consuming a partition for which there is no initial offset saved in the checkpoint. `earliest` consumes from the beginning of the partition, whereas `latest` (default) consumes from the end. + - `enable.auto.commit` The Kafka source manages commit offsets manually using the [checkpoint API](../overview/concepts/indexing.md#checkpoint) and disables auto-commit. - `group.id` -Kafka-based distributed indexing relies on consumer groups. Unless overridden in the client parameters, the default group ID assigned to each consumer managed by the source is `quickwit-{index_uid}-{source_id}` +Kafka-based distributed indexing relies on consumer groups. Unless overridden in the client parameters, the default group ID assigned to each consumer managed by the source is `quickwit-{index_uid}-{source_id}`. - `max.poll.interval.ms` Short max poll interval durations may cause a source to crash when back pressure from the indexer occurs. Therefore, Quickwit recommends using the default value of `300000` (5 minutes). diff --git a/quickwit/quickwit-indexing/src/source/kafka_source.rs b/quickwit/quickwit-indexing/src/source/kafka_source.rs index d2db3477425..845a1a59b72 100644 --- a/quickwit/quickwit-indexing/src/source/kafka_source.rs +++ b/quickwit/quickwit-indexing/src/source/kafka_source.rs @@ -128,7 +128,7 @@ macro_rules! return_if_err { match $expression { Ok(v) => v, Err(_) => { - debug!(concat!($lit, "The source was dropped.")); + debug!(concat!($lit, "the source was dropped")); return; } } @@ -148,19 +148,19 @@ impl ConsumerContext for RdKafkaContext { fn pre_rebalance(&self, rebalance: &Rebalance) { if let Rebalance::Revoke(tpl) = rebalance { let partitions = collect_partitions(tpl, &self.topic); - debug!(partitions=?partitions, "Revoke partitions."); + debug!(partitions=?partitions, "revoke partitions"); let (ack_tx, ack_rx) = oneshot::channel(); return_if_err!( self.events_tx .blocking_send(KafkaEvent::RevokePartitions { ack_tx }), - "Failed to send revoke message to source." + "failed to send revoke message to source" ); - return_if_err!(ack_rx.recv(), "Failed to receive revoke ack from source"); + return_if_err!(ack_rx.recv(), "failed to receive revoke ack from source"); } if let Rebalance::Assign(tpl) = rebalance { let partitions = collect_partitions(tpl, &self.topic); - debug!(partitions=?partitions, "Assign partitions."); + debug!(partitions=?partitions, "assign partitions"); let (assignment_tx, assignment_rx) = oneshot::channel(); return_if_err!( @@ -168,19 +168,23 @@ impl ConsumerContext for RdKafkaContext { partitions, assignment_tx, }), - "Failed to send assign message to source." + "failed to send assign message to source" ); let assignment = return_if_err!( assignment_rx.recv(), - "Failed to receive assignment from source." + "failed to receive assignment from source" ); - for (partition, offset) in assignment { - let mut partition = tpl - .find_partition(&self.topic, partition) - .expect("Failed to find partition in assignment. This should never happen! Please, report on https://github.com/quickwit-oss/quickwit/issues."); - partition - .set_offset(offset) - .expect("Failed to convert `Offset` to `i64`. This should never happen! Please, report on https://github.com/quickwit-oss/quickwit/issues."); + for (partition_id, offset) in assignment { + let Some(mut partition) = tpl.find_partition(&self.topic, partition_id) else { + warn!("partition `{partition_id}` not found in assignment"); + continue; + }; + if let Err(error) = partition.set_offset(offset) { + warn!( + "failed to set offset to `{offset:?}` for partition `{partition_id}`: \ + {error}" + ); + } } } } @@ -373,10 +377,15 @@ impl KafkaSource { for &partition in partitions { let partition_id = PartitionId::from(partition as i64); - let current_position = checkpoint - .position_for_partition(&partition_id) - .cloned() - .unwrap_or(Position::Beginning); + + self.state + .assigned_partitions + .insert(partition, partition_id.clone()); + + let Some(current_position) = checkpoint.position_for_partition(&partition_id).cloned() + else { + continue; + }; let next_offset = match ¤t_position { Position::Beginning => Offset::Beginning, Position::Offset(offset) => { @@ -389,9 +398,6 @@ impl KafkaSource { panic!("position of a Kafka partition should never be EOF") } }; - self.state - .assigned_partitions - .insert(partition, partition_id); self.state .current_positions .insert(partition, current_position); @@ -441,7 +447,7 @@ impl KafkaSource { topic=%self.topic, partition=%partition, num_inactive_partitions=?self.state.num_inactive_partitions, - "Reached end of partition." + "reached end of partition" ); } @@ -901,7 +907,7 @@ mod kafka_broker_tests { format!("Key {id}") } - fn get_source_config(topic: &str) -> (String, SourceConfig) { + fn get_source_config(topic: &str, auto_offset_reset: &str) -> (String, SourceConfig) { let source_id = append_random_suffix("test-kafka-source--source"); let source_config = SourceConfig { source_id: source_id.clone(), @@ -912,6 +918,7 @@ mod kafka_broker_tests { topic: topic.to_string(), client_log_level: None, client_params: json!({ + "auto.offset.reset": auto_offset_reset, "bootstrap.servers": "localhost:9092", }), enable_backfill_mode: true, @@ -1003,7 +1010,7 @@ mod kafka_broker_tests { let metastore = metastore_for_test(); let index_id = append_random_suffix("test-kafka-source--process-message--index"); let index_uid = IndexUid::new_with_random_ulid(&index_id); - let (_source_id, source_config) = get_source_config(&topic); + let (_source_id, source_config) = get_source_config(&topic, "earliest"); let SourceParams::Kafka(params) = source_config.clone().source_params else { panic!( "Expected Kafka source params, got {:?}.", @@ -1127,7 +1134,7 @@ mod kafka_broker_tests { let metastore = metastore_for_test(); let index_id = append_random_suffix("test-kafka-source--process-assign-partitions--index"); - let (source_id, source_config) = get_source_config(&topic); + let (source_id, source_config) = get_source_config(&topic, "earliest"); let index_uid = setup_index(metastore.clone(), &index_id, &source_id, &[(2, -1, 42)]).await; @@ -1169,18 +1176,14 @@ mod kafka_broker_tests { kafka_source.state.assigned_partitions, expected_assigned_partitions ); - let expected_current_positions = - HashMap::from_iter([(1, Position::Beginning), (2, Position::from(42u64))]); + let expected_current_positions = HashMap::from_iter([(2, Position::from(42u64))]); assert_eq!( kafka_source.state.current_positions, expected_current_positions ); let assignment = assignment_rx.await.unwrap(); - assert_eq!( - assignment, - &[(1, Offset::Beginning), (2, Offset::Offset(43))] - ) + assert_eq!(assignment, &[(2, Offset::Offset(43))]) } #[tokio::test] @@ -1192,7 +1195,7 @@ mod kafka_broker_tests { let metastore = metastore_for_test(); let index_id = append_random_suffix("test-kafka-source--process-revoke--partitions--index"); let index_uid = IndexUid::new_with_random_ulid(&index_id); - let (_source_id, source_config) = get_source_config(&topic); + let (_source_id, source_config) = get_source_config(&topic, "earliest"); let SourceParams::Kafka(params) = source_config.clone().source_params else { panic!( "Expected Kafka source params, got {:?}.", @@ -1250,7 +1253,7 @@ mod kafka_broker_tests { let metastore = metastore_for_test(); let index_id = append_random_suffix("test-kafka-source--process-partition-eof--index"); let index_uid = IndexUid::new_with_random_ulid(&index_id); - let (_source_id, source_config) = get_source_config(&topic); + let (_source_id, source_config) = get_source_config(&topic, "earliest"); let SourceParams::Kafka(params) = source_config.clone().source_params else { panic!( "Expected Kafka source params, got {:?}.", @@ -1288,7 +1291,7 @@ mod kafka_broker_tests { let metastore = metastore_for_test(); let index_id = append_random_suffix("test-kafka-source--suggest-truncate--index"); - let (source_id, source_config) = get_source_config(&topic); + let (source_id, source_config) = get_source_config(&topic, "earliest"); let index_uid = setup_index(metastore.clone(), &index_id, &source_id, &[(2, -1, 42)]).await; @@ -1369,9 +1372,10 @@ mod kafka_broker_tests { let source_loader = quickwit_supported_sources(); { + // Test Kafka source with empty topic. let metastore = metastore_for_test(); let index_id = append_random_suffix("test-kafka-source--index"); - let (source_id, source_config) = get_source_config(&topic); + let (source_id, source_config) = get_source_config(&topic, "earliest"); let index_uid = setup_index(metastore.clone(), &index_id, &source_id, &[]).await; let source = source_loader .load_source( @@ -1401,7 +1405,7 @@ mod kafka_broker_tests { "source_id": source_id, "topic": topic, "assigned_partitions": vec![0, 1, 2], - "current_positions": vec![(0, ""), (1, ""), (2, "")], + "current_positions": json!([]), "num_inactive_partitions": 3, "num_bytes_processed": 0, "num_messages_processed": 0, @@ -1430,7 +1434,7 @@ mod kafka_broker_tests { { let metastore = metastore_for_test(); let index_id = append_random_suffix("test-kafka-source--index"); - let (source_id, source_config) = get_source_config(&topic); + let (source_id, source_config) = get_source_config(&topic, "earliest"); let index_uid = setup_index(metastore.clone(), &index_id, &source_id, &[]).await; let source = source_loader .load_source( @@ -1491,9 +1495,10 @@ mod kafka_broker_tests { assert_eq!(exit_state, expected_state); } { + // Test Kafka source with `earliest` offset reset. let metastore = metastore_for_test(); let index_id = append_random_suffix("test-kafka-source--index"); - let (source_id, source_config) = get_source_config(&topic); + let (source_id, source_config) = get_source_config(&topic, "earliest"); let index_uid = setup_index( metastore.clone(), &index_id, @@ -1556,6 +1561,49 @@ mod kafka_broker_tests { }); assert_eq!(exit_state, expected_exit_state); } + { + // Test Kafka source with `latest` offset reset. + let metastore = metastore_for_test(); + let index_id = append_random_suffix("test-kafka-source--index"); + let (source_id, source_config) = get_source_config(&topic, "latest"); + let index_uid = setup_index(metastore.clone(), &index_id, &source_id, &[]).await; + let source = source_loader + .load_source( + SourceRuntimeArgs::for_test( + index_uid, + source_config, + metastore, + PathBuf::from("./queues"), + ), + SourceCheckpoint::default(), + ) + .await?; + let (doc_processor_mailbox, doc_processor_inbox) = universe.create_test_mailbox(); + let source_actor = SourceActor { + source, + doc_processor_mailbox: doc_processor_mailbox.clone(), + }; + let (_source_mailbox, source_handle) = universe.spawn_builder().spawn(source_actor); + let (exit_status, exit_state) = source_handle.join().await; + assert!(exit_status.is_success()); + + let messages: Vec = doc_processor_inbox.drain_for_test_typed(); + assert!(messages.is_empty()); + + let expected_state = json!({ + "index_id": index_id, + "source_id": source_id, + "topic": topic, + "assigned_partitions": vec![0, 1, 2], + "current_positions": json!([]), + "num_inactive_partitions": 3, + "num_bytes_processed": 0, + "num_messages_processed": 0, + "num_invalid_messages": 0, + "num_rebalances": 0, + }); + assert_eq!(exit_state, expected_state); + } Ok(()) } diff --git a/quickwit/quickwit-indexing/src/source/mod.rs b/quickwit/quickwit-indexing/src/source/mod.rs index 09796f7f751..94bf12fbb94 100644 --- a/quickwit/quickwit-indexing/src/source/mod.rs +++ b/quickwit/quickwit-indexing/src/source/mod.rs @@ -75,8 +75,6 @@ mod void_source; use std::path::PathBuf; use std::time::Duration; -#[cfg(not(any(feature = "kafka", feature = "kinesis", feature = "pulsar")))] -use anyhow::bail; use async_trait::async_trait; use bytes::Bytes; pub use file_source::{FileSource, FileSourceFactory}; @@ -398,7 +396,7 @@ pub async fn check_source_connectivity( #[allow(unused_variables)] SourceParams::Kafka(params) => { #[cfg(not(feature = "kafka"))] - bail!("Quickwit binary was not compiled with the `kafka` feature"); + anyhow::bail!("Quickwit binary was not compiled with the `kafka` feature"); #[cfg(feature = "kafka")] { @@ -409,7 +407,7 @@ pub async fn check_source_connectivity( #[allow(unused_variables)] SourceParams::Kinesis(params) => { #[cfg(not(feature = "kinesis"))] - bail!("Quickwit binary was not compiled with the `kinesis` feature"); + anyhow::bail!("Quickwit binary was not compiled with the `kinesis` feature"); #[cfg(feature = "kinesis")] { @@ -420,7 +418,7 @@ pub async fn check_source_connectivity( #[allow(unused_variables)] SourceParams::Pulsar(params) => { #[cfg(not(feature = "pulsar"))] - bail!("Quickwit binary was not compiled with the `pulsar` feature"); + anyhow::bail!("Quickwit binary was not compiled with the `pulsar` feature"); #[cfg(feature = "pulsar")] { From d5264f07c4abdf1aac9ff410118735c35cc0eb34 Mon Sep 17 00:00:00 2001 From: Adrien Guillo Date: Tue, 7 Nov 2023 20:20:59 -0500 Subject: [PATCH 09/27] Stop deleting queue on EOF truncate (#4096) --- .../src/ingest/ingest_controller.rs | 2 +- .../quickwit-ingest/src/ingest_v2/ingester.rs | 44 ++++++++++++------- .../quickwit-ingest/src/ingest_v2/models.rs | 32 ++++++++------ .../quickwit-ingest/src/ingest_v2/router.rs | 14 +++++- 4 files changed, 60 insertions(+), 32 deletions(-) diff --git a/quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs b/quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs index b2292ef6a74..6b92bec7a8d 100644 --- a/quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs +++ b/quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs @@ -188,7 +188,7 @@ impl IngestController { index_id=%index_uid.index_id(), source_id=%source_id, shard_ids=?PrettySample::new(&closed_shard_ids, 5), - "closed {} shards reported by router", + "closed {} shard(s) reported by router", closed_shard_ids.len() ); } diff --git a/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs b/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs index 608e2f626d1..d8204ce0da2 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs @@ -28,7 +28,7 @@ use std::time::Duration; use async_trait::async_trait; use futures::stream::FuturesUnordered; use futures::StreamExt; -use mrecordlog::error::{CreateQueueError, DeleteQueueError, TruncateError}; +use mrecordlog::error::{CreateQueueError, TruncateError}; use mrecordlog::MultiRecordLog; use quickwit_common::tower::Pool; use quickwit_common::ServiceStream; @@ -134,6 +134,11 @@ impl Ingester { .map(|queue_id| queue_id.to_string()) .collect(); + if queue_ids.is_empty() { + return Ok(()); + } + info!("closing {} shard(s)", queue_ids.len()); + for queue_id in queue_ids { append_eof_record_if_necessary(&mut state_guard.mrecordlog, &queue_id).await; @@ -171,7 +176,8 @@ impl Ingester { let primary_shard = PrimaryShard::new(follower_id.clone()); IngesterShard::Primary(primary_shard) } else { - IngesterShard::Solo(SoloShard::default()) + let solo_shard = SoloShard::new(ShardState::Open, Position::Beginning); + IngesterShard::Solo(solo_shard) }; let entry = state.shards.entry(queue_id.clone()); Ok(entry.or_insert(shard)) @@ -505,12 +511,20 @@ impl IngesterService for Ingester { for subrequest in truncate_request.subrequests { let queue_id = subrequest.queue_id(); - let to_position_inclusive = subrequest.to_position_inclusive(); - if let Some(to_offset_inclusive) = to_position_inclusive.as_u64() { + let truncate_position_opt = match subrequest.to_position_inclusive() { + Position::Beginning => None, + Position::Offset(offset) => offset.as_u64(), + Position::Eof => state_guard + .mrecordlog + .current_position(&queue_id) + .ok() + .flatten(), + }; + if let Some(truncate_position) = truncate_position_opt { match state_guard .mrecordlog - .truncate(&queue_id, to_offset_inclusive) + .truncate(&queue_id, truncate_position) .await { Ok(_) | Err(TruncateError::MissingQueue(_)) => {} @@ -518,15 +532,7 @@ impl IngesterService for Ingester { error!("failed to truncate queue `{}`: {}", queue_id, error); } } - } else if to_position_inclusive == Position::Eof { - match state_guard.mrecordlog.delete_queue(&queue_id).await { - Ok(_) | Err(DeleteQueueError::MissingQueue(_)) => {} - Err(error) => { - error!("failed to delete queue `{}`: {}", queue_id, error); - } - } - state_guard.shards.remove(&queue_id); - }; + } } let truncate_response = TruncateResponse {}; Ok(truncate_response) @@ -1230,12 +1236,16 @@ mod tests { ingester.truncate(truncate_request).await.unwrap(); let state_guard = ingester.state.read().await; - assert_eq!(state_guard.shards.len(), 1); - assert!(state_guard.shards.contains_key(&queue_id_01)); + assert_eq!(state_guard.shards.len(), 2); + assert!(state_guard.shards.contains_key(&queue_id_01)); state_guard .mrecordlog .assert_records_eq(&queue_id_01, .., &[(1, "\0\0test-doc-011")]); - assert!(!state_guard.shards.contains_key(&queue_id_02)); + + assert!(state_guard.shards.contains_key(&queue_id_02)); + state_guard + .mrecordlog + .assert_records_eq(&queue_id_02, .., &[]); } } diff --git a/quickwit/quickwit-ingest/src/ingest_v2/models.rs b/quickwit/quickwit-ingest/src/ingest_v2/models.rs index 577e28ebade..c167c81c249 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/models.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/models.rs @@ -105,28 +105,19 @@ impl fmt::Debug for SoloShard { } } -impl Default for SoloShard { - fn default() -> Self { - let (new_records_tx, new_records_rx) = watch::channel(()); - Self { - shard_state: ShardState::Open, - replication_position_inclusive: Position::Beginning, - new_records_tx, - new_records_rx, - } - } -} - impl SoloShard { pub fn new(shard_state: ShardState, replication_position_inclusive: Position) -> Self { + let (new_records_tx, new_records_rx) = watch::channel(()); Self { shard_state, replication_position_inclusive, - ..Default::default() + new_records_tx, + new_records_rx, } } } +#[derive(Debug)] pub(super) enum IngesterShard { /// A primary shard hosted on a leader and replicated on a follower. Primary(PrimaryShard), @@ -192,3 +183,18 @@ impl IngesterShard { .expect("channel should be open"); } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_new_solo_shard() { + let solo_shard = SoloShard::new(ShardState::Closed, Position::from(42u64)); + assert_eq!(solo_shard.shard_state, ShardState::Closed); + assert_eq!( + solo_shard.replication_position_inclusive, + Position::from(42u64) + ); + } +} diff --git a/quickwit/quickwit-ingest/src/ingest_v2/router.rs b/quickwit/quickwit-ingest/src/ingest_v2/router.rs index 1f1967b3580..438c871e105 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/router.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/router.rs @@ -38,7 +38,7 @@ use quickwit_proto::ingest::router::{ use quickwit_proto::ingest::{CommitTypeV2, IngestV2Error, IngestV2Result}; use quickwit_proto::types::{IndexUid, NodeId, ShardId, SourceId, SubrequestId}; use tokio::sync::RwLock; -use tracing::{error, warn}; +use tracing::{error, info, warn}; use super::ingester::PERSIST_REQUEST_TIMEOUT; use super::shard_table::ShardTable; @@ -133,6 +133,18 @@ impl IngestRouter { get_open_shards_subrequests.push(subrequest); } } + if !closed_shards.is_empty() { + info!( + "reporting {} closed shard(s) to control-plane", + closed_shards.len() + ) + } + if !unavailable_leaders.is_empty() { + info!( + "reporting {} unavailable leader(s) to control-plane", + unavailable_leaders.len() + ); + } GetOrCreateOpenShardsRequest { subrequests: get_open_shards_subrequests, closed_shards, From f1a2b0dee8e8214e7ba3ac75c5062e37e71348a4 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Wed, 8 Nov 2023 19:04:00 +0900 Subject: [PATCH 10/27] Added a --v2 hidden in the ingest client. (#4097) --- quickwit/quickwit-cli/src/index.rs | 6 +++ quickwit/quickwit-cli/src/lib.rs | 11 +++++ quickwit/quickwit-cli/src/main.rs | 30 +++++++++++- .../src/test_utils/cluster_sandbox.rs | 2 +- .../src/test_utils/mod.rs | 2 +- .../quickwit-rest-client/src/rest_client.rs | 48 ++++++++++--------- 6 files changed, 73 insertions(+), 26 deletions(-) diff --git a/quickwit/quickwit-cli/src/index.rs b/quickwit/quickwit-cli/src/index.rs index ce07f236224..1e461aaae1d 100644 --- a/quickwit/quickwit-cli/src/index.rs +++ b/quickwit/quickwit-cli/src/index.rs @@ -130,6 +130,12 @@ pub fn build_index_command() -> Command { .short('w') .help("Wait for all documents to be commited and available for search before exiting") .action(ArgAction::SetTrue), + // TODO remove me after Quickwit 0.7. + Arg::new("v2") + .long("v2") + .help("Ingest v2 (experimental! Do not use me.)") + .hide(true) + .action(ArgAction::SetTrue), Arg::new("force") .long("force") .short('f') diff --git a/quickwit/quickwit-cli/src/lib.rs b/quickwit/quickwit-cli/src/lib.rs index 3c5cdafc366..675aa20245e 100644 --- a/quickwit/quickwit-cli/src/lib.rs +++ b/quickwit/quickwit-cli/src/lib.rs @@ -106,6 +106,7 @@ pub struct ClientArgs { pub connect_timeout: Option, pub timeout: Option, pub commit_timeout: Option, + pub ingest_v2: bool, } impl Default for ClientArgs { @@ -115,6 +116,7 @@ impl Default for ClientArgs { connect_timeout: None, timeout: None, commit_timeout: None, + ingest_v2: false, } } } @@ -128,6 +130,9 @@ impl ClientArgs { if let Some(timeout) = self.timeout { builder = builder.timeout(timeout); } + if self.ingest_v2 { + builder = builder.enable_ingest_v2(); + } builder.build() } @@ -180,6 +185,11 @@ impl ClientArgs { } else { None }; + let ingest_v2 = if process_ingest { + matches.get_flag("v2") + } else { + false + }; let commit_timeout = if process_ingest { if let Some(duration) = matches.remove_one::("commit-timeout") { Some(parse_duration_or_none(&duration)?) @@ -194,6 +204,7 @@ impl ClientArgs { connect_timeout, timeout, commit_timeout, + ingest_v2, }) } } diff --git a/quickwit/quickwit-cli/src/main.rs b/quickwit/quickwit-cli/src/main.rs index 5d8ec6676da..ef757430e60 100644 --- a/quickwit/quickwit-cli/src/main.rs +++ b/quickwit/quickwit-cli/src/main.rs @@ -181,6 +181,32 @@ mod tests { Ok(()) } + #[test] + fn test_parse_ingest_v2_args() { + let app = build_cli().no_binary_name(true); + let matches = app + .try_get_matches_from(["index", "ingest", "--index", "wikipedia", "--v2"]) + .unwrap(); + let command = CliCommand::parse_cli_args(matches).unwrap(); + assert!(matches!( + command, + CliCommand::Index(IndexCliCommand::Ingest( + IngestDocsArgs { + client_args, + index_id, + input_path_opt: None, + batch_size_limit_opt: None, + commit_type: CommitType::Auto, + })) if &index_id == "wikipedia" + && client_args.timeout.is_none() + && client_args.connect_timeout.is_none() + && client_args.commit_timeout.is_none() + && client_args.cluster_endpoint == Url::from_str("http://127.0.0.1:7280").unwrap() + && client_args.ingest_v2 + + )); + } + #[test] fn test_parse_ingest_args() -> anyhow::Result<()> { let app = build_cli().no_binary_name(true); @@ -207,6 +233,7 @@ mod tests { && client_args.connect_timeout.is_none() && client_args.commit_timeout.is_none() && client_args.cluster_endpoint == Url::from_str("http://127.0.0.1:8000").unwrap() + && !client_args.ingest_v2 )); let app = build_cli().no_binary_name(true); @@ -234,8 +261,8 @@ mod tests { && client_args.timeout.is_none() && client_args.connect_timeout.is_none() && client_args.commit_timeout.is_none() + && !client_args.ingest_v2 && batch_size_limit == Byte::from_str("8MB").unwrap() - )); let app = build_cli().no_binary_name(true); @@ -263,6 +290,7 @@ mod tests { && client_args.timeout.is_none() && client_args.connect_timeout.is_none() && client_args.commit_timeout.is_none() + && !client_args.ingest_v2 && batch_size_limit == Byte::from_str("4KB").unwrap() )); diff --git a/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs b/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs index 4c94f5d9851..dc08cbd2186 100644 --- a/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs +++ b/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs @@ -108,7 +108,7 @@ macro_rules! ingest_json { }; } -pub async fn ingest_with_retry( +pub(crate) async fn ingest_with_retry( client: &QuickwitClient, index_id: &str, ingest_source: IngestSource, diff --git a/quickwit/quickwit-integration-tests/src/test_utils/mod.rs b/quickwit/quickwit-integration-tests/src/test_utils/mod.rs index 24be4084f9b..79c23571c88 100644 --- a/quickwit/quickwit-integration-tests/src/test_utils/mod.rs +++ b/quickwit/quickwit-integration-tests/src/test_utils/mod.rs @@ -19,4 +19,4 @@ mod cluster_sandbox; -pub use cluster_sandbox::{build_node_configs, ingest_with_retry, ClusterSandbox}; +pub(crate) use cluster_sandbox::{ingest_with_retry, ClusterSandbox}; diff --git a/quickwit/quickwit-rest-client/src/rest_client.rs b/quickwit/quickwit-rest-client/src/rest_client.rs index 3125c318702..52ffb9e00ee 100644 --- a/quickwit/quickwit-rest-client/src/rest_client.rs +++ b/quickwit/quickwit-rest-client/src/rest_client.rs @@ -31,6 +31,7 @@ use reqwest::header::{HeaderMap, HeaderValue, CONTENT_TYPE}; use reqwest::{Client, ClientBuilder, Method, StatusCode, Url}; use serde::Serialize; use serde_json::json; +use tracing::warn; use crate::error::Error; use crate::models::{ApiResponse, IngestSource, Timeout}; @@ -119,6 +120,8 @@ pub struct QuickwitClientBuilder { ingest_timeout: Timeout, /// Timeout for the ingest operations that require waiting for commit. commit_timeout: Timeout, + /// Experimental: if true, use the ingest v2 endpoint. + ingest_v2: bool, } impl QuickwitClientBuilder { @@ -130,6 +133,7 @@ impl QuickwitClientBuilder { search_timeout: DEFAULT_CLIENT_SEARCH_TIMEOUT, ingest_timeout: DEFAULT_CLIENT_INGEST_TIMEOUT, commit_timeout: DEFAULT_CLIENT_COMMIT_TIMEOUT, + ingest_v2: false, } } @@ -143,6 +147,12 @@ impl QuickwitClientBuilder { self } + pub fn enable_ingest_v2(mut self) -> Self { + warn!("ingest v2 experimental feature enabled!"); + self.ingest_v2 = true; + self + } + pub fn search_timeout(mut self, timeout: Timeout) -> Self { self.search_timeout = timeout; self @@ -160,13 +170,14 @@ impl QuickwitClientBuilder { pub fn build(self) -> QuickwitClient { let transport = Transport::new(self.base_url, self.connect_timeout); - QuickwitClient::new( + QuickwitClient { transport, - self.timeout, - self.search_timeout, - self.ingest_timeout, - self.commit_timeout, - ) + timeout: self.timeout, + search_timeout: self.search_timeout, + ingest_timeout: self.ingest_timeout, + commit_timeout: self.commit_timeout, + ingest_v2: self.ingest_v2, + } } } @@ -181,25 +192,12 @@ pub struct QuickwitClient { ingest_timeout: Timeout, /// Timeout for the ingest operations that require waiting for commit. commit_timeout: Timeout, + // TODO remove me after Quickwit 0.7 release. + // If true, rely on ingest v2 + ingest_v2: bool, } impl QuickwitClient { - fn new( - transport: Transport, - timeout: Timeout, - search_timeout: Timeout, - ingest_timeout: Timeout, - commit_timeout: Timeout, - ) -> Self { - Self { - transport, - timeout, - search_timeout, - ingest_timeout, - commit_timeout, - } - } - pub async fn search( &self, index_id: &str, @@ -258,7 +256,11 @@ impl QuickwitClient { on_ingest_event: Option<&(dyn Fn(IngestEvent) + Sync)>, last_block_commit: CommitType, ) -> Result<(), Error> { - let ingest_path = format!("{index_id}/ingest"); + let ingest_path = if self.ingest_v2 { + format!("{index_id}/ingest-v2") + } else { + format!("{index_id}/ingest") + }; let batch_size_limit = batch_size_limit_opt.unwrap_or(INGEST_CONTENT_LENGTH_LIMIT); let mut batch_reader = match ingest_source { IngestSource::File(filepath) => { From dca58f95cedb5ad47ebe42e7b3c020e55eae6e0f Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Wed, 8 Nov 2023 19:13:22 +0000 Subject: [PATCH 11/27] chore: replace byte-unit with bytesize (#4075) * chore: replace byte-unit with bytesize * fix: typo in method invocation * chore: bump up bytesize to 1.3.0 --------- Co-authored-by: Adrien Guillo --- quickwit/Cargo.lock | 36 ++++++------ quickwit/Cargo.toml | 2 +- quickwit/quickwit-cli/Cargo.toml | 2 +- quickwit/quickwit-cli/src/index.rs | 42 ++++++-------- quickwit/quickwit-cli/src/main.rs | 6 +- quickwit/quickwit-cli/src/tool.rs | 2 +- quickwit/quickwit-common/Cargo.toml | 2 +- quickwit/quickwit-common/src/tower/rate.rs | 6 +- quickwit/quickwit-config/Cargo.toml | 2 +- .../quickwit-config/src/index_config/mod.rs | 16 ++--- .../quickwit-config/src/node_config/mod.rs | 40 ++++++------- .../src/node_config/serialize.rs | 12 ++-- quickwit/quickwit-index-management/Cargo.toml | 1 - quickwit/quickwit-indexing/Cargo.toml | 2 +- .../quickwit-indexing/src/actors/indexer.rs | 14 ++--- .../src/actors/merge_pipeline.rs | 6 +- .../src/split_store/indexing_split_store.rs | 18 +++--- .../src/split_store/local_split_store.rs | 30 ++++------ .../src/split_store/split_store_quota.rs | 58 +++++++++---------- quickwit/quickwit-ingest/Cargo.toml | 2 +- quickwit/quickwit-ingest/src/lib.rs | 10 ++-- .../src/actors/delete_task_pipeline.rs | 2 +- .../src/actors/garbage_collector.rs | 2 +- quickwit/quickwit-metastore/Cargo.toml | 2 +- .../quickwit-metastore/src/split_metadata.rs | 10 ++-- quickwit/quickwit-search/src/service.rs | 12 ++-- quickwit/quickwit-search/src/tests.rs | 2 +- quickwit/quickwit-serve/Cargo.toml | 2 +- .../src/elastic_search_api/filter.rs | 20 +++---- .../src/index_api/rest_handler.rs | 4 +- .../src/ingest_api/rest_handler.rs | 4 +- quickwit/quickwit-serve/src/lib.rs | 6 +- quickwit/quickwit-storage/Cargo.toml | 2 +- .../src/split_cache/split_table.rs | 18 +++--- .../quickwit-storage/src/split_cache/tests.rs | 14 ++--- 35 files changed, 192 insertions(+), 217 deletions(-) diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index a22a874b5d0..38a41cc7abd 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -1041,16 +1041,6 @@ version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" -[[package]] -name = "byte-unit" -version = "4.0.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da78b32057b8fdfc352504708feeba7216dcd65a2c9ab02978cbd288d1279b6c" -dependencies = [ - "serde", - "utf8-width", -] - [[package]] name = "bytecheck" version = "0.6.11" @@ -1104,6 +1094,15 @@ dependencies = [ "either", ] +[[package]] +name = "bytesize" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" +dependencies = [ + "serde", +] + [[package]] name = "bytestring" version = "1.3.0" @@ -5137,8 +5136,8 @@ version = "0.6.3" dependencies = [ "anyhow", "async-trait", - "byte-unit", "bytes", + "bytesize", "chitchat", "clap", "colored", @@ -5263,7 +5262,7 @@ dependencies = [ "anyhow", "async-speed-limit", "async-trait", - "byte-unit", + "bytesize", "dyn-clone", "env_logger", "fnv", @@ -5297,8 +5296,8 @@ name = "quickwit-config" version = "0.6.3" dependencies = [ "anyhow", - "byte-unit", "bytes", + "bytesize", "chrono", "cron", "enum-iterator", @@ -5438,7 +5437,6 @@ version = "0.6.3" dependencies = [ "anyhow", "async-trait", - "byte-unit", "futures", "futures-util", "itertools 0.11.0", @@ -5476,8 +5474,8 @@ dependencies = [ "aws-sdk-kinesis", "aws-smithy-client", "backoff", - "byte-unit", "bytes", + "bytesize", "chitchat", "criterion", "fail", @@ -5534,8 +5532,8 @@ version = "0.6.3" dependencies = [ "anyhow", "async-trait", - "byte-unit", "bytes", + "bytesize", "dyn-clone", "flume", "futures", @@ -5694,7 +5692,7 @@ version = "0.6.3" dependencies = [ "anyhow", "async-trait", - "byte-unit", + "bytesize", "dotenv", "futures", "http", @@ -5894,8 +5892,8 @@ dependencies = [ "anyhow", "assert-json-diff 2.0.2", "async-trait", - "byte-unit", "bytes", + "bytesize", "chitchat", "elasticsearch-dsl", "futures", @@ -5965,8 +5963,8 @@ dependencies = [ "azure_storage", "azure_storage_blobs", "base64 0.21.5", - "byte-unit", "bytes", + "bytesize", "fnv", "futures", "hyper", diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index 3226ddeedce..968c7e25f0a 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -46,7 +46,7 @@ async-speed-limit = "0.4" async-trait = "0.1" backoff = { version = "0.4", features = ["tokio"] } base64 = "0.21" -byte-unit = { version = "4", default-features = false, features = ["serde", "std"] } +bytesize = {version = "1.3.0", features = ["serde"]} bytes = { version = "1", features = ["serde"] } bytestring = "1.3.0" chitchat = "0.6" diff --git a/quickwit/quickwit-cli/Cargo.toml b/quickwit/quickwit-cli/Cargo.toml index 6641a82fa14..5cfcf67e1c4 100644 --- a/quickwit/quickwit-cli/Cargo.toml +++ b/quickwit/quickwit-cli/Cargo.toml @@ -21,7 +21,7 @@ path = "src/generate_markdown.rs" [dependencies] anyhow = { workspace = true } async-trait = { workspace = true } -byte-unit = { workspace = true } +bytesize = { workspace = true } bytes = { workspace = true } chitchat = { workspace = true } clap = { workspace = true } diff --git a/quickwit/quickwit-cli/src/index.rs b/quickwit/quickwit-cli/src/index.rs index 1e461aaae1d..73b15cf7b83 100644 --- a/quickwit/quickwit-cli/src/index.rs +++ b/quickwit/quickwit-cli/src/index.rs @@ -26,8 +26,8 @@ use std::time::{Duration, Instant}; use std::{fmt, io}; use anyhow::{bail, Context}; -use byte_unit::Byte; use bytes::Bytes; +use bytesize::ByteSize; use clap::{arg, Arg, ArgAction, ArgMatches, Command}; use colored::{ColoredString, Colorize}; use humantime::format_duration; @@ -211,7 +211,7 @@ pub struct IngestDocsArgs { pub client_args: ClientArgs, pub index_id: String, pub input_path_opt: Option, - pub batch_size_limit_opt: Option, + pub batch_size_limit_opt: Option, pub commit_type: CommitType, } @@ -339,8 +339,7 @@ impl IndexCliCommand { let batch_size_limit_opt = matches .remove_one::("batch-size-limit") - .map(Byte::from_str) - .transpose()?; + .map(|s| s.parse::().unwrap()); let commit_type = match (matches.get_flag("wait"), matches.get_flag("force")) { (false, false) => CommitType::Auto, (false, true) => CommitType::Force, @@ -533,9 +532,9 @@ pub struct IndexStats { pub index_id: String, pub index_uri: Uri, pub num_published_splits: usize, - pub size_published_splits: Byte, + pub size_published_splits: ByteSize, pub num_published_docs: u64, - pub size_published_docs_uncompressed: Byte, + pub size_published_docs_uncompressed: ByteSize, pub timestamp_field_name: Option, pub timestamp_range: Option<(i64, i64)>, pub num_docs_descriptive: Option, @@ -550,13 +549,9 @@ impl Tabled for IndexStats { self.index_id.clone(), self.index_uri.to_string(), self.num_published_docs.to_string(), - self.size_published_docs_uncompressed - .get_appropriate_unit(false) - .to_string(), + self.size_published_docs_uncompressed.to_string(), self.num_published_splits.to_string(), - self.size_published_splits - .get_appropriate_unit(false) - .to_string(), + self.size_published_splits.to_string(), display_option_in_table(&self.timestamp_field_name), display_timestamp_range(&self.timestamp_range), ] @@ -659,9 +654,9 @@ impl IndexStats { index_id: index_config.index_id.clone(), index_uri: index_config.index_uri.clone(), num_published_splits: published_splits.len(), - size_published_splits: Byte::from(total_num_bytes), + size_published_splits: ByteSize(total_num_bytes), num_published_docs: total_num_docs, - size_published_docs_uncompressed: Byte::from(total_uncompressed_num_bytes), + size_published_docs_uncompressed: ByteSize(total_uncompressed_num_bytes), timestamp_field_name: index_config.doc_mapping.timestamp_field, timestamp_range, num_docs_descriptive, @@ -810,7 +805,7 @@ pub async fn ingest_docs_cli(args: IngestDocsArgs) -> anyhow::Result<()> { }; let batch_size_limit_opt = args .batch_size_limit_opt - .map(|batch_size_limit| batch_size_limit.get_bytes() as usize); + .map(|batch_size_limit| batch_size_limit.as_u64() as usize); qw_client .ingest( &args.index_id, @@ -1132,13 +1127,13 @@ mod test { let split_data_1 = Split { split_metadata: split_metadata_1, - split_state: quickwit_metastore::SplitState::Published, + split_state: SplitState::Published, update_timestamp: 0, publish_timestamp: Some(10), }; let split_data_2 = Split { split_metadata: split_metadata_2, - split_state: quickwit_metastore::SplitState::MarkedForDeletion, + split_state: SplitState::MarkedForDeletion, update_timestamp: 0, publish_timestamp: Some(10), }; @@ -1149,14 +1144,11 @@ mod test { assert_eq!(index_stats.index_id, index_id); assert_eq!(index_stats.index_uri.as_str(), index_uri); assert_eq!(index_stats.num_published_splits, 1); - assert_eq!( - index_stats.size_published_splits, - Byte::from(15_000_000usize) - ); + assert_eq!(index_stats.size_published_splits, ByteSize::mb(15)); assert_eq!(index_stats.num_published_docs, 100_000); assert_eq!( index_stats.size_published_docs_uncompressed, - Byte::from(19_000_000usize) + ByteSize::mb(19) ); assert_eq!( index_stats.timestamp_field_name, @@ -1171,7 +1163,7 @@ mod test { fn test_descriptive_stats() -> anyhow::Result<()> { let split_id = "stat-test-split".to_string(); let template_split = Split { - split_state: quickwit_metastore::SplitState::Published, + split_state: SplitState::Published, update_timestamp: 10, publish_timestamp: Some(10), split_metadata: SplitMetadata::default(), @@ -1207,12 +1199,12 @@ mod test { let num_docs_descriptive = DescriptiveStats::maybe_new(&splits_num_docs); let num_bytes_descriptive = DescriptiveStats::maybe_new(&splits_bytes); - let desciptive_stats_none = DescriptiveStats::maybe_new(&[]); + let descriptive_stats_none = DescriptiveStats::maybe_new(&[]); assert!(num_docs_descriptive.is_some()); assert!(num_bytes_descriptive.is_some()); - assert!(desciptive_stats_none.is_none()); + assert!(descriptive_stats_none.is_none()); Ok(()) } diff --git a/quickwit/quickwit-cli/src/main.rs b/quickwit/quickwit-cli/src/main.rs index ef757430e60..7cd88f3b1b2 100644 --- a/quickwit/quickwit-cli/src/main.rs +++ b/quickwit/quickwit-cli/src/main.rs @@ -92,7 +92,7 @@ mod tests { use std::str::FromStr; use std::time::Duration; - use byte_unit::Byte; + use bytesize::ByteSize; use quickwit_cli::cli::{build_cli, CliCommand}; use quickwit_cli::index::{ ClearIndexArgs, CreateIndexArgs, DeleteIndexArgs, DescribeIndexArgs, IndexCliCommand, @@ -262,7 +262,7 @@ mod tests { && client_args.connect_timeout.is_none() && client_args.commit_timeout.is_none() && !client_args.ingest_v2 - && batch_size_limit == Byte::from_str("8MB").unwrap() + && batch_size_limit == batch_size_limit == ByteSize::mb(8) )); let app = build_cli().no_binary_name(true); @@ -291,7 +291,7 @@ mod tests { && client_args.connect_timeout.is_none() && client_args.commit_timeout.is_none() && !client_args.ingest_v2 - && batch_size_limit == Byte::from_str("4KB").unwrap() + && batch_size_limit == ByteSize::kb(4) )); let app = build_cli().no_binary_name(true); diff --git a/quickwit/quickwit-cli/src/tool.rs b/quickwit/quickwit-cli/src/tool.rs index bcc5eff3868..82a040a5522 100644 --- a/quickwit/quickwit-cli/src/tool.rs +++ b/quickwit/quickwit-cli/src/tool.rs @@ -690,7 +690,7 @@ pub async fn garbage_collect_index_cli(args: GarbageCollectIndexArgs) -> anyhow: let deleted_bytes: u64 = removal_info .removed_split_entries .iter() - .map(|split_info| split_info.file_size_bytes.get_bytes()) + .map(|split_info| split_info.file_size_bytes.as_u64()) .sum(); println!( "{}MB of storage garbage collected.", diff --git a/quickwit/quickwit-common/Cargo.toml b/quickwit/quickwit-common/Cargo.toml index a3666468711..f9f799b930c 100644 --- a/quickwit/quickwit-common/Cargo.toml +++ b/quickwit/quickwit-common/Cargo.toml @@ -13,7 +13,7 @@ documentation = "https://quickwit.io/docs/" anyhow = { workspace = true } async-speed-limit = { workspace = true } async-trait = { workspace = true } -byte-unit = { workspace = true } +bytesize = { workspace = true } dyn-clone = { workspace = true } env_logger = { workspace = true } fnv = { workspace = true } diff --git a/quickwit/quickwit-common/src/tower/rate.rs b/quickwit/quickwit-common/src/tower/rate.rs index d885deacc17..a46a50293a5 100644 --- a/quickwit/quickwit-common/src/tower/rate.rs +++ b/quickwit/quickwit-common/src/tower/rate.rs @@ -19,7 +19,7 @@ use std::time::Duration; -use byte_unit::Byte; +use bytesize::ByteSize; pub trait Rate: Clone { /// Returns the amount of work per time period. @@ -49,8 +49,8 @@ impl ConstantRate { Self { work, period } } - pub fn from_bytes(bytes: Byte, period: Duration) -> Self { - let work = bytes.get_bytes(); + pub fn from_bytes(bytes: ByteSize, period: Duration) -> Self { + let work = bytes.as_u64(); Self::new(work, period) } } diff --git a/quickwit/quickwit-config/Cargo.toml b/quickwit/quickwit-config/Cargo.toml index b39ad94b660..5abe67a5009 100644 --- a/quickwit/quickwit-config/Cargo.toml +++ b/quickwit/quickwit-config/Cargo.toml @@ -11,8 +11,8 @@ documentation = "https://quickwit.io/docs/" [dependencies] anyhow = { workspace = true } -byte-unit = { workspace = true } bytes = { workspace = true } +bytesize = { workspace = true } chrono = { workspace = true } cron = { workspace = true } enum-iterator = { workspace = true } diff --git a/quickwit/quickwit-config/src/index_config/mod.rs b/quickwit/quickwit-config/src/index_config/mod.rs index 68654049593..11500068229 100644 --- a/quickwit/quickwit-config/src/index_config/mod.rs +++ b/quickwit/quickwit-config/src/index_config/mod.rs @@ -26,7 +26,7 @@ use std::sync::Arc; use std::time::Duration; use anyhow::Context; -use byte_unit::Byte; +use bytesize::ByteSize; use chrono::Utc; use cron::Schedule; use humantime::parse_duration; @@ -96,7 +96,7 @@ pub struct DocMapping { pub struct IndexingResources { #[schema(value_type = String, default = "2 GB")] #[serde(default = "IndexingResources::default_heap_size")] - pub heap_size: Byte, + pub heap_size: ByteSize, /// Sets the maximum write IO throughput in bytes/sec for the merge and delete pipelines. /// The IO limit is applied both to the downloader and to the merge executor. /// On hardware where IO is limited, this parameter can help limiting the impact of @@ -104,7 +104,7 @@ pub struct IndexingResources { #[schema(value_type = String)] #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] - pub max_merge_write_throughput: Option, + pub max_merge_write_throughput: Option, } impl PartialEq for IndexingResources { @@ -114,14 +114,14 @@ impl PartialEq for IndexingResources { } impl IndexingResources { - fn default_heap_size() -> Byte { - Byte::from_bytes(2_000_000_000) // 2GB + fn default_heap_size() -> ByteSize { + ByteSize::gb(2) } #[cfg(any(test, feature = "testsuite"))] pub fn for_test() -> Self { Self { - heap_size: Byte::from_bytes(20_000_000), // 20MB + heap_size: ByteSize::mb(20), ..Default::default() } } @@ -465,7 +465,7 @@ impl TestableForRegression for IndexConfig { }; let merge_policy = MergePolicyConfig::StableLog(stable_log_config); let indexing_resources = IndexingResources { - heap_size: Byte::from_bytes(3), + heap_size: ByteSize(3), ..Default::default() }; let indexing_settings = IndexingSettings { @@ -609,7 +609,7 @@ mod tests { assert_eq!( index_config.indexing_settings.resources, IndexingResources { - heap_size: Byte::from_bytes(3_000_000_000), + heap_size: ByteSize::gb(3), ..Default::default() } ); diff --git a/quickwit/quickwit-config/src/node_config/mod.rs b/quickwit/quickwit-config/src/node_config/mod.rs index 051208bef00..bdfbeba94f1 100644 --- a/quickwit/quickwit-config/src/node_config/mod.rs +++ b/quickwit/quickwit-config/src/node_config/mod.rs @@ -27,7 +27,7 @@ use std::path::PathBuf; use std::time::Duration; use anyhow::{bail, ensure}; -use byte_unit::Byte; +use bytesize::ByteSize; use quickwit_common::net::HostAddr; use quickwit_common::uri::Uri; use serde::{Deserialize, Serialize}; @@ -44,7 +44,7 @@ pub const DEFAULT_QW_CONFIG_PATH: &str = "config/quickwit.yaml"; #[serde(deny_unknown_fields)] pub struct IndexerConfig { #[serde(default = "IndexerConfig::default_split_store_max_num_bytes")] - pub split_store_max_num_bytes: Byte, + pub split_store_max_num_bytes: ByteSize, #[serde(default = "IndexerConfig::default_split_store_max_num_splits")] pub split_store_max_num_splits: usize, #[serde(default = "IndexerConfig::default_max_concurrent_split_uploads")] @@ -77,8 +77,8 @@ impl IndexerConfig { 12 } - pub fn default_split_store_max_num_bytes() -> Byte { - Byte::from_bytes(100_000_000_000) // 100G + pub fn default_split_store_max_num_bytes() -> ByteSize { + ByteSize::gb(100) } pub fn default_split_store_max_num_splits() -> usize { @@ -90,7 +90,7 @@ impl IndexerConfig { let indexer_config = IndexerConfig { enable_cooperative_indexing: false, enable_otlp_endpoint: true, - split_store_max_num_bytes: Byte::from_bytes(1_000_000), + split_store_max_num_bytes: ByteSize::gb(1), split_store_max_num_splits: 3, max_concurrent_split_uploads: 4, }; @@ -113,7 +113,7 @@ impl Default for IndexerConfig { #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SplitCacheLimits { - pub max_num_bytes: Byte, + pub max_num_bytes: ByteSize, #[serde(default = "SplitCacheLimits::default_max_num_splits")] pub max_num_splits: NonZeroU32, #[serde(default = "SplitCacheLimits::default_num_concurrent_downloads")] @@ -133,7 +133,7 @@ impl SplitCacheLimits { impl Default for SplitCacheLimits { fn default() -> SplitCacheLimits { SplitCacheLimits { - max_num_bytes: Byte::from_bytes(1_000_000_000), // 1 GB. + max_num_bytes: ByteSize::gb(1), max_num_splits: NonZeroU32::new(100).unwrap(), num_concurrent_downloads: NonZeroU32::new(1).unwrap(), } @@ -143,11 +143,11 @@ impl Default for SplitCacheLimits { #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields, default)] pub struct SearcherConfig { - pub aggregation_memory_limit: Byte, + pub aggregation_memory_limit: ByteSize, pub aggregation_bucket_limit: u32, - pub fast_field_cache_capacity: Byte, - pub split_footer_cache_capacity: Byte, - pub partial_request_cache_capacity: Byte, + pub fast_field_cache_capacity: ByteSize, + pub split_footer_cache_capacity: ByteSize, + pub partial_request_cache_capacity: ByteSize, pub max_num_concurrent_split_searches: usize, pub max_num_concurrent_split_streams: usize, // Strangely, if None, this will also have the effect of not forwarding @@ -160,12 +160,12 @@ pub struct SearcherConfig { impl Default for SearcherConfig { fn default() -> Self { Self { - fast_field_cache_capacity: Byte::from_bytes(1_000_000_000), // 1G - split_footer_cache_capacity: Byte::from_bytes(500_000_000), // 500M - partial_request_cache_capacity: Byte::from_bytes(64_000_000), // 64M + fast_field_cache_capacity: ByteSize::gb(1), + split_footer_cache_capacity: ByteSize::mb(500), + partial_request_cache_capacity: ByteSize::mb(64), max_num_concurrent_split_streams: 100, max_num_concurrent_split_searches: 100, - aggregation_memory_limit: Byte::from_bytes(500_000_000), // 500M + aggregation_memory_limit: ByteSize::mb(500), aggregation_bucket_limit: 65000, split_cache: None, } @@ -175,8 +175,8 @@ impl Default for SearcherConfig { #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields, default)] pub struct IngestApiConfig { - pub max_queue_memory_usage: Byte, - pub max_queue_disk_usage: Byte, + pub max_queue_memory_usage: ByteSize, + pub max_queue_disk_usage: ByteSize, pub replication_factor: usize, pub content_length_limit: u64, } @@ -184,10 +184,10 @@ pub struct IngestApiConfig { impl Default for IngestApiConfig { fn default() -> Self { Self { - max_queue_memory_usage: Byte::from_bytes(2 * 1024 * 1024 * 1024), /* 2 GiB // TODO maybe we want more? */ - max_queue_disk_usage: Byte::from_bytes(4 * 1024 * 1024 * 1024), /* 4 GiB // TODO maybe we want more? */ + max_queue_memory_usage: ByteSize::gib(2), // TODO maybe we want more? + max_queue_disk_usage: ByteSize::gib(4), // TODO maybe we want more? replication_factor: 1, - content_length_limit: 10 * 1024 * 1024, // 10 MiB + content_length_limit: ByteSize::mib(10).as_u64(), } } } diff --git a/quickwit/quickwit-config/src/node_config/serialize.rs b/quickwit/quickwit-config/src/node_config/serialize.rs index 72ccfffdd06..4ae055c66b8 100644 --- a/quickwit/quickwit-config/src/node_config/serialize.rs +++ b/quickwit/quickwit-config/src/node_config/serialize.rs @@ -398,7 +398,7 @@ mod tests { use std::num::NonZeroU64; use std::path::Path; - use byte_unit::Byte; + use bytesize::ByteSize; use itertools::Itertools; use super::*; @@ -481,7 +481,7 @@ mod tests { config.indexer_config, IndexerConfig { enable_otlp_endpoint: true, - split_store_max_num_bytes: Byte::from_str("1T").unwrap(), + split_store_max_num_bytes: ByteSize::tb(1), split_store_max_num_splits: 10_000, max_concurrent_split_uploads: 8, enable_cooperative_indexing: false, @@ -497,11 +497,11 @@ mod tests { assert_eq!( config.searcher_config, SearcherConfig { - aggregation_memory_limit: Byte::from_str("1G").unwrap(), + aggregation_memory_limit: ByteSize::gb(1), aggregation_bucket_limit: 500_000, - fast_field_cache_capacity: Byte::from_str("10G").unwrap(), - split_footer_cache_capacity: Byte::from_str("1G").unwrap(), - partial_request_cache_capacity: Byte::from_str("64M").unwrap(), + fast_field_cache_capacity: ByteSize::gb(10), + split_footer_cache_capacity: ByteSize::gb(1), + partial_request_cache_capacity: ByteSize::mb(64), max_num_concurrent_split_searches: 150, max_num_concurrent_split_streams: 120, split_cache: None, diff --git a/quickwit/quickwit-index-management/Cargo.toml b/quickwit/quickwit-index-management/Cargo.toml index 064fc509b7d..6dcebac01f2 100644 --- a/quickwit/quickwit-index-management/Cargo.toml +++ b/quickwit/quickwit-index-management/Cargo.toml @@ -12,7 +12,6 @@ documentation = "https://quickwit.io/docs/" [dependencies] anyhow = { workspace = true } async-trait = { workspace = true } -byte-unit = { workspace = true } futures = { workspace = true } futures-util = { workspace = true } itertools = { workspace = true } diff --git a/quickwit/quickwit-indexing/Cargo.toml b/quickwit/quickwit-indexing/Cargo.toml index e1ff51e155e..0cc153a49d6 100644 --- a/quickwit/quickwit-indexing/Cargo.toml +++ b/quickwit/quickwit-indexing/Cargo.toml @@ -18,8 +18,8 @@ anyhow = { workspace = true } arc-swap = { workspace = true } async-trait = { workspace = true } backoff = { workspace = true, optional = true } -byte-unit = { workspace = true } bytes = { workspace = true } +bytesize = { workspace = true } chitchat = { workspace = true } fail = { workspace = true } flume = { workspace = true } diff --git a/quickwit/quickwit-indexing/src/actors/indexer.rs b/quickwit/quickwit-indexing/src/actors/indexer.rs index 1c7bad7f183..ab9d2594375 100644 --- a/quickwit/quickwit-indexing/src/actors/indexer.rs +++ b/quickwit/quickwit-indexing/src/actors/indexer.rs @@ -25,7 +25,7 @@ use std::time::{Duration, Instant}; use anyhow::Context; use async_trait::async_trait; -use byte_unit::Byte; +use bytesize::ByteSize; use fail::fail_point; use fnv::FnvHashMap; use itertools::Itertools; @@ -230,7 +230,7 @@ impl IndexerState { publish_lock, publish_token_opt, last_delete_opstamp, - memory_usage: Byte::from_bytes(0), + memory_usage: ByteSize(0), }; Ok(workbench) } @@ -321,7 +321,7 @@ impl IndexerState { memory_usage_delta += mem_usage_after - mem_usage_before; ctx.record_progress(); } - *memory_usage = Byte::from_bytes(memory_usage.get_bytes() + memory_usage_delta); + *memory_usage = ByteSize(memory_usage.as_u64() + memory_usage_delta); Ok(()) } } @@ -347,7 +347,7 @@ struct IndexingWorkbench { // We use this value to set the `delete_opstamp` of the workbench splits. last_delete_opstamp: u64, // Number of bytes declared as used by tantivy. - memory_usage: Byte, + memory_usage: ByteSize, } pub struct Indexer { @@ -563,11 +563,11 @@ impl Indexer { }); } - fn memory_usage(&self) -> Byte { + fn memory_usage(&self) -> ByteSize { if let Some(workbench) = &self.indexing_workbench_opt { workbench.memory_usage } else { - Byte::from_bytes(0) + ByteSize(0) } } @@ -869,7 +869,7 @@ mod tests { let body_field = schema.get_field("body").unwrap(); let indexing_directory = TempDirectory::for_test(); let mut indexing_settings = IndexingSettings::for_test(); - indexing_settings.resources.heap_size = Byte::from_bytes(5_000_000); + indexing_settings.resources.heap_size = ByteSize::mb(5); let (index_serializer_mailbox, index_serializer_inbox) = universe.create_test_mailbox(); let mut metastore = MetastoreServiceClient::mock(); metastore.expect_publish_splits().never(); diff --git a/quickwit/quickwit-indexing/src/actors/merge_pipeline.rs b/quickwit/quickwit-indexing/src/actors/merge_pipeline.rs index 1a094c579cb..0ca0b00de78 100644 --- a/quickwit/quickwit-indexing/src/actors/merge_pipeline.rs +++ b/quickwit/quickwit-indexing/src/actors/merge_pipeline.rs @@ -21,7 +21,7 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use async_trait::async_trait; -use byte_unit::Byte; +use bytesize::ByteSize; use quickwit_actors::{ Actor, ActorContext, ActorExitStatus, ActorHandle, Handler, Health, Inbox, Mailbox, SpawnContext, Supervisable, HEARTBEAT, @@ -272,7 +272,7 @@ impl MergePipeline { .params .merge_max_io_num_bytes_per_sec .as_ref() - .map(|bytes_per_sec| bytes_per_sec.get_bytes() as f64) + .map(|bytes_per_sec| bytes_per_sec.as_u64() as f64) .unwrap_or(f64::INFINITY); let split_downloader_io_controls = IoControls::default() @@ -474,7 +474,7 @@ pub struct MergePipelineParams { pub split_store: IndexingSplitStore, pub merge_policy: Arc, pub max_concurrent_split_uploads: usize, //< TODO share with the indexing pipeline. - pub merge_max_io_num_bytes_per_sec: Option, + pub merge_max_io_num_bytes_per_sec: Option, pub event_broker: EventBroker, } diff --git a/quickwit/quickwit-indexing/src/split_store/indexing_split_store.rs b/quickwit/quickwit-indexing/src/split_store/indexing_split_store.rs index 50a849c9dac..097f5304032 100644 --- a/quickwit/quickwit-indexing/src/split_store/indexing_split_store.rs +++ b/quickwit/quickwit-indexing/src/split_store/indexing_split_store.rs @@ -25,7 +25,7 @@ use std::time::Instant; use anyhow::Context; #[cfg(any(test, feature = "testsuite"))] -use byte_unit::Byte; +use bytesize::ByteSize; use quickwit_common::io::{IoControls, IoControlsAccess}; use quickwit_common::uri::Uri; use quickwit_metastore::SplitMetadata; @@ -234,7 +234,7 @@ impl IndexingSplitStore { /// Takes a snapshot of the cache view (only used for testing). #[cfg(any(test, feature = "testsuite"))] - pub async fn inspect_local_store(&self) -> HashMap { + pub async fn inspect_local_store(&self) -> HashMap { self.inner.local_split_store.inspect().await } } @@ -244,7 +244,7 @@ mod tests { use std::sync::Arc; use std::time::Duration; - use byte_unit::Byte; + use bytesize::ByteSize; use quickwit_common::io::IoControls; use quickwit_metastore::{SplitMaturity, SplitMetadata}; use quickwit_storage::{RamStorage, SplitPayloadBuilder}; @@ -300,7 +300,7 @@ mod tests { assert_eq!(local_store_stats.len(), 1); assert_eq!( local_store_stats.get(&split_id1).cloned(), - Some(Byte::from_bytes(4)) + Some(ByteSize(4)) ); } { @@ -322,11 +322,11 @@ mod tests { assert_eq!(local_store_stats.len(), 2); assert_eq!( local_store_stats.get(&split_id1).cloned(), - Some(Byte::from_bytes(4)) + Some(ByteSize(4)) ); assert_eq!( local_store_stats.get(&split_id2).cloned(), - Some(Byte::from_bytes(3)) + Some(ByteSize(3)) ); Ok(()) @@ -339,7 +339,7 @@ mod tests { let split_cache_dir = tempdir()?; let local_split_store = LocalSplitStore::open( split_cache_dir.path().to_path_buf(), - SplitStoreQuota::new(1, Byte::from_bytes(1_000_000u64)), + SplitStoreQuota::new(1, ByteSize::mb(1)), ) .await?; @@ -370,7 +370,7 @@ mod tests { assert_eq!(local_store_stats.len(), 1); assert_eq!( local_store_stats.get(&split_id1).cloned(), - Some(Byte::from_bytes(11)) + Some(ByteSize(11)) ); } { @@ -395,7 +395,7 @@ mod tests { assert_eq!(local_store_stats.len(), 1); assert_eq!( local_store_stats.get(&split_id2).cloned(), - Some(Byte::from_bytes(12)) + Some(ByteSize(12)) ); } { diff --git a/quickwit/quickwit-indexing/src/split_store/local_split_store.rs b/quickwit/quickwit-indexing/src/split_store/local_split_store.rs index e436023d60a..c8911ad8ece 100644 --- a/quickwit/quickwit-indexing/src/split_store/local_split_store.rs +++ b/quickwit/quickwit-indexing/src/split_store/local_split_store.rs @@ -57,7 +57,7 @@ use std::str::FromStr; use std::time::{Duration, SystemTime}; use anyhow::Context; -use byte_unit::Byte; +use bytesize::ByteSize; use quickwit_common::split_file; use quickwit_directories::BundleDirectory; use quickwit_storage::StorageResult; @@ -86,7 +86,7 @@ pub fn get_tantivy_directory_from_split_bundle( } /// Returns the number of bytes held in a given directory. -async fn num_bytes_in_folder(directory_path: &Path) -> io::Result { +async fn num_bytes_in_folder(directory_path: &Path) -> io::Result { let mut total_bytes = 0; let mut read_dir = tokio::fs::read_dir(directory_path).await?; while let Some(dir_entry) = read_dir.next_entry().await? { @@ -100,7 +100,7 @@ async fn num_bytes_in_folder(directory_path: &Path) -> io::Result { ); } } - Ok(Byte::from_bytes(total_bytes)) + Ok(ByteSize(total_bytes)) } /// The local split store is a cache for freshly indexed splits. @@ -109,7 +109,7 @@ async fn num_bytes_in_folder(directory_path: &Path) -> io::Result { /// of a directory and the splits are built on the file upon upload. struct SplitFolder { split_id: Ulid, - num_bytes: Byte, + num_bytes: ByteSize, } impl SplitFolder { @@ -353,7 +353,7 @@ impl LocalSplitStore { } #[cfg(any(test, feature = "testsuite"))] - pub async fn inspect(&self) -> HashMap { + pub async fn inspect(&self) -> HashMap { self.inner .lock() .await @@ -420,7 +420,7 @@ mod tests { use std::io::Write; use std::path::Path; - use byte_unit::Byte; + use bytesize::ByteSize; use quickwit_directories::BundleDirectory; use quickwit_storage::{PutPayload, SplitPayloadBuilder}; use tantivy::directory::FileSlice; @@ -454,14 +454,8 @@ mod tests { LocalSplitStore::open(temp_dir.path().to_path_buf(), split_store_space_quota).await?; let cache_content = split_store.inspect().await; assert_eq!(cache_content.len(), 2); - assert_eq!( - cache_content.get(split_id1).cloned(), - Some(Byte::from_bytes(15)) - ); - assert_eq!( - cache_content.get(split_id2).cloned(), - Some(Byte::from_bytes(13)) - ); + assert_eq!(cache_content.get(split_id1).cloned(), Some(ByteSize(15))); + assert_eq!(cache_content.get(split_id2).cloned(), Some(ByteSize(13))); Ok(()) } @@ -480,7 +474,7 @@ mod tests { create_fake_split(dir.path(), "01GF521CZC1260V8QPA81T46X7", 45) .await .unwrap(); // 2 - let split_store_space_quota = SplitStoreQuota::new(2, Byte::from_bytes(1_000)); + let split_store_space_quota = SplitStoreQuota::new(2, ByteSize::kb(1)); let local_split_store = LocalSplitStore::open(dir.path().to_path_buf(), split_store_space_quota) .await @@ -503,14 +497,14 @@ mod tests { create_fake_split(dir.path(), "01GF521CZC1260V8QPA81T46X7", 45) .await .unwrap(); // 2 - let split_store_space_quota = SplitStoreQuota::new(6, Byte::from_bytes(61)); + let split_store_space_quota = SplitStoreQuota::new(6, ByteSize(61)); let local_split_store = LocalSplitStore::open(dir.path().to_path_buf(), split_store_space_quota) .await .unwrap(); let cache_content = local_split_store.inspect().await; assert_eq!(cache_content.len(), 2); - assert_eq!(cache_content.values().map(Byte::get_bytes).sum::(), 50); + assert_eq!(cache_content.values().map(|v| v.as_u64()).sum::(), 50); } #[tokio::test] @@ -532,7 +526,7 @@ mod tests { create_fake_split(dir.path(), "01GF1ZJBMBMEPMAQSFD09VTST2", 1) .await .unwrap(); - let split_store_space_quota = SplitStoreQuota::new(6, Byte::from_bytes(100)); + let split_store_space_quota = SplitStoreQuota::new(6, ByteSize(100)); let local_split_store = LocalSplitStore::open(dir.path().to_path_buf(), split_store_space_quota) .await diff --git a/quickwit/quickwit-indexing/src/split_store/split_store_quota.rs b/quickwit/quickwit-indexing/src/split_store/split_store_quota.rs index 9d60b1ec15a..a5c6062c488 100644 --- a/quickwit/quickwit-indexing/src/split_store/split_store_quota.rs +++ b/quickwit/quickwit-indexing/src/split_store/split_store_quota.rs @@ -17,7 +17,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use byte_unit::Byte; +use bytesize::ByteSize; use quickwit_config::IndexerConfig; /// A struct for keeping in check multiple SplitStore. @@ -26,18 +26,18 @@ pub struct SplitStoreQuota { /// Current number of splits in the cache. num_splits_in_cache: usize, /// Current size in bytes of splits in the cache. - size_in_bytes_in_cache: Byte, + size_in_bytes_in_cache: ByteSize, /// Maximum number of files allowed in the cache. max_num_splits: usize, /// Maximum size in bytes allowed in the cache. - max_num_bytes: Byte, + max_num_bytes: ByteSize, } impl Default for SplitStoreQuota { fn default() -> Self { Self { num_splits_in_cache: 0, - size_in_bytes_in_cache: Byte::default(), + size_in_bytes_in_cache: ByteSize::default(), max_num_bytes: IndexerConfig::default_split_store_max_num_bytes(), max_num_splits: IndexerConfig::default_split_store_max_num_splits(), } @@ -45,7 +45,7 @@ impl Default for SplitStoreQuota { } impl SplitStoreQuota { - pub fn new(max_num_splits: usize, max_num_bytes: Byte) -> Self { + pub fn new(max_num_splits: usize, max_num_bytes: ByteSize) -> Self { Self { max_num_splits, max_num_bytes, @@ -55,70 +55,68 @@ impl SplitStoreQuota { /// Space quota that prevents any caching. pub fn no_caching() -> Self { - Self::new(0, Byte::default()) + Self::new(0, ByteSize::default()) } - pub fn can_fit_split(&self, split_size_in_bytes: Byte) -> bool { + pub fn can_fit_split(&self, split_size_in_bytes: ByteSize) -> bool { if self.num_splits_in_cache >= self.max_num_splits { return false; } - if self.size_in_bytes_in_cache.get_bytes() + split_size_in_bytes.get_bytes() - > self.max_num_bytes.get_bytes() + if self.size_in_bytes_in_cache.as_u64() + split_size_in_bytes.as_u64() + > self.max_num_bytes.as_u64() { return false; } true } - pub fn add_split(&mut self, split_size_in_bytes: Byte) { + pub fn add_split(&mut self, split_size_in_bytes: ByteSize) { self.num_splits_in_cache += 1; - self.size_in_bytes_in_cache = Byte::from_bytes( - self.size_in_bytes_in_cache.get_bytes() + split_size_in_bytes.get_bytes(), - ); + self.size_in_bytes_in_cache = + ByteSize(self.size_in_bytes_in_cache.as_u64() + split_size_in_bytes.as_u64()); } - pub fn remove_split(&mut self, split_size_in_bytes: Byte) { - self.size_in_bytes_in_cache = Byte::from_bytes( - self.size_in_bytes_in_cache.get_bytes() - split_size_in_bytes.get_bytes(), - ); + pub fn remove_split(&mut self, split_size_in_bytes: ByteSize) { + self.size_in_bytes_in_cache = + ByteSize(self.size_in_bytes_in_cache.as_u64() - split_size_in_bytes.as_u64()); self.num_splits_in_cache -= 1; } - pub fn max_num_bytes(&self) -> Byte { + pub fn max_num_bytes(&self) -> ByteSize { self.max_num_bytes } } #[cfg(test)] mod tests { - use byte_unit::Byte; + use bytesize::ByteSize; use crate::split_store::SplitStoreQuota; #[test] fn test_split_store_quota_max_bytes_accepted() { - let split_store_quota = SplitStoreQuota::new(3, Byte::from_bytes(100)); - assert!(split_store_quota.can_fit_split(Byte::from_bytes(100))); + let split_store_quota = SplitStoreQuota::new(3, ByteSize(100)); + assert!(split_store_quota.can_fit_split(ByteSize(100))); } #[test] fn test_split_store_quota_exceeding_bytes() { - let split_store_quota = SplitStoreQuota::new(3, Byte::from_bytes(100)); - assert!(!split_store_quota.can_fit_split(Byte::from_bytes(101))); + let split_store_quota = SplitStoreQuota::new(3, ByteSize(100)); + assert!(!split_store_quota.can_fit_split(ByteSize(101))); } #[test] fn test_split_store_quota_max_num_files_accepted() { - let mut split_store_quota = SplitStoreQuota::new(2, Byte::from_bytes(100)); - split_store_quota.add_split(Byte::from_bytes(1)); - assert!(split_store_quota.can_fit_split(Byte::from_bytes(1))); + let mut split_store_quota = SplitStoreQuota::new(2, ByteSize(100)); + split_store_quota.add_split(ByteSize(1)); + assert!(split_store_quota.can_fit_split(ByteSize(1))); } #[test] fn test_split_store_quota_exceeding_max_num_files() { - let mut split_store_quota = SplitStoreQuota::new(2, Byte::from_bytes(100)); - split_store_quota.add_split(Byte::from_bytes(1)); - split_store_quota.add_split(Byte::from_bytes(1)); - assert!(!split_store_quota.can_fit_split(Byte::from_bytes(1))); + let mut split_store_quota = SplitStoreQuota::new(2, ByteSize(100)); + split_store_quota.add_split(ByteSize(1)); + split_store_quota.add_split(ByteSize(1)); + assert!(!split_store_quota.can_fit_split(ByteSize(1))); } } diff --git a/quickwit/quickwit-ingest/Cargo.toml b/quickwit/quickwit-ingest/Cargo.toml index f9dcbb39f41..48d95f92664 100644 --- a/quickwit/quickwit-ingest/Cargo.toml +++ b/quickwit/quickwit-ingest/Cargo.toml @@ -8,7 +8,6 @@ description = "Quickwit is a cost-efficient search engine." [dependencies] anyhow = { workspace = true } async-trait = { workspace = true } -byte-unit = { workspace = true } bytes = { workspace = true } dyn-clone = { workspace = true } flume = { workspace = true } @@ -36,6 +35,7 @@ quickwit-config = { workspace = true } quickwit-proto = { workspace = true } [dev-dependencies] +bytesize = { workspace = true } itertools = { workspace = true } mockall = { workspace = true } rand = { workspace = true } diff --git a/quickwit/quickwit-ingest/src/lib.rs b/quickwit/quickwit-ingest/src/lib.rs index 263243f27d8..fc070982785 100644 --- a/quickwit/quickwit-ingest/src/lib.rs +++ b/quickwit/quickwit-ingest/src/lib.rs @@ -72,8 +72,8 @@ pub async fn init_ingest_api( } let ingest_api_actor = IngestApiService::with_queues_dir( queues_dir_path, - config.max_queue_memory_usage.get_bytes() as usize, - config.max_queue_disk_usage.get_bytes() as usize, + config.max_queue_memory_usage.as_u64() as usize, + config.max_queue_disk_usage.as_u64() as usize, ) .await .with_context(|| { @@ -127,7 +127,7 @@ impl CommitType { #[cfg(test)] mod tests { - use byte_unit::Byte; + use bytesize::ByteSize; use quickwit_actors::AskError; use super::*; @@ -218,8 +218,8 @@ mod tests { get_ingest_api_service(&queues_dir_path).await.unwrap_err(); let ingest_api_config = IngestApiConfig { - max_queue_memory_usage: Byte::from_bytes(1200), - max_queue_disk_usage: Byte::from_bytes(1024 * 1024 * 256), + max_queue_memory_usage: ByteSize(1200), + max_queue_disk_usage: ByteSize::mib(256), ..Default::default() }; init_ingest_api(&universe, &queues_dir_path, &ingest_api_config) diff --git a/quickwit/quickwit-janitor/src/actors/delete_task_pipeline.rs b/quickwit/quickwit-janitor/src/actors/delete_task_pipeline.rs index 23e4bf43eb3..78196e48bce 100644 --- a/quickwit/quickwit-janitor/src/actors/delete_task_pipeline.rs +++ b/quickwit/quickwit-janitor/src/actors/delete_task_pipeline.rs @@ -199,7 +199,7 @@ impl DeleteTaskPipeline { .resources .max_merge_write_throughput .as_ref() - .map(|bytes_per_sec| bytes_per_sec.get_bytes() as f64) + .map(|bytes_per_sec| bytes_per_sec.as_u64() as f64) .unwrap_or(f64::INFINITY); let delete_executor_io_controls = IoControls::default() .set_throughput_limit(throughput_limit) diff --git a/quickwit/quickwit-janitor/src/actors/garbage_collector.rs b/quickwit/quickwit-janitor/src/actors/garbage_collector.rs index 61f151b5664..5e80b11d179 100644 --- a/quickwit/quickwit-janitor/src/actors/garbage_collector.rs +++ b/quickwit/quickwit-janitor/src/actors/garbage_collector.rs @@ -161,7 +161,7 @@ impl GarbageCollector { self.counters.num_deleted_files += deleted_file_entries.len(); self.counters.num_deleted_bytes += deleted_file_entries .iter() - .map(|entry| entry.file_size_bytes.get_bytes() as usize) + .map(|entry| entry.file_size_bytes.as_u64() as usize) .sum::(); } } diff --git a/quickwit/quickwit-metastore/Cargo.toml b/quickwit/quickwit-metastore/Cargo.toml index e2353120b14..74e989b928a 100644 --- a/quickwit/quickwit-metastore/Cargo.toml +++ b/quickwit/quickwit-metastore/Cargo.toml @@ -12,7 +12,7 @@ documentation = "https://quickwit.io/docs/" [dependencies] anyhow = { workspace = true } async-trait = { workspace = true } -byte-unit = { workspace = true } +bytesize = { workspace = true } futures = { workspace = true } http = { workspace = true } itertools = { workspace = true } diff --git a/quickwit/quickwit-metastore/src/split_metadata.rs b/quickwit/quickwit-metastore/src/split_metadata.rs index d8f78ffc565..d31f3d2a60e 100644 --- a/quickwit/quickwit-metastore/src/split_metadata.rs +++ b/quickwit/quickwit-metastore/src/split_metadata.rs @@ -24,7 +24,7 @@ use std::path::PathBuf; use std::str::FromStr; use std::time::Duration; -use byte_unit::Byte; +use bytesize::ByteSize; use quickwit_proto::types::IndexUid; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DurationMilliSeconds}; @@ -231,9 +231,9 @@ impl SplitMetadata { let file_name = quickwit_common::split_file(self.split_id()); SplitInfo { - uncompressed_docs_size_bytes: Byte::from_bytes(self.uncompressed_docs_size_in_bytes), + uncompressed_docs_size_bytes: ByteSize(self.uncompressed_docs_size_in_bytes), file_name: PathBuf::from(file_name), - file_size_bytes: Byte::from_bytes(self.footer_offsets.end), + file_size_bytes: ByteSize(self.footer_offsets.end), split_id: self.split_id.clone(), num_docs: self.num_docs, } @@ -249,13 +249,13 @@ pub struct SplitInfo { pub num_docs: usize, /// The sum of the sizes of the original JSON payloads in bytes. #[schema(value_type = u64)] - pub uncompressed_docs_size_bytes: Byte, + pub uncompressed_docs_size_bytes: ByteSize, /// The name of the split file on disk. #[schema(value_type = String)] pub file_name: PathBuf, /// The size of the split file on disk in bytes. #[schema(value_type = u64)] - pub file_size_bytes: Byte, + pub file_size_bytes: ByteSize, } #[cfg(any(test, feature = "testsuite"))] diff --git a/quickwit/quickwit-search/src/service.rs b/quickwit/quickwit-search/src/service.rs index ce28e2962a1..48d0d6e2e05 100644 --- a/quickwit/quickwit-search/src/service.rs +++ b/quickwit/quickwit-search/src/service.rs @@ -431,7 +431,7 @@ impl SearcherContext { /// Creates a new searcher context, given a searcher config, and an optional `SplitCache`. pub fn new(searcher_config: SearcherConfig, split_cache_opt: Option>) -> Self { - let capacity_in_bytes = searcher_config.split_footer_cache_capacity.get_bytes() as usize; + let capacity_in_bytes = searcher_config.split_footer_cache_capacity.as_u64() as usize; let global_split_footer_cache = MemorySizedCache::with_capacity_in_bytes( capacity_in_bytes, &quickwit_storage::STORAGE_METRICS.split_footer_cache, @@ -441,12 +441,10 @@ impl SearcherContext { )); let split_stream_semaphore = Semaphore::new(searcher_config.max_num_concurrent_split_streams); - let fast_field_cache_capacity = - searcher_config.fast_field_cache_capacity.get_bytes() as usize; + let fast_field_cache_capacity = searcher_config.fast_field_cache_capacity.as_u64() as usize; let storage_long_term_cache = Arc::new(QuickwitCache::new(fast_field_cache_capacity)); - let leaf_search_cache = LeafSearchCache::new( - searcher_config.partial_request_cache_capacity.get_bytes() as usize, - ); + let leaf_search_cache = + LeafSearchCache::new(searcher_config.partial_request_cache_capacity.as_u64() as usize); Self { searcher_config, @@ -462,7 +460,7 @@ impl SearcherContext { /// Returns a new instance to track the aggregation memory usage. pub fn get_aggregation_limits(&self) -> AggregationLimits { AggregationLimits::new( - Some(self.searcher_config.aggregation_memory_limit.get_bytes()), + Some(self.searcher_config.aggregation_memory_limit.as_u64()), Some(self.searcher_config.aggregation_bucket_limit), ) } diff --git a/quickwit/quickwit-search/src/tests.rs b/quickwit/quickwit-search/src/tests.rs index 6f84651a62a..aa67848511a 100644 --- a/quickwit/quickwit-search/src/tests.rs +++ b/quickwit/quickwit-search/src/tests.rs @@ -476,7 +476,7 @@ async fn test_single_node_without_timestamp_with_query_start_timestamp_enabled( let start_timestamp = OffsetDateTime::now_utc().unix_timestamp(); for i in 0..30 { let body = format!("info @ t:{}", i + 1); - docs.push(json!({"body": body})); + docs.push(json!({ "body": body })); } test_sandbox.add_documents(docs).await?; diff --git a/quickwit/quickwit-serve/Cargo.toml b/quickwit/quickwit-serve/Cargo.toml index 28d72369c7e..57b722e5633 100644 --- a/quickwit/quickwit-serve/Cargo.toml +++ b/quickwit/quickwit-serve/Cargo.toml @@ -13,7 +13,7 @@ documentation = "https://quickwit.io/docs/" anyhow = { workspace = true } async-trait = { workspace = true } bytes = { workspace = true } -byte-unit = { workspace = true } +bytesize = { workspace = true } elasticsearch-dsl = "0.4.15" futures = { workspace = true } futures-util = { workspace = true } diff --git a/quickwit/quickwit-serve/src/elastic_search_api/filter.rs b/quickwit/quickwit-serve/src/elastic_search_api/filter.rs index eae0bbfdb1c..3208d7e3e8f 100644 --- a/quickwit/quickwit-serve/src/elastic_search_api/filter.rs +++ b/quickwit/quickwit-serve/src/elastic_search_api/filter.rs @@ -17,8 +17,8 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use byte_unit::Byte; use bytes::Bytes; +use bytesize::ByteSize; use serde::de::DeserializeOwned; use warp::reject::LengthRequired; use warp::{Filter, Rejection}; @@ -29,8 +29,8 @@ use crate::elastic_search_api::model::{ }; use crate::search_api::extract_index_id_patterns; -const BODY_LENGTH_LIMIT: Byte = byte_unit::Byte::from_bytes(1_000_000); -const CONTENT_LENGTH_LIMIT: Byte = byte_unit::Byte::from_bytes(10 * 1024 * 1024); // 10MiB +const BODY_LENGTH_LIMIT: ByteSize = ByteSize::mb(1); +const CONTENT_LENGTH_LIMIT: ByteSize = ByteSize::mib(10); // TODO: Make all elastic endpoint models `utoipa` compatible // and register them here. @@ -71,7 +71,7 @@ pub(crate) fn elastic_bulk_filter( warp::path!("_elastic" / "_bulk") .and(warp::post()) .and(warp::body::content_length_limit( - CONTENT_LENGTH_LIMIT.get_bytes(), + CONTENT_LENGTH_LIMIT.as_u64(), )) .and(warp::body::bytes()) .and(serde_qs::warp::query(serde_qs::Config::default())) @@ -80,7 +80,7 @@ pub(crate) fn elastic_bulk_filter( /// Like the warp json filter, but accepts an empty body and interprets it as `T::default`. fn json_or_empty( ) -> impl Filter + Copy { - warp::body::content_length_limit(BODY_LENGTH_LIMIT.get_bytes()) + warp::body::content_length_limit(BODY_LENGTH_LIMIT.as_u64()) .and(warp::body::bytes().and_then(|buf: Bytes| async move { if buf.is_empty() { return Ok(T::default()); @@ -128,7 +128,7 @@ pub(crate) fn elastic_index_bulk_filter( warp::path!("_elastic" / String / "_bulk") .and(warp::post()) .and(warp::body::content_length_limit( - CONTENT_LENGTH_LIMIT.get_bytes(), + CONTENT_LENGTH_LIMIT.as_u64(), )) .and(warp::body::bytes()) .and(serde_qs::warp::query::( @@ -140,9 +140,7 @@ pub(crate) fn elastic_index_bulk_filter( pub(crate) fn elastic_multi_search_filter( ) -> impl Filter + Clone { warp::path!("_elastic" / "_msearch") - .and(warp::body::content_length_limit( - BODY_LENGTH_LIMIT.get_bytes(), - )) + .and(warp::body::content_length_limit(BODY_LENGTH_LIMIT.as_u64())) .and(warp::body::bytes()) .and(warp::post()) .and(serde_qs::warp::query(serde_qs::Config::default())) @@ -162,9 +160,7 @@ fn merge_scroll_body_params( pub(crate) fn elastic_scroll_filter( ) -> impl Filter + Clone { warp::path!("_elastic" / "_search" / "scroll") - .and(warp::body::content_length_limit( - BODY_LENGTH_LIMIT.get_bytes(), - )) + .and(warp::body::content_length_limit(BODY_LENGTH_LIMIT.as_u64())) .and(warp::get().or(warp::post()).unify()) .and(serde_qs::warp::query(serde_qs::Config::default())) .and(json_or_empty()) diff --git a/quickwit/quickwit-serve/src/index_api/rest_handler.rs b/quickwit/quickwit-serve/src/index_api/rest_handler.rs index c203bab2dfb..fb2a65230a0 100644 --- a/quickwit/quickwit-serve/src/index_api/rest_handler.rs +++ b/quickwit/quickwit-serve/src/index_api/rest_handler.rs @@ -1300,7 +1300,7 @@ mod tests { let resp_json: serde_json::Value = serde_json::from_slice(resp.body()).unwrap(); let expected_response_json = serde_json::json!([{ "file_name": "split_1.split", - "file_size_bytes": 800, + "file_size_bytes": "800 B", }]); assert_json_include!(actual: resp_json, expected: expected_response_json); } @@ -1314,7 +1314,7 @@ mod tests { let resp_json: serde_json::Value = serde_json::from_slice(resp.body()).unwrap(); let expected_response_json = serde_json::json!([{ "file_name": "split_1.split", - "file_size_bytes": 800, + "file_size_bytes": "800 B", }]); assert_json_include!(actual: resp_json, expected: expected_response_json); } diff --git a/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs b/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs index efe95991828..d698143d0ec 100644 --- a/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs +++ b/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs @@ -230,7 +230,7 @@ pub(crate) fn lines(body: &Bytes) -> impl Iterator { pub(crate) mod tests { use std::time::Duration; - use byte_unit::Byte; + use bytesize::ByteSize; use quickwit_actors::{Mailbox, Universe}; use quickwit_config::IngestApiConfig; use quickwit_ingest::{ @@ -333,7 +333,7 @@ pub(crate) mod tests { #[tokio::test] async fn test_ingest_api_return_429_if_above_limits() { let config = IngestApiConfig { - max_queue_memory_usage: Byte::from_bytes(1), + max_queue_memory_usage: ByteSize(1), ..Default::default() }; let (universe, _temp_dir, ingest_service, _) = diff --git a/quickwit/quickwit-serve/src/lib.rs b/quickwit/quickwit-serve/src/lib.rs index a838d8e567a..f8e8ae513eb 100644 --- a/quickwit/quickwit-serve/src/lib.rs +++ b/quickwit/quickwit-serve/src/lib.rs @@ -48,7 +48,7 @@ use std::sync::Arc; use std::time::Duration; use anyhow::Context; -use byte_unit::n_mib_bytes; +use bytesize::ByteSize; pub use format::BodyFormat; use futures::{Stream, StreamExt}; use itertools::Itertools; @@ -176,7 +176,7 @@ async fn start_ingest_client_if_needed( ) .await?; let num_buckets = NonZeroUsize::new(60).expect("60 should be non-zero"); - let initial_rate = ConstantRate::new(n_mib_bytes!(50), Duration::from_secs(1)); + let initial_rate = ConstantRate::new(ByteSize::mib(50).as_u64(), Duration::from_secs(1)); let rate_estimator = SmaRateEstimator::new( num_buckets, Duration::from_secs(10), @@ -184,7 +184,7 @@ async fn start_ingest_client_if_needed( ) .with_initial_rate(initial_rate); let memory_capacity = ingest_api_service.ask(GetMemoryCapacity).await?; - let min_rate = ConstantRate::new(n_mib_bytes!(1), Duration::from_millis(100)); + let min_rate = ConstantRate::new(ByteSize::mib(1).as_u64(), Duration::from_millis(100)); let rate_modulator = RateModulator::new(rate_estimator.clone(), memory_capacity, min_rate); let ingest_service = IngestServiceClient::tower() .ingest_layer( diff --git a/quickwit/quickwit-storage/Cargo.toml b/quickwit/quickwit-storage/Cargo.toml index 2745c326e3a..f0134547d0a 100644 --- a/quickwit/quickwit-storage/Cargo.toml +++ b/quickwit/quickwit-storage/Cargo.toml @@ -13,8 +13,8 @@ documentation = "https://quickwit.io/docs/" anyhow = { workspace = true } async-trait = { workspace = true } base64 = { workspace = true } -byte-unit = { workspace = true } bytes = { workspace = true } +bytesize = { workspace = true } fnv = { workspace = true } futures = { workspace = true } hyper = { workspace = true } diff --git a/quickwit/quickwit-storage/src/split_cache/split_table.rs b/quickwit/quickwit-storage/src/split_cache/split_table.rs index 1cbc9594ebc..a1a0a91ae5f 100644 --- a/quickwit/quickwit-storage/src/split_cache/split_table.rs +++ b/quickwit/quickwit-storage/src/split_cache/split_table.rs @@ -369,7 +369,7 @@ impl SplitTable { { return true; } - if self.on_disk_bytes > self.limits.max_num_bytes.get_bytes() { + if self.on_disk_bytes > self.limits.max_num_bytes.as_u64() { return true; } false @@ -451,7 +451,7 @@ pub(crate) struct DownloadOpportunity { mod tests { use std::num::NonZeroU32; - use byte_unit::Byte; + use bytesize::ByteSize; use quickwit_common::uri::Uri; use quickwit_config::SplitCacheLimits; use ulid::Ulid; @@ -471,7 +471,7 @@ mod tests { fn test_split_table() { let mut split_table = SplitTable::with_limits_and_existing_splits( SplitCacheLimits { - max_num_bytes: Byte::from_bytes(1000), + max_num_bytes: ByteSize::kb(1), max_num_splits: NonZeroU32::new(1).unwrap(), num_concurrent_downloads: NonZeroU32::new(1).unwrap(), }, @@ -490,7 +490,7 @@ mod tests { fn test_split_table_prefer_last_touched() { let mut split_table = SplitTable::with_limits_and_existing_splits( SplitCacheLimits { - max_num_bytes: Byte::from_bytes(1000), + max_num_bytes: ByteSize::kb(1), max_num_splits: NonZeroU32::new(1).unwrap(), num_concurrent_downloads: NonZeroU32::new(1).unwrap(), }, @@ -511,7 +511,7 @@ mod tests { fn test_split_table_prefer_start_download_prevent_new_report() { let mut split_table = SplitTable::with_limits_and_existing_splits( SplitCacheLimits { - max_num_bytes: Byte::from_bytes(1000), + max_num_bytes: ByteSize::kb(1), max_num_splits: NonZeroU32::new(1).unwrap(), num_concurrent_downloads: NonZeroU32::new(1).unwrap(), }, @@ -540,7 +540,7 @@ mod tests { fn test_eviction_due_to_size() { let mut split_table = SplitTable::with_limits_and_existing_splits( SplitCacheLimits { - max_num_bytes: Byte::from_bytes(1_000_000), + max_num_bytes: ByteSize::mb(1), max_num_splits: NonZeroU32::new(30).unwrap(), num_concurrent_downloads: NonZeroU32::new(1).unwrap(), }, @@ -577,7 +577,7 @@ mod tests { fn test_eviction_due_to_num_splits() { let mut split_table = SplitTable::with_limits_and_existing_splits( SplitCacheLimits { - max_num_bytes: Byte::from_bytes(10_000_000), + max_num_bytes: ByteSize::mb(10), max_num_splits: NonZeroU32::new(5).unwrap(), num_concurrent_downloads: NonZeroU32::new(1).unwrap(), }, @@ -611,7 +611,7 @@ mod tests { fn test_failed_download_can_be_re_reported() { let mut split_table = SplitTable::with_limits_and_existing_splits( SplitCacheLimits { - max_num_bytes: Byte::from_bytes(10_000_000), + max_num_bytes: ByteSize::mb(10), max_num_splits: NonZeroU32::new(5).unwrap(), num_concurrent_downloads: NonZeroU32::new(1).unwrap(), }, @@ -640,7 +640,7 @@ mod tests { fn test_split_table_truncate_candidates() { let mut split_table = SplitTable::with_limits_and_existing_splits( SplitCacheLimits { - max_num_bytes: Byte::from_bytes(10_000_000), + max_num_bytes: ByteSize::mb(10), max_num_splits: NonZeroU32::new(5).unwrap(), num_concurrent_downloads: NonZeroU32::new(1).unwrap(), }, diff --git a/quickwit/quickwit-storage/src/split_cache/tests.rs b/quickwit/quickwit-storage/src/split_cache/tests.rs index 9c85add52e8..8a63809f0c7 100644 --- a/quickwit/quickwit-storage/src/split_cache/tests.rs +++ b/quickwit/quickwit-storage/src/split_cache/tests.rs @@ -19,7 +19,7 @@ use std::num::NonZeroU32; -use byte_unit::Byte; +use bytesize::ByteSize; use quickwit_common::uri::Uri; use quickwit_config::SplitCacheLimits; use ulid::Ulid; @@ -31,7 +31,7 @@ const TEST_STORAGE_URI: &'static str = "s3://test"; #[test] fn test_split_table() { let mut split_table = SplitTable::with_limits(SplitCacheLimits { - max_num_bytes: Byte::from_bytes(1000), + max_num_bytes: ByteSize::kb(1), max_num_splits: NonZeroU32::new(1).unwrap(), num_concurrent_downloads: NonZeroU32::new(1).unwrap(), }); @@ -46,7 +46,7 @@ fn test_split_table() { #[test] fn test_split_table_prefer_last_touched() { let mut split_table = SplitTable::with_limits(SplitCacheLimits { - max_num_bytes: Byte::from_bytes(1000), + max_num_bytes: ByteSize::kb(1), max_num_splits: NonZeroU32::new(1).unwrap(), num_concurrent_downloads: NonZeroU32::new(1).unwrap(), }); @@ -63,7 +63,7 @@ fn test_split_table_prefer_last_touched() { #[test] fn test_split_table_prefer_start_download_prevent_new_report() { let mut split_table = SplitTable::with_limits(SplitCacheLimits { - max_num_bytes: Byte::from_bytes(1000), + max_num_bytes: ByteSize::kb(1), max_num_splits: NonZeroU32::new(1).unwrap(), num_concurrent_downloads: NonZeroU32::new(1).unwrap(), }); @@ -89,7 +89,7 @@ fn test_split_table_prefer_start_download_prevent_new_report() { #[test] fn test_eviction_due_to_size() { let mut split_table = SplitTable::with_limits(SplitCacheLimits { - max_num_bytes: Byte::from_bytes(1_000_000), + max_num_bytes: ByteSize::mb(1), max_num_splits: NonZeroU32::new(30).unwrap(), num_concurrent_downloads: NonZeroU32::new(1).unwrap(), }); @@ -123,7 +123,7 @@ fn test_eviction_due_to_size() { #[test] fn test_eviction_due_to_num_splits() { let mut split_table = SplitTable::with_limits(SplitCacheLimits { - max_num_bytes: Byte::from_bytes(10_000_000), + max_num_bytes: ByteSize::mb(10), max_num_splits: NonZeroU32::new(5).unwrap(), num_concurrent_downloads: NonZeroU32::new(1).unwrap(), }); @@ -154,7 +154,7 @@ fn test_eviction_due_to_num_splits() { #[test] fn test_failed_download_can_be_re_reported() { let mut split_table = SplitTable::with_limits(SplitCacheLimits { - max_num_bytes: Byte::from_bytes(10_000_000), + max_num_bytes: ByteSize::mb(10), max_num_splits: NonZeroU32::new(5).unwrap(), num_concurrent_downloads: NonZeroU32::new(1).unwrap(), }); From 8fa8f5686ccaee67c5d8c97a91b23806e12a609c Mon Sep 17 00:00:00 2001 From: Adrien Guillo Date: Wed, 8 Nov 2023 14:20:31 -0500 Subject: [PATCH 12/27] Fix typo --- quickwit/Cargo.lock | 230 +++++++++--------- quickwit/Cargo.toml | 2 +- quickwit/quickwit-cli/src/index.rs | 6 +- quickwit/quickwit-cli/src/main.rs | 2 +- .../quickwit-config/src/index_config/mod.rs | 2 +- .../quickwit-config/src/node_config/mod.rs | 2 +- .../file-backed-index/v0.4.expected.json | 2 +- .../test-data/file-backed-index/v0.4.json | 2 +- .../file-backed-index/v0.5.expected.json | 2 +- .../test-data/file-backed-index/v0.5.json | 2 +- .../file-backed-index/v0.6.expected.json | 4 +- .../test-data/file-backed-index/v0.6.json | 4 +- .../index-metadata/v0.4.expected.json | 2 +- .../test-data/index-metadata/v0.4.json | 2 +- .../index-metadata/v0.5.expected.json | 2 +- .../test-data/index-metadata/v0.5.json | 2 +- .../index-metadata/v0.6.expected.json | 4 +- .../test-data/index-metadata/v0.6.json | 4 +- quickwit/quickwit-search/src/tests.rs | 2 +- 19 files changed, 141 insertions(+), 137 deletions(-) diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 38a41cc7abd..d5b43fd830c 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -60,7 +60,7 @@ version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a824f2aa7e75a0c98c5a504fceb80649e9c35265d44525b5f94de4771a395cd" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.11", "once_cell", "version_check", ] @@ -72,7 +72,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" dependencies = [ "cfg-if", - "getrandom 0.2.10", + "getrandom 0.2.11", "once_cell", "version_check", "zerocopy", @@ -301,7 +301,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -312,7 +312,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -790,7 +790,7 @@ dependencies = [ "bytes", "dyn-clone", "futures", - "getrandom 0.2.10", + "getrandom 0.2.11", "http-types", "log", "paste", @@ -856,7 +856,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" dependencies = [ "futures-core", - "getrandom 0.2.10", + "getrandom 0.2.11", "instant", "pin-project-lite", "rand 0.8.5", @@ -1105,9 +1105,9 @@ dependencies = [ [[package]] name = "bytestring" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238e4886760d98c4f899360c834fa93e62cf7f721ac3c2da375cbdf4b8679aae" +checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72" dependencies = [ "bytes", ] @@ -1513,9 +1513,9 @@ dependencies = [ [[package]] name = "crc-catalog" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32c" @@ -1744,7 +1744,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -1766,7 +1766,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -1966,9 +1966,9 @@ dependencies = [ [[package]] name = "dyn-clone" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd" +checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d" [[package]] name = "either" @@ -2111,7 +2111,7 @@ checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -2144,9 +2144,9 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" +checksum = "7c18ee0ed65a5f1f81cac6b1d213b69c35fa47d4252ad41f1486dbd8226fe36e" dependencies = [ "libc", "windows-sys 0.48.0", @@ -2402,7 +2402,7 @@ checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -2478,9 +2478,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", "js-sys", @@ -3093,7 +3093,7 @@ checksum = "ce243b1bfa62ffc028f1cc3b6034ec63d649f3031bc8a4fbbb004e1ac17d1f68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -3120,9 +3120,9 @@ dependencies = [ [[package]] name = "inventory" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1be380c410bf0595e94992a648ea89db4dd3f3354ba54af206fd2a68cf5ac8e" +checksum = "0508c56cfe9bfd5dfeb0c22ab9a6abfda2f27bdca422132e494266351ed8d83c" [[package]] name = "io-lifetimes" @@ -3196,18 +3196,18 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" dependencies = [ "wasm-bindgen", ] [[package]] name = "json_comments" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ee439ee368ba4a77ac70d04f14015415af8600d6c894dc1f11bd79758c57d5" +checksum = "9dbbfed4e59ba9750e15ba154fdfd9329cee16ff3df539c2666b70f58cc32105" [[package]] name = "jsonwebtoken" @@ -3286,9 +3286,9 @@ checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" [[package]] name = "libc" -version = "0.2.149" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "libm" @@ -3296,6 +3296,17 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "libredox" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +dependencies = [ + "bitflags 2.4.1", + "libc", + "redox_syscall 0.4.1", +] + [[package]] name = "libsqlite3-sys" version = "0.26.0" @@ -3560,9 +3571,9 @@ checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "linux-raw-sys" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" [[package]] name = "lock_api" @@ -3826,7 +3837,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.11", ] [[package]] @@ -4003,7 +4014,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -4029,7 +4040,7 @@ checksum = "c38841cdd844847e3e7c8d29cef9dcfed8877f8f56f9071f77843ecf3baf937f" dependencies = [ "base64 0.13.1", "chrono", - "getrandom 0.2.10", + "getrandom 0.2.11", "http", "rand 0.8.5", "reqwest", @@ -4137,9 +4148,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.57" +version = "0.10.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +checksum = "7a257ad03cd8fb16ad4172fedf8094451e1af1c4b70097636ef2eac9a5f0cc33" dependencies = [ "bitflags 2.4.1", "cfg-if", @@ -4158,7 +4169,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -4178,9 +4189,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.93" +version = "0.9.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9" dependencies = [ "cc", "libc", @@ -4318,7 +4329,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -4482,7 +4493,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -4570,7 +4581,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -4744,9 +4755,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b559898e0b4931ed2d3b959ab0c2da4d99cc644c4b0b1a35b4d344027f474023" +checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" [[package]] name = "postcard" @@ -4849,7 +4860,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -5226,7 +5237,7 @@ dependencies = [ "prost-build", "quote", "serde", - "syn 2.0.38", + "syn 2.0.39", "tonic-build", ] @@ -5314,7 +5325,7 @@ dependencies = [ "serde", "serde_json", "serde_with 3.4.0", - "serde_yaml 0.9.25", + "serde_yaml 0.9.27", "tokio", "toml 0.7.8", "tracing", @@ -5421,7 +5432,7 @@ dependencies = [ "regex", "serde", "serde_json", - "serde_yaml 0.9.25", + "serde_yaml 0.9.27", "siphasher", "tantivy", "thiserror", @@ -5452,7 +5463,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "serde_yaml 0.9.25", + "serde_yaml 0.9.27", "tantivy", "tempfile", "thiserror", @@ -5673,7 +5684,7 @@ dependencies = [ "proc-macro2", "quickwit-macros-impl", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -5683,7 +5694,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -6090,7 +6101,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.11", ] [[package]] @@ -6161,9 +6172,9 @@ dependencies = [ [[package]] name = "rdkafka-sys" -version = "4.6.0+2.2.0" +version = "4.7.0+2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad63c279fca41a27c231c450a2d2ad18288032e9cbb159ad16c9d96eba35aaaf" +checksum = "55e0d2f9ba6253f6ec72385e453294f8618e9e15c2c6aba2a5c01ccf9622d615" dependencies = [ "cmake", "libc", @@ -6175,15 +6186,6 @@ dependencies = [ "zstd-sys", ] -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.3.5" @@ -6204,12 +6206,12 @@ dependencies = [ [[package]] name = "redox_users" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" dependencies = [ - "getrandom 0.2.10", - "redox_syscall 0.2.16", + "getrandom 0.2.11", + "libredox", "thiserror", ] @@ -6345,7 +6347,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" dependencies = [ "cc", - "getrandom 0.2.10", + "getrandom 0.2.11", "libc", "spin 0.9.8", "untrusted 0.9.0", @@ -6438,7 +6440,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.38", + "syn 2.0.39", "walkdir", ] @@ -6522,7 +6524,7 @@ dependencies = [ "bitflags 2.4.1", "errno", "libc", - "linux-raw-sys 0.4.10", + "linux-raw-sys 0.4.11", "windows-sys 0.48.0", ] @@ -6698,14 +6700,14 @@ dependencies = [ [[package]] name = "sea-query-derive" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd78f2e0ee8e537e9195d1049b752e0433e2cac125426bccb7b5c3e508096117" +checksum = "25a82fcb49253abcb45cdcb2adf92956060ec0928635eb21b4f7a6d8f25ab0bc" dependencies = [ "heck", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.39", "thiserror", ] @@ -6746,9 +6748,9 @@ checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" [[package]] name = "serde" -version = "1.0.190" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" +checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001" dependencies = [ "serde_derive", ] @@ -6765,13 +6767,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.190" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" +checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -6898,7 +6900,7 @@ dependencies = [ "darling 0.20.3", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -6915,9 +6917,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.25" +version = "0.9.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" +checksum = "3cc7a1570e38322cfe4154732e5110f887ea57e22b76f4bfd32b5bdd3368666c" dependencies = [ "indexmap 2.1.0", "itoa", @@ -7426,9 +7428,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.38" +version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ "proc-macro2", "quote", @@ -7706,7 +7708,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -7869,7 +7871,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -8150,7 +8152,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -8278,7 +8280,7 @@ checksum = "bfc13d450dc4a695200da3074dacf43d449b968baee95e341920e47f61a3b40f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -8472,7 +8474,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -8481,7 +8483,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.11", "rand 0.8.5", "serde", "wasm-bindgen", @@ -8694,9 +8696,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -8704,24 +8706,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.37" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02" dependencies = [ "cfg-if", "js-sys", @@ -8731,9 +8733,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -8741,22 +8743,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" [[package]] name = "wasm-streams" @@ -8773,9 +8775,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" dependencies = [ "js-sys", "wasm-bindgen", @@ -9033,9 +9035,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.5.17" +version = "0.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3b801d0e0a6726477cc207f60162da452f3a95adb368399bef20a946e06f65c" +checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b" dependencies = [ "memchr", ] @@ -9052,9 +9054,9 @@ dependencies = [ [[package]] name = "wiremock" -version = "0.5.19" +version = "0.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f71803d3a1c80377a06221e0530be02035d5b3e854af56c6ece7ac20ac441d" +checksum = "079aee011e8a8e625d16df9e785de30a6b77f80a6126092d76a57375f96448da" dependencies = [ "assert-json-diff 2.0.2", "async-trait", @@ -9129,22 +9131,22 @@ checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" [[package]] name = "zerocopy" -version = "0.7.16" +version = "0.7.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c552e97c5a9b90bc8ddc545b5106e798807376356688ebaa3aee36f44f8c4b9e" +checksum = "8cd369a67c0edfef15010f980c3cbe45d7f651deac2cd67ce097cd801de16557" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.16" +version = "0.7.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "964bc0588d7ac1c0243d0427ef08482618313702bbb014806cb7ab3da34d3d99" +checksum = "c2f140bda219a26ccc0cdb03dba58af72590c53b22642577d88a927bc5c87d6b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index 968c7e25f0a..d923ff1cde8 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -46,8 +46,8 @@ async-speed-limit = "0.4" async-trait = "0.1" backoff = { version = "0.4", features = ["tokio"] } base64 = "0.21" -bytesize = {version = "1.3.0", features = ["serde"]} bytes = { version = "1", features = ["serde"] } +bytesize = { version = "1.3.0", features = ["serde"] } bytestring = "1.3.0" chitchat = "0.6" chrono = { version = "0.4.23", default-features = false, features = ["clock", "std"] } diff --git a/quickwit/quickwit-cli/src/index.rs b/quickwit/quickwit-cli/src/index.rs index 73b15cf7b83..809622a302b 100644 --- a/quickwit/quickwit-cli/src/index.rs +++ b/quickwit/quickwit-cli/src/index.rs @@ -25,7 +25,7 @@ use std::str::FromStr; use std::time::{Duration, Instant}; use std::{fmt, io}; -use anyhow::{bail, Context}; +use anyhow::{anyhow, bail, Context}; use bytes::Bytes; use bytesize::ByteSize; use clap::{arg, Arg, ArgAction, ArgMatches, Command}; @@ -339,7 +339,9 @@ impl IndexCliCommand { let batch_size_limit_opt = matches .remove_one::("batch-size-limit") - .map(|s| s.parse::().unwrap()); + .map(|limit| limit.parse::()) + .transpose() + .map_err(|error| anyhow!(error))?; let commit_type = match (matches.get_flag("wait"), matches.get_flag("force")) { (false, false) => CommitType::Auto, (false, true) => CommitType::Force, diff --git a/quickwit/quickwit-cli/src/main.rs b/quickwit/quickwit-cli/src/main.rs index 7cd88f3b1b2..bbe3bd90f72 100644 --- a/quickwit/quickwit-cli/src/main.rs +++ b/quickwit/quickwit-cli/src/main.rs @@ -262,7 +262,7 @@ mod tests { && client_args.connect_timeout.is_none() && client_args.commit_timeout.is_none() && !client_args.ingest_v2 - && batch_size_limit == batch_size_limit == ByteSize::mb(8) + && batch_size_limit == ByteSize::mb(8) )); let app = build_cli().no_binary_name(true); diff --git a/quickwit/quickwit-config/src/index_config/mod.rs b/quickwit/quickwit-config/src/index_config/mod.rs index 11500068229..565d13f07b9 100644 --- a/quickwit/quickwit-config/src/index_config/mod.rs +++ b/quickwit/quickwit-config/src/index_config/mod.rs @@ -465,7 +465,7 @@ impl TestableForRegression for IndexConfig { }; let merge_policy = MergePolicyConfig::StableLog(stable_log_config); let indexing_resources = IndexingResources { - heap_size: ByteSize(3), + heap_size: ByteSize::mb(50), ..Default::default() }; let indexing_settings = IndexingSettings { diff --git a/quickwit/quickwit-config/src/node_config/mod.rs b/quickwit/quickwit-config/src/node_config/mod.rs index bdfbeba94f1..453e5f020b2 100644 --- a/quickwit/quickwit-config/src/node_config/mod.rs +++ b/quickwit/quickwit-config/src/node_config/mod.rs @@ -90,7 +90,7 @@ impl IndexerConfig { let indexer_config = IndexerConfig { enable_cooperative_indexing: false, enable_otlp_endpoint: true, - split_store_max_num_bytes: ByteSize::gb(1), + split_store_max_num_bytes: ByteSize::mb(1), split_store_max_num_splits: 3, max_concurrent_split_uploads: 4, }; diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.expected.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.expected.json index 169cca9abcf..91c552514d7 100644 --- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.expected.json +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.expected.json @@ -98,7 +98,7 @@ "type": "stable_log" }, "resources": { - "heap_size": 3 + "heap_size": "50.0 MB" }, "split_num_docs_target": 10000001 }, diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.json index de49a1ab545..a0c2f309a9f 100644 --- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.json +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.4.json @@ -86,7 +86,7 @@ "type": "stable_log" }, "resources": { - "heap_size": 3 + "heap_size": 50000000 }, "split_num_docs_target": 10000001 }, diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.expected.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.expected.json index b9abd44b004..239836b57ff 100644 --- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.expected.json +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.expected.json @@ -98,7 +98,7 @@ "type": "stable_log" }, "resources": { - "heap_size": 3 + "heap_size": "50.0 MB" }, "split_num_docs_target": 10000001 }, diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.json index 9977998242d..f4f7d26d1f5 100644 --- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.json +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.5.json @@ -86,7 +86,7 @@ "type": "stable_log" }, "resources": { - "heap_size": 3 + "heap_size": 50000000 }, "split_num_docs_target": 10000001 }, diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.expected.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.expected.json index 3a48e362219..c8a478a3c3e 100644 --- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.expected.json +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.expected.json @@ -40,6 +40,7 @@ }, { "fast": true, + "fast_precision": "seconds", "indexed": true, "input_formats": [ "rfc3339", @@ -47,7 +48,6 @@ ], "name": "timestamp", "output_format": "rfc3339", - "fast_precision": "seconds", "stored": true, "type": "datetime" }, @@ -105,7 +105,7 @@ "type": "stable_log" }, "resources": { - "heap_size": 3 + "heap_size": "50.0 MB" }, "split_num_docs_target": 10000001 }, diff --git a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.json b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.json index 3a48e362219..c8a478a3c3e 100644 --- a/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.json +++ b/quickwit/quickwit-metastore/test-data/file-backed-index/v0.6.json @@ -40,6 +40,7 @@ }, { "fast": true, + "fast_precision": "seconds", "indexed": true, "input_formats": [ "rfc3339", @@ -47,7 +48,6 @@ ], "name": "timestamp", "output_format": "rfc3339", - "fast_precision": "seconds", "stored": true, "type": "datetime" }, @@ -105,7 +105,7 @@ "type": "stable_log" }, "resources": { - "heap_size": 3 + "heap_size": "50.0 MB" }, "split_num_docs_target": 10000001 }, diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.expected.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.expected.json index b2edb60f832..1962205d42d 100644 --- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.expected.json +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.expected.json @@ -87,7 +87,7 @@ "type": "stable_log" }, "resources": { - "heap_size": 3 + "heap_size": "50.0 MB" }, "split_num_docs_target": 10000001 }, diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.json index b3c3cdf99e6..a6316834104 100644 --- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.json +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.4.json @@ -72,7 +72,7 @@ "type": "stable_log" }, "resources": { - "heap_size": 3 + "heap_size": 50000000 }, "split_num_docs_target": 10000001 }, diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.expected.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.expected.json index b2edb60f832..1962205d42d 100644 --- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.expected.json +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.expected.json @@ -87,7 +87,7 @@ "type": "stable_log" }, "resources": { - "heap_size": 3 + "heap_size": "50.0 MB" }, "split_num_docs_target": 10000001 }, diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.json index 87fc824fc15..66233c904d7 100644 --- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.json +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.5.json @@ -72,7 +72,7 @@ "type": "stable_log" }, "resources": { - "heap_size": 3 + "heap_size": 50000000 }, "split_num_docs_target": 10000001 }, diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.expected.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.expected.json index d38a9e51e02..f6522a1ba38 100644 --- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.expected.json +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.expected.json @@ -29,6 +29,7 @@ }, { "fast": true, + "fast_precision": "seconds", "indexed": true, "input_formats": [ "rfc3339", @@ -36,7 +37,6 @@ ], "name": "timestamp", "output_format": "rfc3339", - "fast_precision": "seconds", "stored": true, "type": "datetime" }, @@ -94,7 +94,7 @@ "type": "stable_log" }, "resources": { - "heap_size": 3 + "heap_size": "50.0 MB" }, "split_num_docs_target": 10000001 }, diff --git a/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.json b/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.json index d38a9e51e02..f6522a1ba38 100644 --- a/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.json +++ b/quickwit/quickwit-metastore/test-data/index-metadata/v0.6.json @@ -29,6 +29,7 @@ }, { "fast": true, + "fast_precision": "seconds", "indexed": true, "input_formats": [ "rfc3339", @@ -36,7 +37,6 @@ ], "name": "timestamp", "output_format": "rfc3339", - "fast_precision": "seconds", "stored": true, "type": "datetime" }, @@ -94,7 +94,7 @@ "type": "stable_log" }, "resources": { - "heap_size": 3 + "heap_size": "50.0 MB" }, "split_num_docs_target": 10000001 }, diff --git a/quickwit/quickwit-search/src/tests.rs b/quickwit/quickwit-search/src/tests.rs index aa67848511a..6f84651a62a 100644 --- a/quickwit/quickwit-search/src/tests.rs +++ b/quickwit/quickwit-search/src/tests.rs @@ -476,7 +476,7 @@ async fn test_single_node_without_timestamp_with_query_start_timestamp_enabled( let start_timestamp = OffsetDateTime::now_utc().unix_timestamp(); for i in 0..30 { let body = format!("info @ t:{}", i + 1); - docs.push(json!({ "body": body })); + docs.push(json!({"body": body})); } test_sandbox.add_documents(docs).await?; From 8baaa23c0ac520605e80cc5775450b69be06d311 Mon Sep 17 00:00:00 2001 From: Adrien Guillo Date: Wed, 8 Nov 2023 19:18:47 -0500 Subject: [PATCH 13/27] Close fetch stream with an error if it does not reach EOF (#4092) --- .../quickwit-ingest/src/ingest_v2/fetch.rs | 70 +++++++++++++++---- 1 file changed, 56 insertions(+), 14 deletions(-) diff --git a/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs b/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs index 919a12498c6..ff8f48716d7 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs @@ -136,13 +136,7 @@ impl FetchTask { .mrecordlog .range(&self.queue_id, self.fetch_range) else { - warn!( - client_id=%self.client_id, - index_uid=%self.index_uid, - source_id=%self.source_id, - shard_id=%self.shard_id, - "failed to read from record log because it was dropped." - ); + // The queue was dropped. break; }; for (_position, mrecord) in mrecords { @@ -200,13 +194,21 @@ impl FetchTask { break; } } - debug!( - client_id=%self.client_id, - index_uid=%self.index_uid, - source_id=%self.source_id, - shard_id=%self.shard_id, - "fetch task completed" - ); + if !has_reached_eof || !self.fetch_range.is_empty() { + error!( + client_id=%self.client_id, + index_uid=%self.index_uid, + source_id=%self.source_id, + shard_id=%self.shard_id, + "fetch stream ended unexpectedly" + ); + let _ = self + .fetch_response_tx + .send(Err(IngestV2Error::Internal( + "fetch stream ended unexpectedly".to_string(), + ))) + .await; + } ( num_records_total, self.fetch_range.from_position_exclusive(), @@ -776,6 +778,46 @@ mod tests { assert_eq!(last_position, Position::from(4u64)); } + #[tokio::test] + async fn test_fetch_task_error() { + let tempdir = tempfile::tempdir().unwrap(); + let mrecordlog = MultiRecordLog::open(tempdir.path()).await.unwrap(); + let client_id = "test-client".to_string(); + let index_uid = "test-index:0".to_string(); + let source_id = "test-source".to_string(); + let open_fetch_stream_request = OpenFetchStreamRequest { + client_id: client_id.clone(), + index_uid: index_uid.clone(), + source_id: source_id.clone(), + shard_id: 1, + from_position_exclusive: None, + to_position_inclusive: None, + }; + let (_new_records_tx, new_records_rx) = watch::channel(()); + let state = Arc::new(RwLock::new(IngesterState { + mrecordlog, + shards: HashMap::new(), + replication_streams: HashMap::new(), + replication_tasks: HashMap::new(), + })); + let (mut fetch_stream, fetch_task_handle) = FetchTask::spawn( + open_fetch_stream_request, + state.clone(), + new_records_rx, + 1024, + ); + let ingest_error = timeout(Duration::from_millis(50), fetch_stream.next()) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert!(matches!(ingest_error, IngestV2Error::Internal(_))); + + let (num_records, last_position) = fetch_task_handle.await.unwrap(); + assert_eq!(num_records, 0); + assert_eq!(last_position, Position::Beginning); + } + #[tokio::test] async fn test_fetch_task_up_to_position() { let tempdir = tempfile::tempdir().unwrap(); From 2613aeef73f3f87975a4ffd26504a6a72ccdc25a Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Thu, 9 Nov 2023 12:48:53 +0900 Subject: [PATCH 14/27] Exposing the node capacity via chitchat (#4083) * Exposing the node capacity via chitchat * Refactoring introducing the Cpu capacity. This new CpuCapacity type is used to both expressed the available capacity in indexer nodes, and the load associated on shards. As a result, the quantity we use for the latter is semantically shifted from the CPU spent on the indexer actor, to the estimated CPU cost of an entire pipeline. (x4) --- quickwit/Cargo.lock | 5 +- quickwit/Cargo.toml | 2 +- quickwit/quickwit-cli/src/tool.rs | 19 +- quickwit/quickwit-cluster/src/cluster.rs | 20 +- quickwit/quickwit-cluster/src/lib.rs | 27 ++- quickwit/quickwit-cluster/src/member.rs | 53 +++--- quickwit/quickwit-cluster/src/node.rs | 8 +- quickwit/quickwit-config/Cargo.toml | 1 + .../quickwit-config/src/index_config/mod.rs | 2 +- .../quickwit-config/src/node_config/mod.rs | 61 ++++++ .../src/node_config/serialize.rs | 1 + .../src/indexing_scheduler/mod.rs | 32 ++-- .../src/indexing_scheduler/scheduling/mod.rs | 96 +++++----- .../scheduling/scheduling_logic.rs | 127 ++++++++----- .../scheduling/scheduling_logic_model.rs | 46 +++-- quickwit/quickwit-control-plane/src/lib.rs | 3 +- quickwit/quickwit-control-plane/src/tests.rs | 4 +- .../quickwit-indexing/src/actors/indexer.rs | 12 +- quickwit/quickwit-indexing/src/lib.rs | 3 +- quickwit/quickwit-proto/src/indexing/mod.rs | 179 +++++++++++++++++- quickwit/quickwit-serve/src/lib.rs | 4 +- 21 files changed, 501 insertions(+), 204 deletions(-) diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index d5b43fd830c..4578b2506aa 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -1215,9 +1215,9 @@ dependencies = [ [[package]] name = "chitchat" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e5c649e5309f040f18aa0d660e63fe913d3d642e4771dd3cb66515ad75d239d" +checksum = "cb13c7104e3c77027520e984339ab3e805fd19c05c8555cc6338b2a4f455cbfb" dependencies = [ "anyhow", "async-trait", @@ -5316,6 +5316,7 @@ dependencies = [ "itertools 0.11.0", "json_comments", "new_string_template", + "num_cpus", "once_cell", "quickwit-common", "quickwit-doc-mapper", diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index d923ff1cde8..43c4e51f1f0 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -49,7 +49,7 @@ base64 = "0.21" bytes = { version = "1", features = ["serde"] } bytesize = { version = "1.3.0", features = ["serde"] } bytestring = "1.3.0" -chitchat = "0.6" +chitchat = "0.7" chrono = { version = "0.4.23", default-features = false, features = ["clock", "std"] } clap = { version = "4.4.1", features = ["env", "string"] } colored = "2.0.0" diff --git a/quickwit/quickwit-cli/src/tool.rs b/quickwit/quickwit-cli/src/tool.rs index 82a040a5522..ecacae7167b 100644 --- a/quickwit/quickwit-cli/src/tool.rs +++ b/quickwit/quickwit-cli/src/tool.rs @@ -49,6 +49,7 @@ use quickwit_indexing::models::{ use quickwit_indexing::IndexingPipeline; use quickwit_ingest::IngesterPool; use quickwit_metastore::IndexMetadataResponseExt; +use quickwit_proto::indexing::CpuCapacity; use quickwit_proto::metastore::{IndexMetadataRequest, MetastoreService, MetastoreServiceClient}; use quickwit_proto::search::{CountHits, SearchResponse}; use quickwit_proto::types::NodeId; @@ -931,15 +932,16 @@ impl ThroughputCalculator { async fn create_empty_cluster(config: &NodeConfig) -> anyhow::Result { let node_id: NodeId = config.node_id.clone().into(); - let self_node = ClusterMember::new( + let self_node = ClusterMember { node_id, - quickwit_cluster::GenerationId::now(), - false, - HashSet::new(), - config.gossip_advertise_addr, - config.grpc_advertise_addr, - Vec::new(), - ); + generation_id: quickwit_cluster::GenerationId::now(), + is_ready: false, + enabled_services: HashSet::new(), + gossip_advertise_addr: config.gossip_advertise_addr, + grpc_advertise_addr: config.grpc_advertise_addr, + indexing_cpu_capacity: CpuCapacity::zero(), + indexing_tasks: Vec::new(), + }; let cluster = Cluster::join( config.cluster_id.clone(), self_node, @@ -949,5 +951,6 @@ async fn create_empty_cluster(config: &NodeConfig) -> anyhow::Result { &ChannelTransport::default(), ) .await?; + Ok(cluster) } diff --git a/quickwit/quickwit-cluster/src/cluster.rs b/quickwit/quickwit-cluster/src/cluster.rs index 53a6c0dbbd5..0c9f95533cf 100644 --- a/quickwit/quickwit-cluster/src/cluster.rs +++ b/quickwit/quickwit-cluster/src/cluster.rs @@ -18,7 +18,7 @@ // along with this program. If not, see . use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fmt::Debug; +use std::fmt::{Debug, Display}; use std::net::SocketAddr; use std::sync::Arc; use std::time::Duration; @@ -229,7 +229,7 @@ impl Cluster { } /// Sets a key-value pair on the cluster node's state. - pub async fn set_self_key_value, V: Into>(&self, key: K, value: V) { + pub async fn set_self_key_value(&self, key: impl Display, value: impl Display) { self.chitchat() .await .lock() @@ -517,17 +517,19 @@ pub async fn create_cluster_for_test_with_id( transport: &dyn Transport, self_node_readiness: bool, ) -> anyhow::Result { + use quickwit_proto::indexing::PIPELINE_FULL_CAPACITY; let gossip_advertise_addr: SocketAddr = ([127, 0, 0, 1], node_id).into(); let node_id: NodeId = format!("node_{node_id}").into(); - let self_node = ClusterMember::new( + let self_node = ClusterMember { node_id, - crate::GenerationId(1), - self_node_readiness, - enabled_services.clone(), + generation_id: crate::GenerationId(1), + is_ready: self_node_readiness, + enabled_services: enabled_services.clone(), gossip_advertise_addr, - grpc_addr_from_listen_addr_for_test(gossip_advertise_addr), - Vec::new(), - ); + grpc_advertise_addr: grpc_addr_from_listen_addr_for_test(gossip_advertise_addr), + indexing_tasks: Vec::new(), + indexing_cpu_capacity: PIPELINE_FULL_CAPACITY, + }; let failure_detector_config = create_failure_detector_config_for_test(); let cluster = Cluster::join( cluster_id, diff --git a/quickwit/quickwit-cluster/src/lib.rs b/quickwit/quickwit-cluster/src/lib.rs index e1e4afc180c..5821d41c65d 100644 --- a/quickwit/quickwit-cluster/src/lib.rs +++ b/quickwit/quickwit-cluster/src/lib.rs @@ -30,6 +30,7 @@ use chitchat::transport::UdpTransport; use chitchat::FailureDetectorConfig; use quickwit_config::service::QuickwitService; use quickwit_config::NodeConfig; +use quickwit_proto::indexing::CpuCapacity; use quickwit_proto::types::NodeId; use time::OffsetDateTime; @@ -37,7 +38,7 @@ pub use crate::change::ClusterChange; #[cfg(any(test, feature = "testsuite"))] pub use crate::cluster::{create_cluster_for_test, grpc_addr_from_listen_addr_for_test}; pub use crate::cluster::{Cluster, ClusterSnapshot, NodeIdSchema}; -pub use crate::member::ClusterMember; +pub use crate::member::{ClusterMember, INDEXING_CPU_CAPACITY_KEY}; pub use crate::node::ClusterNode; #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -68,15 +69,21 @@ pub async fn start_cluster_service(node_config: &NodeConfig) -> anyhow::Result anyhow::Result anyhow::Result; @@ -91,30 +94,12 @@ pub struct ClusterMember { /// None if the node is not an indexer or the indexer has not yet started some indexing /// pipelines. pub indexing_tasks: Vec, + /// Indexing cpu capacity of the node expressed in milli cpu. + pub indexing_cpu_capacity: CpuCapacity, pub is_ready: bool, } impl ClusterMember { - pub fn new( - node_id: NodeId, - generation_id: GenerationId, - is_ready: bool, - enabled_services: HashSet, - gossip_advertise_addr: SocketAddr, - grpc_advertise_addr: SocketAddr, - indexing_tasks: Vec, - ) -> Self { - Self { - node_id, - generation_id, - is_ready, - enabled_services, - gossip_advertise_addr, - grpc_advertise_addr, - indexing_tasks, - } - } - pub fn chitchat_id(&self) -> ChitchatId { ChitchatId::new( self.node_id.clone().into(), @@ -130,6 +115,18 @@ impl From for ChitchatId { } } +fn parse_indexing_cpu_capacity(node_state: &NodeState) -> CpuCapacity { + let Some(indexing_capacity_str) = node_state.get(INDEXING_CPU_CAPACITY_KEY) else { + return CpuCapacity::zero(); + }; + if let Ok(indexing_capacity) = CpuCapacity::from_str(indexing_capacity_str) { + indexing_capacity + } else { + error!(indexing_capacity=?indexing_capacity_str, "Received an unparseable indexing capacity from node."); + CpuCapacity::zero() + } +} + // Builds a cluster member from a [`NodeState`]. pub(crate) fn build_cluster_member( chitchat_id: ChitchatId, @@ -150,15 +147,17 @@ pub(crate) fn build_cluster_member( })?; let grpc_advertise_addr = node_state.grpc_advertise_addr()?; let indexing_tasks = parse_indexing_tasks(node_state, &chitchat_id.node_id); - let member = ClusterMember::new( - chitchat_id.node_id.into(), - chitchat_id.generation_id.into(), + let indexing_cpu_capacity = parse_indexing_cpu_capacity(node_state); + let member = ClusterMember { + node_id: chitchat_id.node_id.into(), + generation_id: chitchat_id.generation_id.into(), is_ready, enabled_services, - chitchat_id.gossip_advertise_addr, + gossip_advertise_addr: chitchat_id.gossip_advertise_addr, grpc_advertise_addr, indexing_tasks, - ); + indexing_cpu_capacity, + }; Ok(member) } diff --git a/quickwit/quickwit-cluster/src/node.rs b/quickwit/quickwit-cluster/src/node.rs index 1091ade7068..0b7d39dc7ce 100644 --- a/quickwit/quickwit-cluster/src/node.rs +++ b/quickwit/quickwit-cluster/src/node.rs @@ -24,7 +24,7 @@ use std::sync::Arc; use chitchat::{ChitchatId, NodeState}; use quickwit_config::service::QuickwitService; -use quickwit_proto::indexing::IndexingTask; +use quickwit_proto::indexing::{CpuCapacity, IndexingTask}; use tonic::transport::Channel; use crate::member::build_cluster_member; @@ -49,6 +49,7 @@ impl ClusterNode { enabled_services: member.enabled_services, grpc_advertise_addr: member.grpc_advertise_addr, indexing_tasks: member.indexing_tasks, + indexing_capacity: member.indexing_cpu_capacity, is_ready: member.is_ready, is_self_node, }; @@ -112,6 +113,10 @@ impl ClusterNode { &self.inner.indexing_tasks } + pub fn indexing_capacity(&self) -> CpuCapacity { + self.inner.indexing_capacity + } + pub fn is_ready(&self) -> bool { self.inner.is_ready } @@ -149,6 +154,7 @@ struct InnerNode { enabled_services: HashSet, grpc_advertise_addr: SocketAddr, indexing_tasks: Vec, + indexing_capacity: CpuCapacity, is_ready: bool, is_self_node: bool, } diff --git a/quickwit/quickwit-config/Cargo.toml b/quickwit/quickwit-config/Cargo.toml index 5abe67a5009..0606f942609 100644 --- a/quickwit/quickwit-config/Cargo.toml +++ b/quickwit/quickwit-config/Cargo.toml @@ -20,6 +20,7 @@ humantime = { workspace = true } itertools = { workspace = true } json_comments = { workspace = true } new_string_template = { workspace = true } +num_cpus = { workspace = true } once_cell = { workspace = true } regex = { workspace = true } serde = { workspace = true } diff --git a/quickwit/quickwit-config/src/index_config/mod.rs b/quickwit/quickwit-config/src/index_config/mod.rs index 565d13f07b9..7f12b1b6a51 100644 --- a/quickwit/quickwit-config/src/index_config/mod.rs +++ b/quickwit/quickwit-config/src/index_config/mod.rs @@ -637,7 +637,7 @@ mod tests { } #[test] - fn test_index_config_default_values() { + fn test_indexer_config_default_values() { let default_index_root_uri = Uri::for_test("s3://defaultbucket/"); { let index_config_filepath = get_index_config_filepath("minimal-hdfs-logs.yaml"); diff --git a/quickwit/quickwit-config/src/node_config/mod.rs b/quickwit/quickwit-config/src/node_config/mod.rs index 453e5f020b2..53bc51d908b 100644 --- a/quickwit/quickwit-config/src/node_config/mod.rs +++ b/quickwit/quickwit-config/src/node_config/mod.rs @@ -30,6 +30,7 @@ use anyhow::{bail, ensure}; use bytesize::ByteSize; use quickwit_common::net::HostAddr; use quickwit_common::uri::Uri; +use quickwit_proto::indexing::CpuCapacity; use serde::{Deserialize, Serialize}; use tracing::warn; @@ -55,6 +56,8 @@ pub struct IndexerConfig { pub enable_otlp_endpoint: bool, #[serde(default = "IndexerConfig::default_enable_cooperative_indexing")] pub enable_cooperative_indexing: bool, + #[serde(default = "IndexerConfig::default_cpu_capacity")] + pub cpu_capacity: CpuCapacity, } impl IndexerConfig { @@ -85,14 +88,20 @@ impl IndexerConfig { 1_000 } + fn default_cpu_capacity() -> CpuCapacity { + CpuCapacity::one_cpu_thread() * (num_cpus::get() as u32) + } + #[cfg(any(test, feature = "testsuite"))] pub fn for_test() -> anyhow::Result { + use quickwit_proto::indexing::PIPELINE_FULL_CAPACITY; let indexer_config = IndexerConfig { enable_cooperative_indexing: false, enable_otlp_endpoint: true, split_store_max_num_bytes: ByteSize::mb(1), split_store_max_num_splits: 3, max_concurrent_split_uploads: 4, + cpu_capacity: PIPELINE_FULL_CAPACITY * 4u32, }; Ok(indexer_config) } @@ -106,6 +115,7 @@ impl Default for IndexerConfig { split_store_max_num_bytes: Self::default_split_store_max_num_bytes(), split_store_max_num_splits: Self::default_split_store_max_num_splits(), max_concurrent_split_uploads: Self::default_max_concurrent_split_uploads(), + cpu_capacity: Self::default_cpu_capacity(), } } } @@ -368,3 +378,54 @@ impl NodeConfig { serialize::node_config_for_test() } } + +#[cfg(test)] +mod tests { + use quickwit_proto::indexing::CpuCapacity; + + use crate::IndexerConfig; + + #[test] + fn test_index_config_serialization() { + { + let indexer_config: IndexerConfig = serde_json::from_str(r#"{}"#).unwrap(); + assert_eq!(&indexer_config, &IndexerConfig::default()); + assert!(indexer_config.cpu_capacity.cpu_millis() > 0); + assert_eq!(indexer_config.cpu_capacity.cpu_millis() % 1_000, 0); + } + { + let indexer_config: IndexerConfig = + serde_yaml::from_str(r#"cpu_capacity: 1.5"#).unwrap(); + assert_eq!( + indexer_config.cpu_capacity, + CpuCapacity::from_cpu_millis(1500) + ); + let indexer_config_json = serde_json::to_value(&indexer_config).unwrap(); + assert_eq!( + indexer_config_json + .get("cpu_capacity") + .unwrap() + .as_str() + .unwrap(), + "1500m" + ); + } + { + let indexer_config: IndexerConfig = + serde_yaml::from_str(r#"cpu_capacity: 1500m"#).unwrap(); + assert_eq!( + indexer_config.cpu_capacity, + CpuCapacity::from_cpu_millis(1500) + ); + let indexer_config_json = serde_json::to_value(&indexer_config).unwrap(); + assert_eq!( + indexer_config_json + .get("cpu_capacity") + .unwrap() + .as_str() + .unwrap(), + "1500m" + ); + } + } +} diff --git a/quickwit/quickwit-config/src/node_config/serialize.rs b/quickwit/quickwit-config/src/node_config/serialize.rs index 4ae055c66b8..9a99d0fb60e 100644 --- a/quickwit/quickwit-config/src/node_config/serialize.rs +++ b/quickwit/quickwit-config/src/node_config/serialize.rs @@ -484,6 +484,7 @@ mod tests { split_store_max_num_bytes: ByteSize::tb(1), split_store_max_num_splits: 10_000, max_concurrent_split_uploads: 8, + cpu_capacity: IndexerConfig::default_cpu_capacity(), enable_cooperative_indexing: false, } ); diff --git a/quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs b/quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs index 01fc6a42f99..4176535111e 100644 --- a/quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs +++ b/quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs @@ -26,7 +26,9 @@ use std::time::{Duration, Instant}; use fnv::{FnvHashMap, FnvHashSet}; use itertools::Itertools; -use quickwit_proto::indexing::{ApplyIndexingPlanRequest, IndexingService, IndexingTask}; +use quickwit_proto::indexing::{ + ApplyIndexingPlanRequest, CpuCapacity, IndexingService, IndexingTask, PIPELINE_FULL_CAPACITY, +}; use quickwit_proto::metastore::SourceType; use quickwit_proto::types::NodeId; use scheduling::{SourceToSchedule, SourceToScheduleType}; @@ -35,12 +37,9 @@ use tracing::{debug, error, info, warn}; use crate::control_plane_model::ControlPlaneModel; use crate::indexing_plan::PhysicalIndexingPlan; -use crate::indexing_scheduler::scheduling::{build_physical_indexing_plan, Load}; +use crate::indexing_scheduler::scheduling::build_physical_indexing_plan; use crate::{IndexerNodeInfo, IndexerPool}; -const PIPELINE_FULL_LOAD: Load = 1_000u32; -const LOAD_PER_NODE: Load = 2_000u32; - pub(crate) const MIN_DURATION_BETWEEN_SCHEDULING: Duration = if cfg!(any(test, feature = "testsuite")) { Duration::from_millis(50) @@ -158,7 +157,8 @@ fn get_sources_to_schedule(model: &ControlPlaneModel) -> Vec { source_type: SourceToScheduleType::NonSharded { num_pipelines: source_config.desired_num_pipelines.get() as u32, // FIXME - load_per_pipeline: NonZeroU32::new(PIPELINE_FULL_LOAD).unwrap(), + load_per_pipeline: NonZeroU32::new(PIPELINE_FULL_CAPACITY.cpu_millis()) + .unwrap(), }, }); } @@ -185,7 +185,7 @@ impl IndexingScheduler { // has happened. pub(crate) fn schedule_indexing_plan_if_needed(&mut self, model: &ControlPlaneModel) { crate::metrics::CONTROL_PLANE_METRICS.schedule_total.inc(); - let mut indexers = self.get_indexers_from_indexer_pool(); + let mut indexers: Vec<(String, IndexerNodeInfo)> = self.get_indexers_from_indexer_pool(); if indexers.is_empty() { warn!("No indexer available, cannot schedule an indexing plan."); return; @@ -193,17 +193,16 @@ impl IndexingScheduler { let sources = get_sources_to_schedule(model); - let indexer_max_loads: FnvHashMap = indexers + let indexer_id_to_cpu_capacities: FnvHashMap = indexers .iter() - .map(|(indexer_id, _)| { - // TODO Get info from chitchat. - (indexer_id.to_string(), LOAD_PER_NODE) + .map(|(indexer_id, indexer_node_info)| { + (indexer_id.to_string(), indexer_node_info.indexing_capacity) }) .collect(); let new_physical_plan = build_physical_indexing_plan( &sources, - &indexer_max_loads, + &indexer_id_to_cpu_capacities, self.state.last_applied_physical_plan.as_ref(), ); if let Some(last_applied_plan) = &self.state.last_applied_physical_plan { @@ -741,8 +740,8 @@ mod tests { }, ]; let mut indexer_max_loads = FnvHashMap::default(); - indexer_max_loads.insert("indexer1".to_string(), 3_000); - indexer_max_loads.insert("indexer2".to_string(), 3_000); + indexer_max_loads.insert("indexer1".to_string(), mcpu(3_000)); + indexer_max_loads.insert("indexer2".to_string(), mcpu(3_000)); let physical_plan = build_physical_indexing_plan(&sources[..], &indexer_max_loads, None); assert_eq!(physical_plan.indexing_tasks_per_indexer().len(), 2); let indexing_tasks_1 = physical_plan.indexer("indexer1").unwrap(); @@ -771,7 +770,7 @@ mod tests { let mut indexer_max_loads = FnvHashMap::default(); for i in 0..num_indexers { let indexer_id = format!("indexer-{i}"); - indexer_max_loads.insert(indexer_id, 4_000); + indexer_max_loads.insert(indexer_id, mcpu(4_000)); } let physical_indexing_plan = build_physical_indexing_plan(&sources, &indexer_max_loads, None); let source_map: FnvHashMap<&SourceUid, &SourceToSchedule> = sources @@ -802,12 +801,13 @@ mod tests { } } } - assert!(load_in_node <= *indexer_max_loads.get(node_id).unwrap()); + assert!(load_in_node <= indexer_max_loads.get(node_id).unwrap().cpu_millis()); } } } use quickwit_config::SourceInputFormat; + use quickwit_proto::indexing::mcpu; fn kafka_source_params_for_test() -> SourceParams { SourceParams::Kafka(KafkaSourceParams { diff --git a/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/mod.rs b/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/mod.rs index 7dce1cb841f..22cd4a6ebd6 100644 --- a/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/mod.rs +++ b/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/mod.rs @@ -24,9 +24,8 @@ use std::collections::hash_map::Entry; use std::num::NonZeroU32; use fnv::FnvHashMap; -use quickwit_proto::indexing::IndexingTask; +use quickwit_proto::indexing::{CpuCapacity, IndexingTask}; use quickwit_proto::types::{IndexUid, ShardId}; -pub use scheduling_logic_model::Load; use scheduling_logic_model::{IndexerOrd, SourceOrd}; use tracing::error; use tracing::log::warn; @@ -35,7 +34,6 @@ use crate::indexing_plan::PhysicalIndexingPlan; use crate::indexing_scheduler::scheduling::scheduling_logic_model::{ SchedulingProblem, SchedulingSolution, }; -use crate::indexing_scheduler::PIPELINE_FULL_LOAD; use crate::SourceUid; /// If we have several pipelines below this threshold we @@ -49,10 +47,10 @@ use crate::SourceUid; /// /// Coming back to a single pipeline requires having a load per pipeline /// of 30%. Which translates into an overall load of 60%. -const LOAD_PER_PIPELINE_LOW_THRESHOLD: u32 = PIPELINE_FULL_LOAD * 3 / 10; +const CPU_PER_PIPELINE_LOAD_THRESHOLD: CpuCapacity = CpuCapacity::from_cpu_millis(1_200); /// That's 80% of a period -const MAX_LOAD_PER_PIPELINE: u32 = PIPELINE_FULL_LOAD * 8 / 10; +const MAX_LOAD_PER_PIPELINE: CpuCapacity = CpuCapacity::from_cpu_millis(3_200); fn indexing_task(source_uid: SourceUid, shard_ids: Vec) -> IndexingTask { IndexingTask { @@ -260,35 +258,37 @@ fn group_shards_into_pipelines( source_uid: &SourceUid, shard_ids: &[ShardId], previous_indexing_tasks: &[IndexingTask], - load_per_shard: Load, + cpu_load_per_shard: CpuCapacity, ) -> Vec { let num_shards = shard_ids.len() as u32; if num_shards == 0 { return Vec::new(); } let max_num_shards_per_pipeline: NonZeroU32 = - NonZeroU32::new(MAX_LOAD_PER_PIPELINE / load_per_shard).unwrap_or_else(|| { - // We throttle shard at ingestion to ensure that a shard does not - // exceed 5MB/s. - // - // This value has been chosen to make sure that one full pipeline - // should always be able to handle the load of one shard. - // - // However it is possible for the system to take more than this - // when it is playing catch up. - // - // This is a transitory state, and not a problem per se. - warn!("load per shard is higher than `MAX_LOAD_PER_PIPELINE`"); - NonZeroU32::new(1).unwrap() - }); + NonZeroU32::new(MAX_LOAD_PER_PIPELINE.cpu_millis() / cpu_load_per_shard.cpu_millis()) + .unwrap_or_else(|| { + // We throttle shard at ingestion to ensure that a shard does not + // exceed 5MB/s. + // + // This value has been chosen to make sure that one full pipeline + // should always be able to handle the load of one shard. + // + // However it is possible for the system to take more than this + // when it is playing catch up. + // + // This is a transitory state, and not a problem per se. + warn!("load per shard is higher than `MAX_LOAD_PER_PIPELINE`"); + NonZeroU32::MIN // also colloquially known as `1` + }); // We compute the number of pipelines we will create, cooking in some hysteresis effect here. // We have two different threshold to increase and to decrease the number of pipelines. let min_num_pipelines: u32 = (num_shards + max_num_shards_per_pipeline.get() - 1) / max_num_shards_per_pipeline; assert!(min_num_pipelines > 0); - let max_num_pipelines: u32 = - min_num_pipelines.max(num_shards * load_per_shard / LOAD_PER_PIPELINE_LOW_THRESHOLD); + let max_num_pipelines: u32 = min_num_pipelines.max( + num_shards * cpu_load_per_shard.cpu_millis() / CPU_PER_PIPELINE_LOAD_THRESHOLD.cpu_millis(), + ); let previous_num_pipelines = previous_indexing_tasks.len() as u32; let num_pipelines: u32 = if previous_num_pipelines > min_num_pipelines { previous_num_pipelines.min(max_num_pipelines) @@ -409,7 +409,7 @@ fn convert_scheduling_solution_to_physical_plan( &source.source_uid, &shard_ids_for_node, indexing_tasks, - load_per_shard.get(), + CpuCapacity::from_cpu_millis(load_per_shard.get()), ); for indexing_task in indexing_tasks { physical_indexing_plan.add_indexing_task(node_id, indexing_task); @@ -471,7 +471,7 @@ fn convert_scheduling_solution_to_physical_plan( /// TODO cut into pipelines. pub fn build_physical_indexing_plan( sources: &[SourceToSchedule], - indexer_id_to_max_load: &FnvHashMap, + indexer_id_to_cpu_capacities: &FnvHashMap, previous_plan_opt: Option<&PhysicalIndexingPlan>, ) -> PhysicalIndexingPlan { // TODO make the load per node something that can be configured on each node. @@ -480,14 +480,15 @@ pub fn build_physical_indexing_plan( let mut id_to_ord_map = IdToOrdMap::default(); // We use a Vec as a `IndexOrd` -> Max load map. - let mut indexer_max_loads: Vec = Vec::with_capacity(indexer_id_to_max_load.len()); - for (indexer_id, &max_load) in indexer_id_to_max_load { + let mut indexer_cpu_capacities: Vec = + Vec::with_capacity(indexer_id_to_cpu_capacities.len()); + for (indexer_id, &cpu_capacity) in indexer_id_to_cpu_capacities { let indexer_ord = id_to_ord_map.add_indexer_id(indexer_id.clone()); - assert_eq!(indexer_ord, indexer_max_loads.len() as IndexerOrd); - indexer_max_loads.push(max_load); + assert_eq!(indexer_ord, indexer_cpu_capacities.len() as IndexerOrd); + indexer_cpu_capacities.push(cpu_capacity); } - let mut problem = SchedulingProblem::with_indexer_maximum_load(indexer_max_loads); + let mut problem = SchedulingProblem::with_indexer_cpu_capacities(indexer_cpu_capacities); for source in sources { if let Some(source_ord) = populate_problem(source, &mut problem) { let registered_source_ord = id_to_ord_map.add_source_uid(source.source_uid.clone()); @@ -528,7 +529,7 @@ mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; use fnv::FnvHashMap; - use quickwit_proto::indexing::IndexingTask; + use quickwit_proto::indexing::{mcpu, IndexingTask}; use quickwit_proto::types::{IndexUid, ShardId}; use super::{ @@ -573,25 +574,28 @@ mod tests { source_uid: source_uid0.clone(), source_type: SourceToScheduleType::Sharded { shards: vec![0, 1, 2, 3, 4, 5, 6, 7], - load_per_shard: NonZeroU32::new(250).unwrap(), + load_per_shard: NonZeroU32::new(1_000).unwrap(), }, }; let source_1 = SourceToSchedule { source_uid: source_uid1.clone(), source_type: SourceToScheduleType::NonSharded { num_pipelines: 2, - load_per_pipeline: NonZeroU32::new(800).unwrap(), + load_per_pipeline: NonZeroU32::new(3_200).unwrap(), }, }; let source_2 = SourceToSchedule { source_uid: source_uid2.clone(), source_type: SourceToScheduleType::IngestV1, }; - let mut indexer_max_load = FnvHashMap::default(); - indexer_max_load.insert(indexer1.clone(), 4_000); - indexer_max_load.insert(indexer2.clone(), 4_000); - let indexing_plan = - build_physical_indexing_plan(&[source_0, source_1, source_2], &indexer_max_load, None); + let mut indexer_id_to_cpu_capacities = FnvHashMap::default(); + indexer_id_to_cpu_capacities.insert(indexer1.clone(), mcpu(16_000)); + indexer_id_to_cpu_capacities.insert(indexer2.clone(), mcpu(16_000)); + let indexing_plan = build_physical_indexing_plan( + &[source_0, source_1, source_2], + &indexer_id_to_cpu_capacities, + None, + ); assert_eq!(indexing_plan.indexing_tasks_per_indexer().len(), 2); let node1_plan = indexing_plan.indexer(&indexer1).unwrap(); @@ -633,7 +637,7 @@ mod tests { let indexer1 = "indexer1".to_string(); let mut indexer_max_loads = FnvHashMap::default(); { - indexer_max_loads.insert(indexer1.clone(), 1_999); + indexer_max_loads.insert(indexer1.clone(), mcpu(1_999)); // This test what happens when there isn't enough capacity on the cluster. let physical_plan = build_physical_indexing_plan(&sources, &indexer_max_loads, None); assert_eq!(physical_plan.indexing_tasks_per_indexer().len(), 1); @@ -644,7 +648,7 @@ mod tests { ); } { - indexer_max_loads.insert(indexer1.clone(), 2_000); + indexer_max_loads.insert(indexer1.clone(), mcpu(2_000)); // This test what happens when there isn't enough capacity on the cluster. let physical_plan = build_physical_indexing_plan(&sources, &indexer_max_loads, None); assert_eq!(physical_plan.indexing_tasks_per_indexer().len(), 1); @@ -662,7 +666,7 @@ mod tests { #[test] fn test_group_shards_empty() { let source_uid = source_id(); - let indexing_tasks = group_shards_into_pipelines(&source_uid, &[], &[], 250); + let indexing_tasks = group_shards_into_pipelines(&source_uid, &[], &[], mcpu(250)); assert!(indexing_tasks.is_empty()); } @@ -690,7 +694,7 @@ mod tests { &source_uid, &[0, 1, 3, 4, 5], &previous_indexing_tasks, - 250, + mcpu(1_000), ); assert_eq!(indexing_tasks.len(), 2); assert_eq!(&indexing_tasks[0].shard_ids, &[0, 1]); @@ -700,7 +704,7 @@ mod tests { #[test] fn test_group_shards_load_per_shard_too_high() { let source_uid = source_id(); - let indexing_tasks = group_shards_into_pipelines(&source_uid, &[1, 2], &[], 1_000); + let indexing_tasks = group_shards_into_pipelines(&source_uid, &[1, 2], &[], mcpu(4_000)); assert_eq!(indexing_tasks.len(), 2); } @@ -712,7 +716,7 @@ mod tests { &source_uid, &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], &previous_indexing_tasks, - 100, + mcpu(400), ); assert_eq!(indexing_tasks_1.len(), 2); assert_eq!(&indexing_tasks_1[0].shard_ids, &[0, 2, 4, 6, 8, 10]); @@ -722,7 +726,7 @@ mod tests { &source_uid, &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], &indexing_tasks_1, - 150, + mcpu(600), ); assert_eq!(indexing_tasks_2.len(), 3); assert_eq!(&indexing_tasks_2[0].shard_ids, &[0, 2, 4, 6, 8]); @@ -734,7 +738,7 @@ mod tests { &source_uid, &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], &indexing_tasks_2, - 100, + mcpu(400), ); assert_eq!(indexing_tasks_3.len(), 3); assert_eq!(&indexing_tasks_3[0].shard_ids, &[0, 2, 4, 6, 8]); @@ -745,7 +749,7 @@ mod tests { &source_uid, &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], &indexing_tasks_3, - 80, + mcpu(320), ); assert_eq!(indexing_tasks_4.len(), 2); assert_eq!(&indexing_tasks_4[0].shard_ids, &[0, 2, 4, 6, 8, 10]); diff --git a/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/scheduling_logic.rs b/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/scheduling_logic.rs index b2f327fd336..82af447012a 100644 --- a/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/scheduling_logic.rs +++ b/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/scheduling_logic.rs @@ -21,6 +21,8 @@ use std::cmp::Reverse; use std::collections::btree_map::Entry; use std::collections::{BTreeMap, BinaryHeap, HashMap}; +use quickwit_proto::indexing::CpuCapacity; + use super::scheduling_logic_model::*; // ------------------------------------------------------------------------------------ @@ -43,7 +45,7 @@ pub fn solve( let mut solution = previous_solution; check_contract_conditions(problem, &solution); remove_extraneous_shards(problem, &mut solution); - enforce_nodes_max_load(problem, &mut solution); + enforce_indexers_cpu_capacity(problem, &mut solution); let still_unassigned = place_unassigned_shards(problem, &mut solution); // TODO ideally we should have some smarter logic here to bread first search for a better // solution. @@ -84,7 +86,7 @@ fn remove_extraneous_shards(problem: &SchedulingProblem, solution: &mut Scheduli } } - let mut indexer_available_capacity: Vec = solution + let mut indexer_available_capacity: Vec = solution .indexer_assignments .iter() .map(|indexer_assignment| indexer_assignment.indexer_available_capacity(problem)) @@ -136,48 +138,50 @@ fn assert_remove_extraneous_shards_post_condition( // Phase 2 // Releave sources from the node that are exceeding their maximum load. -fn enforce_nodes_max_load(problem: &SchedulingProblem, solution: &mut SchedulingSolution) { +fn enforce_indexers_cpu_capacity(problem: &SchedulingProblem, solution: &mut SchedulingSolution) { for indexer_assignment in solution.indexer_assignments.iter_mut() { - let node_max_load: Load = problem.indexer_max_load(indexer_assignment.indexer_ord); - enforce_node_max_load(problem, node_max_load, indexer_assignment); + let indexer_cpu_capacity: CpuCapacity = + problem.indexer_cpu_capacity(indexer_assignment.indexer_ord); + enforce_indexer_cpu_capacity(problem, indexer_cpu_capacity, indexer_assignment); } } -fn enforce_node_max_load( +fn enforce_indexer_cpu_capacity( problem: &SchedulingProblem, - node_max_load: Load, + indexer_cpu_capacity: CpuCapacity, indexer_assignment: &mut IndexerAssignment, ) { - let total_load = indexer_assignment.total_load(problem); - if total_load <= node_max_load { + let total_load = indexer_assignment.total_cpu_load(problem); + if total_load <= indexer_cpu_capacity { return; } - let mut load_to_remove = total_load - node_max_load; - let mut load_sources: Vec<(Load, SourceOrd)> = indexer_assignment + let mut load_to_remove: CpuCapacity = total_load - indexer_cpu_capacity; + let mut source_cpu_capacities: Vec<(CpuCapacity, SourceOrd)> = indexer_assignment .num_shards_per_source .iter() - .map(|(source_ord, num_shards)| { - let load_for_source = problem.source_load_per_shard(*source_ord).get() * num_shards; - (load_for_source, *source_ord) + .map(|(&source_ord, num_shards)| { + let load_for_source = problem.source_load_per_shard(source_ord).get() * num_shards; + (CpuCapacity::from_cpu_millis(load_for_source), source_ord) }) .collect(); - load_sources.sort(); - for (load, source_ord) in load_sources { + source_cpu_capacities.sort(); + for (source_cpu_capacity, source_ord) in source_cpu_capacities { indexer_assignment.num_shards_per_source.remove(&source_ord); - load_to_remove = load_to_remove.saturating_sub(load); - if load_to_remove == 0 { + load_to_remove = if load_to_remove <= source_cpu_capacity { break; - } + } else { + load_to_remove - source_cpu_capacity + }; } - assert_enforce_nodes_max_load_post_condition(problem, indexer_assignment); + assert_enforce_nodes_cpu_capacity_post_condition(problem, indexer_assignment); } -fn assert_enforce_nodes_max_load_post_condition( +fn assert_enforce_nodes_cpu_capacity_post_condition( problem: &SchedulingProblem, indexer_assignment: &IndexerAssignment, ) { - let total_load = indexer_assignment.total_load(problem); - assert!(total_load <= problem.indexer_max_load(indexer_assignment.indexer_ord)); + let total_load = indexer_assignment.total_cpu_load(problem); + assert!(total_load <= problem.indexer_cpu_capacity(indexer_assignment.indexer_ord)); } // ---------------------------------------------------- @@ -199,12 +203,15 @@ fn place_unassigned_shards( let load = source.num_shards * source.load_per_shard.get(); Reverse(load) }); - let mut node_with_least_loads: BinaryHeap<(Load, IndexerOrd)> = + let mut indexers_with_most_available_capacity: BinaryHeap<(CpuCapacity, IndexerOrd)> = compute_indexer_available_capacity(problem, solution); let mut unassignable_shards = BTreeMap::new(); for source in unassigned_shards { - let num_shards_unassigned = - place_unassigned_shards_single_source(&source, &mut node_with_least_loads, solution); + let num_shards_unassigned = place_unassigned_shards_single_source( + &source, + &mut indexers_with_most_available_capacity, + solution, + ); // We haven't been able to place this source entirely. if num_shards_unassigned != 0 { unassignable_shards.insert(source.source_ord, num_shards_unassigned); @@ -237,28 +244,29 @@ fn assert_place_unassigned_shards_post_condition( } // We make sure that all unassigned shard cannot be placed. for indexer_assignment in &solution.indexer_assignments { - let available_capacity: Load = indexer_assignment.indexer_available_capacity(problem); + let available_capacity: CpuCapacity = + indexer_assignment.indexer_available_capacity(problem); for (&source_ord, &num_shards) in unassigned_shards { assert!(num_shards > 0); let source = problem.source(source_ord); - assert!(source.load_per_shard.get() > available_capacity); + assert!(source.load_per_shard.get() > available_capacity.cpu_millis()); } } } fn place_unassigned_shards_single_source( source: &Source, - indexer_available_capacities: &mut BinaryHeap<(Load, IndexerOrd)>, + indexer_available_capacities: &mut BinaryHeap<(CpuCapacity, IndexerOrd)>, solution: &mut SchedulingSolution, ) -> u32 { let mut num_shards = source.num_shards; while num_shards > 0 { - let Some(mut node_with_least_load) = indexer_available_capacities.peek_mut() else { + let Some(mut node_with_most_capacity) = indexer_available_capacities.peek_mut() else { break; }; - let node_id = node_with_least_load.1; - let available_capacity = &mut node_with_least_load.0; - let num_placable_shards = *available_capacity / source.load_per_shard; + let node_id = node_with_most_capacity.1; + let available_capacity = &mut node_with_most_capacity.0; + let num_placable_shards = available_capacity.cpu_millis() / source.load_per_shard; let num_shards_to_place = num_placable_shards.min(num_shards); // We cannot place more shards with this load. if num_shards_to_place == 0 { @@ -267,7 +275,8 @@ fn place_unassigned_shards_single_source( // TODO take in account colocation. // Update the solution, the shard load, and the number of shards to place. solution.indexer_assignments[node_id].add_shards(source.source_ord, num_shards_to_place); - *available_capacity -= num_shards_to_place * source.load_per_shard.get(); + *available_capacity = *available_capacity + - CpuCapacity::from_cpu_millis(num_shards_to_place * source.load_per_shard.get()); num_shards -= num_shards_to_place; } num_shards @@ -298,8 +307,8 @@ fn compute_unassigned_sources( fn compute_indexer_available_capacity( problem: &SchedulingProblem, solution: &SchedulingSolution, -) -> BinaryHeap<(Load, IndexerOrd)> { - let mut indexer_available_capacity: BinaryHeap<(Load, IndexerOrd)> = +) -> BinaryHeap<(CpuCapacity, IndexerOrd)> { + let mut indexer_available_capacity: BinaryHeap<(CpuCapacity, IndexerOrd)> = BinaryHeap::with_capacity(problem.num_indexers()); for indexer_assignment in &solution.indexer_assignments { let available_capacity = indexer_assignment.indexer_available_capacity(problem); @@ -312,12 +321,14 @@ mod tests { use std::num::NonZeroU32; use proptest::prelude::*; + use quickwit_proto::indexing::mcpu; use super::*; #[test] fn test_remove_extranous_shards() { - let mut problem = SchedulingProblem::with_indexer_maximum_load(vec![4_000, 5_000]); + let mut problem = + SchedulingProblem::with_indexer_cpu_capacities(vec![mcpu(4_000), mcpu(5_000)]); problem.add_source(1, NonZeroU32::new(1_000u32).unwrap()); let mut solution = problem.new_solution(); solution.indexer_assignments[0].add_shards(0, 3); @@ -329,7 +340,8 @@ mod tests { #[test] fn test_remove_extranous_shards_2() { - let mut problem = SchedulingProblem::with_indexer_maximum_load(vec![5_000, 4_000]); + let mut problem = + SchedulingProblem::with_indexer_cpu_capacities(vec![mcpu(5_000), mcpu(4_000)]); problem.add_source(2, NonZeroU32::new(1_000).unwrap()); let mut solution = problem.new_solution(); solution.indexer_assignments[0].add_shards(0, 3); @@ -341,7 +353,8 @@ mod tests { #[test] fn test_remove_missing_sources() { - let mut problem = SchedulingProblem::with_indexer_maximum_load(vec![5_000, 4_000]); + let mut problem = + SchedulingProblem::with_indexer_cpu_capacities(vec![mcpu(5_000), mcpu(4_000)]); // Source 0 problem.add_source(0, NonZeroU32::new(1_000).unwrap()); // Source 1 @@ -358,9 +371,14 @@ mod tests { } #[test] - fn test_enforce_nodes_max_load() { - let mut problem = - SchedulingProblem::with_indexer_maximum_load(vec![5_000, 5_000, 5_000, 5_000, 7_000]); + fn test_enforce_nodes_cpu_capacity() { + let mut problem = SchedulingProblem::with_indexer_cpu_capacities(vec![ + mcpu(5_000), + mcpu(5_000), + mcpu(5_000), + mcpu(5_000), + mcpu(7_000), + ]); // Source 0 problem.add_source(10, NonZeroU32::new(3_000).unwrap()); problem.add_source(10, NonZeroU32::new(2_000).unwrap()); @@ -388,7 +406,7 @@ mod tests { solution.indexer_assignments[4].add_shards(1, 1); solution.indexer_assignments[4].add_shards(2, 2); - enforce_nodes_max_load(&problem, &mut solution); + enforce_indexers_cpu_capacity(&problem, &mut solution); assert_eq!(solution.indexer_assignments[0].num_shards(0), 1); assert_eq!(solution.indexer_assignments[0].num_shards(1), 0); @@ -414,7 +432,8 @@ mod tests { #[test] fn test_compute_unassigned_shards_simple() { - let mut problem = SchedulingProblem::with_indexer_maximum_load(vec![0, 4_000]); + let mut problem = + SchedulingProblem::with_indexer_cpu_capacities(vec![mcpu(0), mcpu(4_000)]); problem.add_source(4, NonZeroU32::new(1000).unwrap()); problem.add_source(4, NonZeroU32::new(1_000).unwrap()); let solution = problem.new_solution(); @@ -431,7 +450,8 @@ mod tests { #[test] fn test_compute_unassigned_shards_with_non_trivial_solution() { - let mut problem = SchedulingProblem::with_indexer_maximum_load(vec![50_000, 40_000]); + let mut problem = + SchedulingProblem::with_indexer_cpu_capacities(vec![mcpu(50_000), mcpu(40_000)]); problem.add_source(5, NonZeroU32::new(1_000).unwrap()); problem.add_source(15, NonZeroU32::new(2_000).unwrap()); let mut solution = problem.new_solution(); @@ -461,7 +481,7 @@ mod tests { #[test] fn test_place_unassigned_shards_simple() { - let mut problem = SchedulingProblem::with_indexer_maximum_load(vec![4_000]); + let mut problem = SchedulingProblem::with_indexer_cpu_capacities(vec![mcpu(4_000)]); problem.add_source(4, NonZeroU32::new(1_000).unwrap()); let mut solution = problem.new_solution(); let unassigned = place_unassigned_shards(&problem, &mut solution); @@ -471,7 +491,8 @@ mod tests { #[test] fn test_place_unassigned_shards_reach_capacity() { - let mut problem = SchedulingProblem::with_indexer_maximum_load(vec![50_000, 40_000]); + let mut problem = + SchedulingProblem::with_indexer_cpu_capacities(vec![mcpu(50_000), mcpu(40_000)]); problem.add_source(5, NonZeroU32::new(1_000).unwrap()); problem.add_source(15, NonZeroU32::new(2_000).unwrap()); let mut solution = problem.new_solution(); @@ -504,20 +525,21 @@ mod tests { #[test] fn test_solve() { - let mut problem = SchedulingProblem::with_indexer_maximum_load(vec![800]); + let mut problem = SchedulingProblem::with_indexer_cpu_capacities(vec![mcpu(800)]); problem.add_source(43, NonZeroU32::new(1).unwrap()); problem.add_source(379, NonZeroU32::new(1).unwrap()); let previous_solution = problem.new_solution(); solve(&problem, previous_solution); } - fn node_max_load_strat() -> impl Strategy { + fn indexer_cpu_capacity_strat() -> impl Strategy { prop_oneof![ 0u32..10_000u32, Just(0u32), 800u32..1200u32, 1900u32..2100u32, ] + .prop_map(CpuCapacity::from_cpu_millis) } fn num_shards() -> impl Strategy { @@ -543,10 +565,11 @@ mod tests { num_nodes: usize, num_sources: usize, ) -> impl Strategy { - let node_max_loads_strat = proptest::collection::vec(node_max_load_strat(), num_nodes); + let indexer_cpu_capacity_strat = + proptest::collection::vec(indexer_cpu_capacity_strat(), num_nodes); let sources_strat = proptest::collection::vec(source_strat(), num_sources); - (node_max_loads_strat, sources_strat).prop_map(|(node_max_loads, sources)| { - let mut problem = SchedulingProblem::with_indexer_maximum_load(node_max_loads); + (indexer_cpu_capacity_strat, sources_strat).prop_map(|(node_cpu_capacities, sources)| { + let mut problem = SchedulingProblem::with_indexer_cpu_capacities(node_cpu_capacities); for (num_shards, load_per_shard) in sources { problem.add_source(num_shards, load_per_shard); } diff --git a/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/scheduling_logic_model.rs b/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/scheduling_logic_model.rs index 91255aa7f10..7e61a5cb3ca 100644 --- a/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/scheduling_logic_model.rs +++ b/quickwit/quickwit-control-plane/src/indexing_scheduler/scheduling/scheduling_logic_model.rs @@ -21,10 +21,10 @@ use std::collections::btree_map::Entry; use std::collections::BTreeMap; use std::num::NonZeroU32; +use quickwit_proto::indexing::CpuCapacity; + pub type SourceOrd = u32; pub type IndexerOrd = usize; -pub type Load = u32; - #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct Source { pub source_ord: SourceOrd, @@ -35,22 +35,24 @@ pub struct Source { #[derive(Debug)] pub struct SchedulingProblem { sources: Vec, - indexer_max_loads: Vec, + indexer_cpu_capacities: Vec, } impl SchedulingProblem { pub fn new_solution(&self) -> SchedulingSolution { - SchedulingSolution::with_num_indexers(self.indexer_max_loads.len()) + SchedulingSolution::with_num_indexers(self.indexer_cpu_capacities.len()) } - pub fn indexer_max_load(&self, indexer_ord: IndexerOrd) -> Load { - self.indexer_max_loads[indexer_ord] + pub fn indexer_cpu_capacity(&self, indexer_ord: IndexerOrd) -> CpuCapacity { + self.indexer_cpu_capacities[indexer_ord] } - pub fn with_indexer_maximum_load(node_max_loads: Vec) -> SchedulingProblem { + pub fn with_indexer_cpu_capacities( + indexer_cpu_capacities: Vec, + ) -> SchedulingProblem { SchedulingProblem { sources: Vec::new(), - indexer_max_loads: node_max_loads, + indexer_cpu_capacities, } } @@ -81,7 +83,7 @@ impl SchedulingProblem { } pub fn num_indexers(&self) -> usize { - self.indexer_max_loads.len() + self.indexer_cpu_capacities.len() } } @@ -99,17 +101,25 @@ impl IndexerAssignment { } } - pub fn indexer_available_capacity(&self, problem: &SchedulingProblem) -> Load { - problem.indexer_max_loads[self.indexer_ord].saturating_sub(self.total_load(problem)) + pub fn indexer_available_capacity(&self, problem: &SchedulingProblem) -> CpuCapacity { + let total_cpu_capacity = self.total_cpu_load(problem); + let indexer_cpu_capacity = problem.indexer_cpu_capacities[self.indexer_ord]; + if indexer_cpu_capacity <= total_cpu_capacity { + CpuCapacity::zero() + } else { + indexer_cpu_capacity - total_cpu_capacity + } } - pub fn total_load(&self, problem: &SchedulingProblem) -> Load { - self.num_shards_per_source - .iter() - .map(|(source_ord, num_shards)| { - problem.source_load_per_shard(*source_ord).get() * num_shards - }) - .sum() + pub fn total_cpu_load(&self, problem: &SchedulingProblem) -> CpuCapacity { + CpuCapacity::from_cpu_millis( + self.num_shards_per_source + .iter() + .map(|(source_ord, num_shards)| { + problem.source_load_per_shard(*source_ord).get() * num_shards + }) + .sum(), + ) } pub fn num_shards(&self, source_ord: SourceOrd) -> u32 { diff --git a/quickwit/quickwit-control-plane/src/lib.rs b/quickwit/quickwit-control-plane/src/lib.rs index cc0022bef08..d800544f77b 100644 --- a/quickwit/quickwit-control-plane/src/lib.rs +++ b/quickwit/quickwit-control-plane/src/lib.rs @@ -25,7 +25,7 @@ pub mod ingest; pub(crate) mod metrics; use quickwit_common::tower::Pool; -use quickwit_proto::indexing::{IndexingServiceClient, IndexingTask}; +use quickwit_proto::indexing::{CpuCapacity, IndexingServiceClient, IndexingTask}; use quickwit_proto::types::{IndexUid, SourceId}; /// It can however appear only once in a given index. @@ -41,6 +41,7 @@ pub struct SourceUid { pub struct IndexerNodeInfo { pub client: IndexingServiceClient, pub indexing_tasks: Vec, + pub indexing_capacity: CpuCapacity, } pub type IndexerPool = Pool; diff --git a/quickwit/quickwit-control-plane/src/tests.rs b/quickwit/quickwit-control-plane/src/tests.rs index 384b6681316..2c3d4b8fe29 100644 --- a/quickwit/quickwit-control-plane/src/tests.rs +++ b/quickwit/quickwit-control-plane/src/tests.rs @@ -31,7 +31,7 @@ use quickwit_config::service::QuickwitService; use quickwit_config::{KafkaSourceParams, SourceConfig, SourceInputFormat, SourceParams}; use quickwit_indexing::IndexingService; use quickwit_metastore::{IndexMetadata, ListIndexesMetadataResponseExt}; -use quickwit_proto::indexing::{ApplyIndexingPlanRequest, IndexingServiceClient}; +use quickwit_proto::indexing::{ApplyIndexingPlanRequest, CpuCapacity, IndexingServiceClient}; use quickwit_proto::metastore::{ ListIndexesMetadataResponse, ListShardsResponse, MetastoreServiceClient, }; @@ -91,6 +91,7 @@ pub fn test_indexer_change_stream( IndexerNodeInfo { client, indexing_tasks, + indexing_capacity: CpuCapacity::from_cpu_millis(4_000), }, )) } @@ -156,6 +157,7 @@ async fn start_control_plane( #[tokio::test] async fn test_scheduler_scheduling_and_control_loop_apply_plan_again() { + quickwit_common::setup_logging_for_tests(); let transport = ChannelTransport::default(); let cluster = create_cluster_for_test(Vec::new(), &["indexer", "control_plane"], &transport, true) diff --git a/quickwit/quickwit-indexing/src/actors/indexer.rs b/quickwit/quickwit-indexing/src/actors/indexer.rs index ab9d2594375..b13823fbf01 100644 --- a/quickwit/quickwit-indexing/src/actors/indexer.rs +++ b/quickwit/quickwit-indexing/src/actors/indexer.rs @@ -38,7 +38,9 @@ use quickwit_common::temp_dir::TempDirectory; use quickwit_config::IndexingSettings; use quickwit_doc_mapper::DocMapper; use quickwit_metastore::checkpoint::{IndexCheckpointDelta, SourceCheckpointDelta}; -use quickwit_proto::indexing::{IndexingPipelineId, PipelineMetrics}; +use quickwit_proto::indexing::{ + CpuCapacity, IndexingPipelineId, PipelineMetrics, PIPELINE_FULL_CAPACITY, +}; use quickwit_proto::metastore::{ LastDeleteOpstampRequest, MetastoreService, MetastoreServiceClient, }; @@ -551,11 +553,9 @@ impl Indexer { fn update_pipeline_metrics(&mut self, elapsed: Duration, uncompressed_num_bytes: u64) { let commit_timeout = self.indexer_state.indexing_settings.commit_timeout(); - let cpu_millis: u16 = if elapsed >= commit_timeout { - 1_000 - } else { - (elapsed.as_micros() * 1_000 / commit_timeout.as_micros()) as u16 - }; + let pipeline_throughput_fraction = + (elapsed.as_micros() as f32 / commit_timeout.as_micros() as f32).min(1.0f32); + let cpu_millis: CpuCapacity = PIPELINE_FULL_CAPACITY * pipeline_throughput_fraction; self.counters.pipeline_metrics_opt = Some(PipelineMetrics { cpu_millis, throughput_mb_per_sec: (uncompressed_num_bytes / (1u64 + elapsed.as_micros() as u64)) diff --git a/quickwit/quickwit-indexing/src/lib.rs b/quickwit/quickwit-indexing/src/lib.rs index 61c0bc655b5..ba0169b3d5e 100644 --- a/quickwit/quickwit-indexing/src/lib.rs +++ b/quickwit/quickwit-indexing/src/lib.rs @@ -47,6 +47,7 @@ mod split_store; #[cfg(any(test, feature = "testsuite"))] mod test_utils; +use quickwit_proto::indexing::CpuCapacity; #[cfg(any(test, feature = "testsuite"))] pub use test_utils::{mock_split, mock_split_meta, MockSplitBuilder, TestSandbox}; @@ -54,7 +55,7 @@ use self::merge_policy::MergePolicy; pub use self::source::check_source_connectivity; #[derive(utoipa::OpenApi)] -#[openapi(components(schemas(IndexingStatistics, PipelineMetrics)))] +#[openapi(components(schemas(IndexingStatistics, PipelineMetrics, CpuCapacity)))] /// Schema used for the OpenAPI generation which are apart of this crate. pub struct IndexingApiSchemas; diff --git a/quickwit/quickwit-proto/src/indexing/mod.rs b/quickwit/quickwit-proto/src/indexing/mod.rs index 4d380315f53..f55c4a83ebd 100644 --- a/quickwit/quickwit-proto/src/indexing/mod.rs +++ b/quickwit/quickwit-proto/src/indexing/mod.rs @@ -17,8 +17,9 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::fmt::Formatter; +use std::fmt::{Display, Formatter}; use std::hash::Hash; +use std::ops::{Add, Mul, Sub}; use std::{fmt, io}; use anyhow::anyhow; @@ -150,13 +151,13 @@ pub struct IndexingPipelineId { pub pipeline_ord: usize, } -impl fmt::Display for IndexingPipelineId { +impl Display for IndexingPipelineId { fn fmt(&self, f: &mut Formatter) -> fmt::Result { write!(f, "{}:{}", self.index_uid, &self.source_id) } } -impl fmt::Display for IndexingTask { +impl Display for IndexingTask { fn fmt(&self, f: &mut Formatter) -> fmt::Result { write!(f, "{}:{}", self.index_uid, &self.source_id) } @@ -209,13 +210,146 @@ impl TryFrom<&str> for IndexingTask { #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, utoipa::ToSchema)] pub struct PipelineMetrics { - pub cpu_millis: u16, + pub cpu_millis: CpuCapacity, pub throughput_mb_per_sec: u16, } -impl fmt::Display for PipelineMetrics { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}m,{}MB/s", self.cpu_millis, self.throughput_mb_per_sec) +impl Display for PipelineMetrics { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "{},{}MB/s", self.cpu_millis, self.throughput_mb_per_sec) + } +} + +/// One full pipeline (including merging) is assumed to consume 4 CPU threads. +/// The actual number somewhere between 3 and 4. +pub const PIPELINE_FULL_CAPACITY: CpuCapacity = CpuCapacity::from_cpu_millis(4_000u32); + +/// The CpuCapacity represents an amount of CPU resource available. +/// +/// It is usually expressed in CPU millis (For instance, one full CPU thread is +/// displayed as `1000m`). +#[derive( + Copy, Clone, Debug, Eq, PartialEq, Deserialize, Serialize, Ord, PartialOrd, utoipa::ToSchema, +)] +#[serde( + into = "CpuCapacityForSerialization", + try_from = "CpuCapacityForSerialization" +)] +pub struct CpuCapacity(u32); + +/// Short helper function to build `CpuCapacity`. +#[inline(always)] +pub const fn mcpu(milli_cpus: u32) -> CpuCapacity { + CpuCapacity::from_cpu_millis(milli_cpus) +} + +impl CpuCapacity { + #[inline(always)] + pub const fn from_cpu_millis(cpu_millis: u32) -> CpuCapacity { + CpuCapacity(cpu_millis) + } + + #[inline(always)] + pub fn cpu_millis(self) -> u32 { + self.0 + } + + #[inline(always)] + pub fn zero() -> CpuCapacity { + CpuCapacity::from_cpu_millis(0u32) + } + + #[inline(always)] + pub fn one_cpu_thread() -> CpuCapacity { + CpuCapacity::from_cpu_millis(1_000u32) + } +} + +impl Sub for CpuCapacity { + type Output = CpuCapacity; + + #[inline(always)] + fn sub(self, rhs: CpuCapacity) -> Self::Output { + CpuCapacity::from_cpu_millis(self.0 - rhs.0) + } +} + +impl Add for CpuCapacity { + type Output = CpuCapacity; + + #[inline(always)] + fn add(self, rhs: CpuCapacity) -> Self::Output { + CpuCapacity::from_cpu_millis(self.0 + rhs.0) + } +} + +impl Mul for CpuCapacity { + type Output = CpuCapacity; + + #[inline(always)] + fn mul(self, rhs: u32) -> CpuCapacity { + CpuCapacity::from_cpu_millis(self.0 * rhs) + } +} + +impl Mul for CpuCapacity { + type Output = CpuCapacity; + + #[inline(always)] + fn mul(self, scale: f32) -> CpuCapacity { + CpuCapacity::from_cpu_millis((self.0 as f32 * scale) as u32) + } +} + +impl Display for CpuCapacity { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + write!(f, "{}m", self.0) + } +} + +#[derive(Serialize, Deserialize)] +#[serde(untagged)] +enum CpuCapacityForSerialization { + Float(f32), + MilliCpuWithUnit(String), +} + +impl TryFrom for CpuCapacity { + type Error = String; + + fn try_from( + cpu_capacity_for_serialization: CpuCapacityForSerialization, + ) -> Result { + match cpu_capacity_for_serialization { + CpuCapacityForSerialization::Float(cpu_capacity) => { + Ok(CpuCapacity((cpu_capacity * 1000.0f32) as u32)) + } + CpuCapacityForSerialization::MilliCpuWithUnit(cpu_capacity_str) => { + Self::from_str(&cpu_capacity_str) + } + } + } +} + +impl FromStr for CpuCapacity { + type Err = String; + + fn from_str(cpu_capacity_str: &str) -> Result { + let Some(milli_cpus_without_unit_str) = cpu_capacity_str.strip_suffix('m') else { + return Err(format!( + "invalid cpu capacity: `{cpu_capacity_str}`. String format expects a trailing 'm'." + )); + }; + let milli_cpus: u32 = milli_cpus_without_unit_str + .parse::() + .map_err(|_err| format!("invalid cpu capacity: `{cpu_capacity_str}`."))?; + Ok(CpuCapacity(milli_cpus)) + } +} + +impl From for CpuCapacityForSerialization { + fn from(cpu_capacity: CpuCapacity) -> CpuCapacityForSerialization { + CpuCapacityForSerialization::MilliCpuWithUnit(format!("{}m", cpu_capacity.0)) } } @@ -223,6 +357,37 @@ impl fmt::Display for PipelineMetrics { mod tests { use super::*; + #[test] + fn test_cpu_capacity_serialization() { + assert_eq!(CpuCapacity::from_str("2000m").unwrap(), mcpu(2000)); + assert_eq!(CpuCapacity::from_cpu_millis(2500), mcpu(2500)); + assert_eq!( + CpuCapacity::from_str("2.5").unwrap_err(), + "invalid cpu capacity: `2.5`. String format expects a trailing 'm'." + ); + assert_eq!( + serde_json::from_value::(serde_json::Value::String("1200m".to_string())) + .unwrap(), + mcpu(1200) + ); + assert_eq!( + serde_json::from_value::(serde_json::Value::Number( + serde_json::Number::from_f64(1.2f64).unwrap() + )) + .unwrap(), + mcpu(1200) + ); + assert_eq!( + serde_json::from_value::(serde_json::Value::Number( + serde_json::Number::from(1u32) + )) + .unwrap(), + mcpu(1000) + ); + assert_eq!(CpuCapacity::from_cpu_millis(2500).to_string(), "2500m"); + assert_eq!(serde_json::to_string(&mcpu(2500)).unwrap(), "\"2500m\""); + } + #[test] fn test_indexing_task_serialization() { let original = IndexingTask { diff --git a/quickwit/quickwit-serve/src/lib.rs b/quickwit/quickwit-serve/src/lib.rs index f8e8ae513eb..e903211a632 100644 --- a/quickwit/quickwit-serve/src/lib.rs +++ b/quickwit/quickwit-serve/src/lib.rs @@ -694,7 +694,7 @@ fn setup_indexer_pool( { let node_id = node.node_id().to_string(); let indexing_tasks = node.indexing_tasks().to_vec(); - + let indexing_capacity = node.indexing_capacity(); if node.is_self_node() { if let Some(indexing_service_clone) = indexing_service_clone_opt { let client = @@ -704,6 +704,7 @@ fn setup_indexer_pool( IndexerNodeInfo { client, indexing_tasks, + indexing_capacity, }, )) } else { @@ -721,6 +722,7 @@ fn setup_indexer_pool( IndexerNodeInfo { client, indexing_tasks, + indexing_capacity, }, )) } From 9fe59f8662dcef2a860543dc83b11a4f0f980320 Mon Sep 17 00:00:00 2001 From: PSeitz Date: Thu, 9 Nov 2023 17:45:24 +0100 Subject: [PATCH 15/27] update tantivy (#4102) --- quickwit/Cargo.lock | 30 ++++++++++++------- quickwit/Cargo.toml | 2 +- .../src/actors/merge_executor.rs | 6 ++-- .../src/split_store/indexing_split_store.rs | 4 +-- quickwit/quickwit-search/src/collector.rs | 3 +- 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 4578b2506aa..cfca821f1ff 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -951,6 +951,14 @@ dependencies = [ "serde", ] +[[package]] +name = "bitpacking" +version = "0.8.3" +source = "git+https://github.com/quickwit-oss/bitpacking?rev=f730b75#f730b75598589d7c2a9c4eee3fc17025c59e2b2c" +dependencies = [ + "crunchy", +] + [[package]] name = "bitpacking" version = "0.8.4" @@ -4347,7 +4355,7 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "ownedbytes" version = "0.6.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=ecb9a89#ecb9a89a9ff0573fba5c2cf8b83f353f35022310" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=927b443#927b4432c935b55cb8a2275daea3f77fc6823bd6" dependencies = [ "stable_deref_trait", ] @@ -7503,13 +7511,13 @@ dependencies = [ [[package]] name = "tantivy" version = "0.21.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=ecb9a89#ecb9a89a9ff0573fba5c2cf8b83f353f35022310" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=927b443#927b4432c935b55cb8a2275daea3f77fc6823bd6" dependencies = [ "aho-corasick", "arc-swap", "async-trait", "base64 0.21.5", - "bitpacking", + "bitpacking 0.8.3", "byteorder", "census", "crc32fast", @@ -7558,15 +7566,15 @@ dependencies = [ [[package]] name = "tantivy-bitpacker" version = "0.5.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=ecb9a89#ecb9a89a9ff0573fba5c2cf8b83f353f35022310" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=927b443#927b4432c935b55cb8a2275daea3f77fc6823bd6" dependencies = [ - "bitpacking", + "bitpacking 0.8.4", ] [[package]] name = "tantivy-columnar" version = "0.2.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=ecb9a89#ecb9a89a9ff0573fba5c2cf8b83f353f35022310" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=927b443#927b4432c935b55cb8a2275daea3f77fc6823bd6" dependencies = [ "fastdivide", "fnv", @@ -7581,7 +7589,7 @@ dependencies = [ [[package]] name = "tantivy-common" version = "0.6.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=ecb9a89#ecb9a89a9ff0573fba5c2cf8b83f353f35022310" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=927b443#927b4432c935b55cb8a2275daea3f77fc6823bd6" dependencies = [ "async-trait", "byteorder", @@ -7604,7 +7612,7 @@ dependencies = [ [[package]] name = "tantivy-query-grammar" version = "0.21.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=ecb9a89#ecb9a89a9ff0573fba5c2cf8b83f353f35022310" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=927b443#927b4432c935b55cb8a2275daea3f77fc6823bd6" dependencies = [ "nom", ] @@ -7612,7 +7620,7 @@ dependencies = [ [[package]] name = "tantivy-sstable" version = "0.2.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=ecb9a89#ecb9a89a9ff0573fba5c2cf8b83f353f35022310" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=927b443#927b4432c935b55cb8a2275daea3f77fc6823bd6" dependencies = [ "tantivy-common", "tantivy-fst", @@ -7622,7 +7630,7 @@ dependencies = [ [[package]] name = "tantivy-stacker" version = "0.2.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=ecb9a89#ecb9a89a9ff0573fba5c2cf8b83f353f35022310" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=927b443#927b4432c935b55cb8a2275daea3f77fc6823bd6" dependencies = [ "murmurhash32", "tantivy-common", @@ -7631,7 +7639,7 @@ dependencies = [ [[package]] name = "tantivy-tokenizer-api" version = "0.2.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=ecb9a89#ecb9a89a9ff0573fba5c2cf8b83f353f35022310" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=927b443#927b4432c935b55cb8a2275daea3f77fc6823bd6" dependencies = [ "serde", ] diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index 43c4e51f1f0..2bb69d35805 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -241,7 +241,7 @@ quickwit-serve = { version = "0.6.3", path = "./quickwit-serve" } quickwit-storage = { version = "0.6.3", path = "./quickwit-storage" } quickwit-telemetry = { version = "0.6.3", path = "./quickwit-telemetry" } -tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "ecb9a89", default-features = false, features = [ +tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "927b443", default-features = false, features = [ "mmap", "lz4-compression", "zstd-compression", diff --git a/quickwit/quickwit-indexing/src/actors/merge_executor.rs b/quickwit/quickwit-indexing/src/actors/merge_executor.rs index d4ffb1fdf60..9444a05c22c 100644 --- a/quickwit/quickwit-indexing/src/actors/merge_executor.rs +++ b/quickwit/quickwit-indexing/src/actors/merge_executor.rs @@ -41,11 +41,9 @@ use quickwit_proto::metastore::{ }; use quickwit_query::get_quickwit_fastfield_normalizer_manager; use quickwit_query::query_ast::QueryAst; -use tantivy::directory::{DirectoryClone, MmapDirectory, RamDirectory}; +use tantivy::directory::{Advice, DirectoryClone, MmapDirectory, RamDirectory}; use tantivy::tokenizer::TokenizerManager; -use tantivy::{ - Advice, DateTime, Directory, Index, IndexMeta, IndexWriter, SegmentId, SegmentReader, -}; +use tantivy::{DateTime, Directory, Index, IndexMeta, IndexWriter, SegmentId, SegmentReader}; use tokio::runtime::Handle; use tracing::{debug, info, instrument, warn}; diff --git a/quickwit/quickwit-indexing/src/split_store/indexing_split_store.rs b/quickwit/quickwit-indexing/src/split_store/indexing_split_store.rs index 097f5304032..6f5e2a231fe 100644 --- a/quickwit/quickwit-indexing/src/split_store/indexing_split_store.rs +++ b/quickwit/quickwit-indexing/src/split_store/indexing_split_store.rs @@ -30,8 +30,8 @@ use quickwit_common::io::{IoControls, IoControlsAccess}; use quickwit_common::uri::Uri; use quickwit_metastore::SplitMetadata; use quickwit_storage::{PutPayload, Storage, StorageResult}; -use tantivy::directory::MmapDirectory; -use tantivy::{Advice, Directory}; +use tantivy::directory::{Advice, MmapDirectory}; +use tantivy::Directory; use time::OffsetDateTime; use tracing::{info, info_span, instrument, Instrument}; diff --git a/quickwit/quickwit-search/src/collector.rs b/quickwit/quickwit-search/src/collector.rs index b91033f16fd..bd0982ed6eb 100644 --- a/quickwit/quickwit-search/src/collector.rs +++ b/quickwit/quickwit-search/src/collector.rs @@ -1227,8 +1227,9 @@ mod tests { } fn make_index() -> tantivy::Index { + use tantivy::indexer::UserOperation; use tantivy::schema::{NumericOptions, Schema}; - use tantivy::{Index, UserOperation}; + use tantivy::Index; let dataset = sort_dataset(); From 98e8dbb1ee9f46a87629acdeec830884de5476fd Mon Sep 17 00:00:00 2001 From: Harrison Burt <57491488+ChillFish8@users.noreply.github.com> Date: Thu, 9 Nov 2023 17:19:33 +0000 Subject: [PATCH 16/27] Update comment (#2903) --- .../quickwit-indexing/src/source/pulsar_source.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/quickwit/quickwit-indexing/src/source/pulsar_source.rs b/quickwit/quickwit-indexing/src/source/pulsar_source.rs index 691609464ce..4ee7718841e 100644 --- a/quickwit/quickwit-indexing/src/source/pulsar_source.rs +++ b/quickwit/quickwit-indexing/src/source/pulsar_source.rs @@ -339,7 +339,16 @@ async fn create_pulsar_consumer( fn msg_id_to_position(msg: &MessageIdData) -> Position { // The order of these fields are important as they affect the sorting // of the checkpoint positions. - // TODO: Confirm this layout is correct? + // + // The key parts of the ID used for ordering are: + // - The ledger ID which is a sequentially increasing ID. + // - The entry ID the unique ID of the message within the ledger. + // - The batch position for the current chunk of messages. + // + // The remaining keys are not required for sorting but are required + // in order to re-construct the message ID in order to send back to pulsar. + // The ledger_id, entry_id and the batch_index form a unique composite key which will + // prevent the remaining parts of the ID from interfering with the sorting. let id_str = format!( "{:0>20},{:0>20},{},{},{}", msg.ledger_id, From da2f0b41dc1f0169d6568b56f03b4eedd363d1a0 Mon Sep 17 00:00:00 2001 From: Remi Dettai Date: Thu, 9 Nov 2023 18:59:10 +0100 Subject: [PATCH 17/27] Shorten overly verbose debug log (#4104) --- quickwit/quickwit-search/src/root.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/quickwit/quickwit-search/src/root.rs b/quickwit/quickwit-search/src/root.rs index bbbe67f3ade..4d0f459e65a 100644 --- a/quickwit/quickwit-search/src/root.rs +++ b/quickwit/quickwit-search/src/root.rs @@ -498,8 +498,13 @@ pub(crate) async fn search_partial_hits_phase( .await .context("failed to merge leaf search responses")? .map_err(|error: TantivyError| crate::SearchError::Internal(error.to_string()))?; - debug!(leaf_search_response = ?leaf_search_response, "Merged leaf search response."); - + debug!( + num_hits = leaf_search_response.num_hits, + failed_splits = ?leaf_search_response.failed_splits, + num_attempted_splits = leaf_search_response.num_attempted_splits, + has_intermediate_aggregation_result = leaf_search_response.intermediate_aggregation_result.is_some(), + "Merged leaf search response." + ); if !leaf_search_response.failed_splits.is_empty() { error!(failed_splits = ?leaf_search_response.failed_splits, "Leaf search response contains at least one failed split."); let errors: String = leaf_search_response.failed_splits.iter().join(", "); @@ -1061,7 +1066,10 @@ pub async fn root_list_terms( merged_iter.collect() }; - debug!(leaf_list_terms_response = ?leaf_list_terms_response, "Merged leaf search response."); + debug!( + leaf_list_terms_response_count = leaf_list_terms_response.len(), + "Merged leaf search response." + ); let elapsed = start_instant.elapsed(); From 83c2cf506d71adf86ef9c93119c731f99f5ee687 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Nov 2023 15:56:11 -0500 Subject: [PATCH 18/27] Bump tokio from 1.33.0 to 1.34.0 in /quickwit (#4108) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.33.0 to 1.34.0. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.33.0...tokio-1.34.0) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- quickwit/Cargo.lock | 8 ++++---- quickwit/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index cfca821f1ff..a0a3bea8598 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -7844,9 +7844,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.33.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" dependencies = [ "backtrace", "bytes", @@ -7874,9 +7874,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index 2bb69d35805..13f78cc72dd 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -167,7 +167,7 @@ thousands = "0.2.0" tikv-jemalloc-ctl = "0.5" tikv-jemallocator = "0.5" time = { version = "0.3.17", features = ["std", "formatting", "macros"] } -tokio = { version = "1.33", features = ["full"] } +tokio = { version = "1.34", features = ["full"] } tokio-stream = { version = "0.1", features = ["sync"] } tokio-util = { version = "0.7", features = ["full"] } toml = "0.7.6" From beabe88d890960429cba11fe127c331edd5044e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Massot?= Date: Fri, 10 Nov 2023 00:01:39 +0100 Subject: [PATCH 19/27] Add ingester close shards gRPC (#4088) * Add ingester close shards gRPC * Apply suggestions from code review Co-authored-by: Adrien Guillo * Clean. --------- Co-authored-by: Adrien Guillo --- .../src/ingest/ingest_controller.rs | 4 +- .../quickwit-ingest/src/ingest_v2/fetch.rs | 4 +- .../quickwit-ingest/src/ingest_v2/ingester.rs | 158 ++++++++++++- .../quickwit-ingest/src/ingest_v2/models.rs | 9 + .../src/ingest_v2/replication.rs | 44 ++-- .../quickwit-ingest/src/ingest_v2/router.rs | 4 +- .../src/ingest_v2/shard_table.rs | 3 +- .../protos/quickwit/control_plane.proto | 8 +- .../protos/quickwit/ingest.proto | 6 + .../protos/quickwit/ingester.proto | 9 + .../quickwit/quickwit.control_plane.rs | 13 +- .../quickwit/quickwit.ingest.ingester.rs | 209 ++++++++++++++++++ .../src/codegen/quickwit/quickwit.ingest.rs | 11 + quickwit/quickwit-proto/src/ingest/mod.rs | 10 +- 14 files changed, 430 insertions(+), 62 deletions(-) diff --git a/quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs b/quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs index 6b92bec7a8d..2d086d6d54a 100644 --- a/quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs +++ b/quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs @@ -25,12 +25,12 @@ use itertools::Itertools; use quickwit_common::{PrettySample, Progress}; use quickwit_ingest::IngesterPool; use quickwit_proto::control_plane::{ - ClosedShards, ControlPlaneError, ControlPlaneResult, GetOrCreateOpenShardsFailure, + ControlPlaneError, ControlPlaneResult, GetOrCreateOpenShardsFailure, GetOrCreateOpenShardsFailureReason, GetOrCreateOpenShardsRequest, GetOrCreateOpenShardsResponse, GetOrCreateOpenShardsSuccess, }; use quickwit_proto::ingest::ingester::{IngesterService, PingRequest}; -use quickwit_proto::ingest::{IngestV2Error, ShardState}; +use quickwit_proto::ingest::{ClosedShards, IngestV2Error, ShardState}; use quickwit_proto::metastore; use quickwit_proto::metastore::{MetastoreService, MetastoreServiceClient}; use quickwit_proto::types::{IndexUid, NodeId}; diff --git a/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs b/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs index ff8f48716d7..97efd81d0d2 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs @@ -513,13 +513,13 @@ async fn fault_tolerant_fetch_task( } #[derive(Debug, Clone, Copy)] -struct FetchRange { +pub(super) struct FetchRange { from_position_exclusive_opt: Option, to_position_inclusive_opt: Option, } impl FetchRange { - fn new(from_position_exclusive: Position, to_position_inclusive: Position) -> Self { + pub(super) fn new(from_position_exclusive: Position, to_position_inclusive: Position) -> Self { Self { from_position_exclusive_opt: from_position_exclusive.as_u64(), to_position_inclusive_opt: to_position_inclusive.as_u64(), diff --git a/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs b/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs index d8204ce0da2..a92755d3c2f 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs @@ -33,11 +33,12 @@ use mrecordlog::MultiRecordLog; use quickwit_common::tower::Pool; use quickwit_common::ServiceStream; use quickwit_proto::ingest::ingester::{ - AckReplicationMessage, FetchResponseV2, IngesterService, IngesterServiceClient, - IngesterServiceStream, OpenFetchStreamRequest, OpenReplicationStreamRequest, - OpenReplicationStreamResponse, PersistFailure, PersistFailureReason, PersistRequest, - PersistResponse, PersistSuccess, PingRequest, PingResponse, ReplicateRequest, - ReplicateSubrequest, SynReplicationMessage, TruncateRequest, TruncateResponse, + AckReplicationMessage, CloseShardsRequest, CloseShardsResponse, FetchResponseV2, + IngesterService, IngesterServiceClient, IngesterServiceStream, OpenFetchStreamRequest, + OpenReplicationStreamRequest, OpenReplicationStreamResponse, PersistFailure, + PersistFailureReason, PersistRequest, PersistResponse, PersistSuccess, PingRequest, + PingResponse, ReplicateRequest, ReplicateSubrequest, SynReplicationMessage, TruncateRequest, + TruncateResponse, }; use quickwit_proto::ingest::{CommitTypeV2, IngestV2Error, IngestV2Result, ShardState}; use quickwit_proto::types::{NodeId, Position, QueueId}; @@ -396,11 +397,6 @@ impl IngesterService for Ingester { persist_successes.push(persist_success); } } - let _state_guard = self.state.write().await; - - for persist_success in &persist_successes { - let _queue_id = persist_success.queue_id(); - } let leader_id = self.self_node_id.to_string(); let persist_response = PersistResponse { leader_id, @@ -537,6 +533,29 @@ impl IngesterService for Ingester { let truncate_response = TruncateResponse {}; Ok(truncate_response) } + + async fn close_shards( + &mut self, + close_shards_request: CloseShardsRequest, + ) -> IngestV2Result { + let mut state_guard = self.state.write().await; + for close_shard in close_shards_request.closed_shards { + for queue_id in close_shard.queue_ids() { + if !state_guard.mrecordlog.queue_exists(&queue_id) { + continue; + } + append_eof_record_if_necessary(&mut state_guard.mrecordlog, &queue_id).await; + let shard = state_guard + .shards + .get_mut(&queue_id) + .expect("shard must exist"); + // Notify fetch task. + shard.notify_new_records(); + shard.close(); + } + } + Ok(CloseShardsResponse {}) + } } /// Appends an EOF record to the queue if the it is empty or the last record is not an EOF @@ -574,11 +593,12 @@ mod tests { IngesterServiceGrpcServer, IngesterServiceGrpcServerAdapter, PersistSubrequest, TruncateSubrequest, }; - use quickwit_proto::ingest::DocBatchV2; + use quickwit_proto::ingest::{ClosedShards, DocBatchV2}; use quickwit_proto::types::queue_id; use tonic::transport::{Endpoint, Server}; use super::*; + use crate::ingest_v2::fetch::FetchRange; use crate::ingest_v2::test_utils::{IngesterShardTestExt, MultiRecordLogTestExt}; #[tokio::test] @@ -1248,4 +1268,120 @@ mod tests { .mrecordlog .assert_records_eq(&queue_id_02, .., &[]); } + + #[tokio::test] + async fn test_ingester_close_shards() { + let tempdir = tempfile::tempdir().unwrap(); + let self_node_id: NodeId = "test-ingester-0".into(); + let ingester_pool = IngesterPool::default(); + let wal_dir_path = tempdir.path(); + let replication_factor = 1; + let mut ingester = Ingester::try_new( + self_node_id.clone(), + ingester_pool, + wal_dir_path, + replication_factor, + ) + .await + .unwrap(); + + let queue_id_01 = queue_id("test-index:0", "test-source:0", 1); + let queue_id_02 = queue_id("test-index:0", "test-source:0", 2); + let queue_id_03 = queue_id("test-index:1", "test-source:1", 3); + + let mut state_guard = ingester.state.write().await; + for queue_id in &[&queue_id_01, &queue_id_02, &queue_id_03] { + ingester + .create_shard(&mut state_guard, queue_id, &self_node_id, None) + .await + .unwrap(); + let records = [ + MRecord::new_doc("test-doc-010").encode(), + MRecord::new_doc("test-doc-011").encode(), + ] + .into_iter(); + state_guard + .mrecordlog + .append_records(&queue_id_01, None, records) + .await + .unwrap(); + } + + drop(state_guard); + + let client_id = "test-client".to_string(); + let open_fetch_stream_request = OpenFetchStreamRequest { + client_id: client_id.clone(), + index_uid: "test-index:0".to_string(), + source_id: "test-source:0".to_string(), + shard_id: 1, + from_position_exclusive: None, + to_position_inclusive: None, + }; + + let mut fetch_stream = ingester + .open_fetch_stream(open_fetch_stream_request) + .await + .unwrap(); + let fetch_response = fetch_stream.next().await.unwrap().unwrap(); + assert_eq!( + fetch_response.from_position_exclusive(), + Position::Beginning + ); + + let close_shard_1 = ClosedShards { + index_uid: "test-index:0".to_string(), + source_id: "test-source:0".to_string(), + shard_ids: vec![1, 2], + }; + let close_shard_2 = ClosedShards { + index_uid: "test-index:1".to_string(), + source_id: "test-source:1".to_string(), + shard_ids: vec![3], + }; + let close_shard_with_no_queue = ClosedShards { + index_uid: "test-index:2".to_string(), + source_id: "test-source:2".to_string(), + shard_ids: vec![4], + }; + let closed_shards = vec![ + close_shard_1.clone(), + close_shard_2.clone(), + close_shard_with_no_queue, + ]; + let close_shards_request = CloseShardsRequest { + closed_shards: closed_shards.clone(), + }; + ingester.close_shards(close_shards_request).await.unwrap(); + + // Check that shards are closed and EOF records are appended. + let state_guard = ingester.state.read().await; + for shard in state_guard.shards.values() { + shard.assert_is_closed(); + } + for closed_shards in [&close_shard_1, &close_shard_2] { + for queue_id in closed_shards.queue_ids() { + let last_position = state_guard + .mrecordlog + .range( + &queue_id, + FetchRange::new(Position::Beginning, Position::Beginning), + ) + .unwrap() + .last() + .unwrap(); + assert!(is_eof_mrecord(&last_position.1)); + } + } + + // Check that fetch task is notified. + // Note: fetch stream should not block if the close shard call notified the fetch task. + let fetch_response = + tokio::time::timeout(std::time::Duration::from_millis(50), fetch_stream.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(fetch_response.to_position_inclusive(), Position::Eof); + } } diff --git a/quickwit/quickwit-ingest/src/ingest_v2/models.rs b/quickwit/quickwit-ingest/src/ingest_v2/models.rs index c167c81c249..9cf73a1acb4 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/models.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/models.rs @@ -137,6 +137,15 @@ impl IngesterShard { .is_closed() } + pub fn close(&mut self) { + let shard_state = match self { + IngesterShard::Primary(primary_shard) => &mut primary_shard.shard_state, + IngesterShard::Replica(replica_shard) => &mut replica_shard.shard_state, + IngesterShard::Solo(solo_shard) => &mut solo_shard.shard_state, + }; + *shard_state = ShardState::Closed; + } + pub fn replication_position_inclusive(&self) -> Position { match self { IngesterShard::Primary(primary_shard) => &primary_shard.replication_position_inclusive, diff --git a/quickwit/quickwit-ingest/src/ingest_v2/replication.rs b/quickwit/quickwit-ingest/src/ingest_v2/replication.rs index e6fe41bbe68..1feb4a5e1a0 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/replication.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/replication.rs @@ -323,24 +323,23 @@ impl ReplicationTask { let from_position_exclusive = subrequest.from_position_exclusive(); let to_position_inclusive = subrequest.to_position_inclusive(); - let replica_shard: &mut IngesterShard = - if from_position_exclusive == Position::Beginning { - // Initialize the replica shard and corresponding mrecordlog queue. - state_guard - .mrecordlog - .create_queue(&queue_id) - .await - .expect("TODO"); - let leader_id: NodeId = replicate_request.leader_id.clone().into(); - let replica_shard = ReplicaShard::new(leader_id); - let shard = IngesterShard::Replica(replica_shard); - state_guard.shards.entry(queue_id.clone()).or_insert(shard) - } else { - state_guard - .shards - .get_mut(&queue_id) - .expect("replica shard should be initialized") - }; + let replica_shard: &IngesterShard = if from_position_exclusive == Position::Beginning { + // Initialize the replica shard and corresponding mrecordlog queue. + state_guard + .mrecordlog + .create_queue(&queue_id) + .await + .expect("TODO"); + let leader_id: NodeId = replicate_request.leader_id.clone().into(); + let replica_shard = ReplicaShard::new(leader_id); + let shard = IngesterShard::Replica(replica_shard); + state_guard.shards.entry(queue_id.clone()).or_insert(shard) + } else { + state_guard + .shards + .get(&queue_id) + .expect("replica shard should be initialized") + }; if replica_shard.is_closed() { // TODO } @@ -380,17 +379,16 @@ impl ReplicationTask { .replicated_num_docs_total .inc_by(batch_num_docs); - let replica_shard = state_guard - .shards - .get_mut(&queue_id) - .expect("replica shard should exist"); - if current_position_inclusive != to_position_inclusive { return Err(IngestV2Error::Internal(format!( "bad replica position: expected {to_position_inclusive:?}, got \ {current_position_inclusive:?}" ))); } + let replica_shard = state_guard + .shards + .get_mut(&queue_id) + .expect("replica shard should be initialized"); replica_shard.set_replication_position_inclusive(current_position_inclusive.clone()); let replicate_success = ReplicateSuccess { diff --git a/quickwit/quickwit-ingest/src/ingest_v2/router.rs b/quickwit/quickwit-ingest/src/ingest_v2/router.rs index 438c871e105..71ec7f481c6 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/router.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/router.rs @@ -26,7 +26,7 @@ use async_trait::async_trait; use futures::stream::FuturesUnordered; use futures::{Future, StreamExt}; use quickwit_proto::control_plane::{ - ClosedShards, ControlPlaneService, ControlPlaneServiceClient, GetOrCreateOpenShardsRequest, + ControlPlaneService, ControlPlaneServiceClient, GetOrCreateOpenShardsRequest, GetOrCreateOpenShardsSubrequest, }; use quickwit_proto::ingest::ingester::{ @@ -35,7 +35,7 @@ use quickwit_proto::ingest::ingester::{ use quickwit_proto::ingest::router::{ IngestRequestV2, IngestResponseV2, IngestRouterService, IngestSubrequest, }; -use quickwit_proto::ingest::{CommitTypeV2, IngestV2Error, IngestV2Result}; +use quickwit_proto::ingest::{ClosedShards, CommitTypeV2, IngestV2Error, IngestV2Result}; use quickwit_proto::types::{IndexUid, NodeId, ShardId, SourceId, SubrequestId}; use tokio::sync::RwLock; use tracing::{error, info, warn}; diff --git a/quickwit/quickwit-ingest/src/ingest_v2/shard_table.rs b/quickwit/quickwit-ingest/src/ingest_v2/shard_table.rs index 3f5ba077c59..8109ff5388d 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/shard_table.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/shard_table.rs @@ -20,8 +20,7 @@ use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicUsize, Ordering}; -use quickwit_proto::control_plane::ClosedShards; -use quickwit_proto::ingest::{Shard, ShardState}; +use quickwit_proto::ingest::{ClosedShards, Shard, ShardState}; use quickwit_proto::types::{IndexId, IndexUid, NodeId, ShardId, SourceId}; use tracing::warn; diff --git a/quickwit/quickwit-proto/protos/quickwit/control_plane.proto b/quickwit/quickwit-proto/protos/quickwit/control_plane.proto index 32e9686c7e5..ed630375612 100644 --- a/quickwit/quickwit-proto/protos/quickwit/control_plane.proto +++ b/quickwit/quickwit-proto/protos/quickwit/control_plane.proto @@ -65,7 +65,7 @@ service ControlPlaneService { message GetOrCreateOpenShardsRequest { repeated GetOrCreateOpenShardsSubrequest subrequests = 1; - repeated ClosedShards closed_shards = 2; + repeated quickwit.ingest.ClosedShards closed_shards = 2; repeated string unavailable_leaders = 3; } @@ -75,12 +75,6 @@ message GetOrCreateOpenShardsSubrequest { string source_id = 3; } -message ClosedShards { - string index_uid = 1; - string source_id = 2; - repeated uint64 shard_ids = 3; -} - message GetOrCreateOpenShardsResponse { repeated GetOrCreateOpenShardsSuccess successes = 1; repeated GetOrCreateOpenShardsFailure failures = 2; diff --git a/quickwit/quickwit-proto/protos/quickwit/ingest.proto b/quickwit/quickwit-proto/protos/quickwit/ingest.proto index e6b0288759f..e4653bd9b16 100644 --- a/quickwit/quickwit-proto/protos/quickwit/ingest.proto +++ b/quickwit/quickwit-proto/protos/quickwit/ingest.proto @@ -78,3 +78,9 @@ message Shard { // For instance, if an indexer goes rogue, eventually the control plane will detect it and assign the shard to another indexer, which will override the publish token. optional string publish_token = 10; } + +message ClosedShards { + string index_uid = 1; + string source_id = 2; + repeated uint64 shard_ids = 3; +} diff --git a/quickwit/quickwit-proto/protos/quickwit/ingester.proto b/quickwit/quickwit-proto/protos/quickwit/ingester.proto index c7dd2e9c6be..be924bc950b 100644 --- a/quickwit/quickwit-proto/protos/quickwit/ingester.proto +++ b/quickwit/quickwit-proto/protos/quickwit/ingester.proto @@ -35,6 +35,8 @@ service IngesterService { // rpc OpenWatchStream(OpenWatchStreamRequest) returns (stream WatchMessage); + rpc CloseShards(CloseShardsRequest) returns (CloseShardsResponse); + // Pings an ingester to check if it is ready to host shards and serve requests. rpc Ping(PingRequest) returns (PingResponse); @@ -186,6 +188,13 @@ message FetchResponseV2 { quickwit.ingest.Position to_position_inclusive = 6; } +message CloseShardsRequest { + repeated quickwit.ingest.ClosedShards closed_shards = 1; +} + +message CloseShardsResponse { +} + message PingRequest { string leader_id = 1; optional string follower_id = 2; diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs index 5e7c01caade..e276eb95832 100644 --- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs +++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.control_plane.rs @@ -5,7 +5,7 @@ pub struct GetOrCreateOpenShardsRequest { #[prost(message, repeated, tag = "1")] pub subrequests: ::prost::alloc::vec::Vec, #[prost(message, repeated, tag = "2")] - pub closed_shards: ::prost::alloc::vec::Vec, + pub closed_shards: ::prost::alloc::vec::Vec, #[prost(string, repeated, tag = "3")] pub unavailable_leaders: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } @@ -23,17 +23,6 @@ pub struct GetOrCreateOpenShardsSubrequest { #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct ClosedShards { - #[prost(string, tag = "1")] - pub index_uid: ::prost::alloc::string::String, - #[prost(string, tag = "2")] - pub source_id: ::prost::alloc::string::String, - #[prost(uint64, repeated, tag = "3")] - pub shard_ids: ::prost::alloc::vec::Vec, -} -#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] pub struct GetOrCreateOpenShardsResponse { #[prost(message, repeated, tag = "1")] pub successes: ::prost::alloc::vec::Vec, diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.ingester.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.ingester.rs index 8ceb73fc7ee..26322f37715 100644 --- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.ingester.rs +++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.ingester.rs @@ -263,6 +263,17 @@ pub struct FetchResponseV2 { #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct CloseShardsRequest { + #[prost(message, repeated, tag = "1")] + pub closed_shards: ::prost::alloc::vec::Vec, +} +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CloseShardsResponse {} +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct PingRequest { #[prost(string, tag = "1")] pub leader_id: ::prost::alloc::string::String, @@ -334,6 +345,10 @@ pub trait IngesterService: std::fmt::Debug + dyn_clone::DynClone + Send + Sync + &mut self, request: OpenFetchStreamRequest, ) -> crate::ingest::IngestV2Result>; + async fn close_shards( + &mut self, + request: CloseShardsRequest, + ) -> crate::ingest::IngestV2Result; /// Pings an ingester to check if it is ready to host shards and serve requests. async fn ping( &mut self, @@ -442,6 +457,12 @@ impl IngesterService for IngesterServiceClient { ) -> crate::ingest::IngestV2Result> { self.inner.open_fetch_stream(request).await } + async fn close_shards( + &mut self, + request: CloseShardsRequest, + ) -> crate::ingest::IngestV2Result { + self.inner.close_shards(request).await + } async fn ping( &mut self, request: PingRequest, @@ -486,6 +507,12 @@ pub mod ingester_service_mock { > { self.inner.lock().await.open_fetch_stream(request).await } + async fn close_shards( + &mut self, + request: super::CloseShardsRequest, + ) -> crate::ingest::IngestV2Result { + self.inner.lock().await.close_shards(request).await + } async fn ping( &mut self, request: super::PingRequest, @@ -563,6 +590,22 @@ impl tower::Service for Box { Box::pin(fut) } } +impl tower::Service for Box { + type Response = CloseShardsResponse; + type Error = crate::ingest::IngestV2Error; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + fn call(&mut self, request: CloseShardsRequest) -> Self::Future { + let mut svc = self.clone(); + let fut = async move { svc.close_shards(request).await }; + Box::pin(fut) + } +} impl tower::Service for Box { type Response = PingResponse; type Error = crate::ingest::IngestV2Error; @@ -614,6 +657,11 @@ struct IngesterServiceTowerBlock { IngesterServiceStream, crate::ingest::IngestV2Error, >, + close_shards_svc: quickwit_common::tower::BoxService< + CloseShardsRequest, + CloseShardsResponse, + crate::ingest::IngestV2Error, + >, ping_svc: quickwit_common::tower::BoxService< PingRequest, PingResponse, @@ -632,6 +680,7 @@ impl Clone for IngesterServiceTowerBlock { persist_svc: self.persist_svc.clone(), open_replication_stream_svc: self.open_replication_stream_svc.clone(), open_fetch_stream_svc: self.open_fetch_stream_svc.clone(), + close_shards_svc: self.close_shards_svc.clone(), ping_svc: self.ping_svc.clone(), truncate_svc: self.truncate_svc.clone(), } @@ -657,6 +706,12 @@ impl IngesterService for IngesterServiceTowerBlock { ) -> crate::ingest::IngestV2Result> { self.open_fetch_stream_svc.ready().await?.call(request).await } + async fn close_shards( + &mut self, + request: CloseShardsRequest, + ) -> crate::ingest::IngestV2Result { + self.close_shards_svc.ready().await?.call(request).await + } async fn ping( &mut self, request: PingRequest, @@ -700,6 +755,15 @@ pub struct IngesterServiceTowerBlockBuilder { >, >, #[allow(clippy::type_complexity)] + close_shards_layer: Option< + quickwit_common::tower::BoxLayer< + Box, + CloseShardsRequest, + CloseShardsResponse, + crate::ingest::IngestV2Error, + >, + >, + #[allow(clippy::type_complexity)] ping_layer: Option< quickwit_common::tower::BoxLayer< Box, @@ -742,6 +806,12 @@ impl IngesterServiceTowerBlockBuilder { Error = crate::ingest::IngestV2Error, > + Clone + Send + Sync + 'static, >::Future: Send + 'static, + L::Service: tower::Service< + CloseShardsRequest, + Response = CloseShardsResponse, + Error = crate::ingest::IngestV2Error, + > + Clone + Send + Sync + 'static, + >::Future: Send + 'static, L::Service: tower::Service< PingRequest, Response = PingResponse, @@ -764,6 +834,10 @@ impl IngesterServiceTowerBlockBuilder { .open_fetch_stream_layer = Some( quickwit_common::tower::BoxLayer::new(layer.clone()), ); + self + .close_shards_layer = Some( + quickwit_common::tower::BoxLayer::new(layer.clone()), + ); self.ping_layer = Some(quickwit_common::tower::BoxLayer::new(layer.clone())); self.truncate_layer = Some(quickwit_common::tower::BoxLayer::new(layer)); self @@ -815,6 +889,19 @@ impl IngesterServiceTowerBlockBuilder { ); self } + pub fn close_shards_layer(mut self, layer: L) -> Self + where + L: tower::Layer> + Send + Sync + 'static, + L::Service: tower::Service< + CloseShardsRequest, + Response = CloseShardsResponse, + Error = crate::ingest::IngestV2Error, + > + Clone + Send + Sync + 'static, + >::Future: Send + 'static, + { + self.close_shards_layer = Some(quickwit_common::tower::BoxLayer::new(layer)); + self + } pub fn ping_layer(mut self, layer: L) -> Self where L: tower::Layer> + Send + Sync + 'static, @@ -895,6 +982,11 @@ impl IngesterServiceTowerBlockBuilder { } else { quickwit_common::tower::BoxService::new(boxed_instance.clone()) }; + let close_shards_svc = if let Some(layer) = self.close_shards_layer { + layer.layer(boxed_instance.clone()) + } else { + quickwit_common::tower::BoxService::new(boxed_instance.clone()) + }; let ping_svc = if let Some(layer) = self.ping_layer { layer.layer(boxed_instance.clone()) } else { @@ -910,6 +1002,7 @@ impl IngesterServiceTowerBlockBuilder { persist_svc, open_replication_stream_svc, open_fetch_stream_svc, + close_shards_svc, ping_svc, truncate_svc, }; @@ -1012,6 +1105,12 @@ where crate::ingest::IngestV2Error, >, > + + tower::Service< + CloseShardsRequest, + Response = CloseShardsResponse, + Error = crate::ingest::IngestV2Error, + Future = BoxFuture, + > + tower::Service< PingRequest, Response = PingResponse, @@ -1043,6 +1142,12 @@ where ) -> crate::ingest::IngestV2Result> { self.call(request).await } + async fn close_shards( + &mut self, + request: CloseShardsRequest, + ) -> crate::ingest::IngestV2Result { + self.call(request).await + } async fn ping( &mut self, request: PingRequest, @@ -1128,6 +1233,16 @@ where }) .map_err(|error| error.into()) } + async fn close_shards( + &mut self, + request: CloseShardsRequest, + ) -> crate::ingest::IngestV2Result { + self.inner + .close_shards(request) + .await + .map(|response| response.into_inner()) + .map_err(|error| error.into()) + } async fn ping( &mut self, request: PingRequest, @@ -1206,6 +1321,17 @@ for IngesterServiceGrpcServerAdapter { .map(|stream| tonic::Response::new(stream.map_err(|error| error.into()))) .map_err(|error| error.into()) } + async fn close_shards( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + self.inner + .clone() + .close_shards(request.into_inner()) + .await + .map(tonic::Response::new) + .map_err(|error| error.into()) + } async fn ping( &self, request: tonic::Request, @@ -1409,6 +1535,36 @@ pub mod ingester_service_grpc_client { ); self.inner.server_streaming(req, path, codec).await } + pub async fn close_shards( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/quickwit.ingest.ingester.IngesterService/CloseShards", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "quickwit.ingest.ingester.IngesterService", + "CloseShards", + ), + ); + self.inner.unary(req, path, codec).await + } /// Pings an ingester to check if it is ready to host shards and serve requests. pub async fn ping( &mut self, @@ -1507,6 +1663,13 @@ pub mod ingester_service_grpc_server { tonic::Response, tonic::Status, >; + async fn close_shards( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; /// Pings an ingester to check if it is ready to host shards and serve requests. async fn ping( &self, @@ -1741,6 +1904,52 @@ pub mod ingester_service_grpc_server { }; Box::pin(fut) } + "/quickwit.ingest.ingester.IngesterService/CloseShards" => { + #[allow(non_camel_case_types)] + struct CloseShardsSvc(pub Arc); + impl< + T: IngesterServiceGrpc, + > tonic::server::UnaryService + for CloseShardsSvc { + type Response = super::CloseShardsResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + (*inner).close_shards(request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = CloseShardsSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/quickwit.ingest.ingester.IngesterService/Ping" => { #[allow(non_camel_case_types)] struct PingSvc(pub Arc); diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.rs index e339aad99a0..6ad8e0e259d 100644 --- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.rs +++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.rs @@ -52,6 +52,17 @@ pub struct Shard { pub publish_token: ::core::option::Option<::prost::alloc::string::String>, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ClosedShards { + #[prost(string, tag = "1")] + pub index_uid: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub source_id: ::prost::alloc::string::String, + #[prost(uint64, repeated, tag = "3")] + pub shard_ids: ::prost::alloc::vec::Vec, +} +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] #[serde(rename_all = "snake_case")] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] diff --git a/quickwit/quickwit-proto/src/ingest/mod.rs b/quickwit/quickwit-proto/src/ingest/mod.rs index ef5f54ec368..2563cba0394 100644 --- a/quickwit/quickwit-proto/src/ingest/mod.rs +++ b/quickwit/quickwit-proto/src/ingest/mod.rs @@ -23,7 +23,7 @@ use self::ingester::FetchResponseV2; use super::types::NodeId; use super::{ServiceError, ServiceErrorCode}; use crate::control_plane::ControlPlaneError; -use crate::types::{queue_id, Position}; +use crate::types::{queue_id, Position, QueueId}; pub mod ingester; pub mod router; @@ -192,3 +192,11 @@ impl ShardState { *self == ShardState::Closed } } + +impl ClosedShards { + pub fn queue_ids(&self) -> impl Iterator + '_ { + self.shard_ids + .iter() + .map(|shard_id| queue_id(&self.index_uid, &self.source_id, *shard_id)) + } +} From 472d173807a0c4d2e08855e8fa2940ea86629afb Mon Sep 17 00:00:00 2001 From: Adrien Guillo Date: Fri, 10 Nov 2023 05:32:19 -0500 Subject: [PATCH 20/27] Bound memory and disk usage of mrecordlog (#4100) --- quickwit/Cargo.lock | 2 +- quickwit/Cargo.toml | 65 +++- .../quickwit-config/src/node_config/mod.rs | 4 +- quickwit/quickwit-ingest/Cargo.toml | 2 +- .../quickwit-ingest/src/ingest_v2/ingester.rs | 348 +++++++++++++++--- quickwit/quickwit-ingest/src/ingest_v2/mod.rs | 29 ++ .../quickwit-ingest/src/ingest_v2/mrecord.rs | 9 +- .../src/ingest_v2/mrecordlog_utils.rs | 159 ++++++++ .../src/ingest_v2/replication.rs | 225 ++++++++++- quickwit/quickwit-ingest/src/queue.rs | 4 +- .../protos/quickwit/ingester.proto | 10 +- .../quickwit/quickwit.ingest.ingester.rs | 41 ++- .../src/ingest_api/rest_handler.rs | 6 +- quickwit/quickwit-serve/src/lib.rs | 2 + 14 files changed, 823 insertions(+), 83 deletions(-) create mode 100644 quickwit/quickwit-ingest/src/ingest_v2/mrecordlog_utils.rs diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index a0a3bea8598..1fadf51bb17 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -3797,7 +3797,7 @@ dependencies = [ [[package]] name = "mrecordlog" version = "0.4.0" -source = "git+https://github.com/quickwit-oss/mrecordlog?rev=0d1a7aa#0d1a7aa2bdf11aec832919503cefaf282d70e0d8" +source = "git+https://github.com/quickwit-oss/mrecordlog?rev=ebee0fd#ebee0fdb6556a18f7cb73eca32e40bf44b0c4c6c" dependencies = [ "async-trait", "bytes", diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index 13f78cc72dd..c559f5fde78 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -50,7 +50,10 @@ bytes = { version = "1", features = ["serde"] } bytesize = { version = "1.3.0", features = ["serde"] } bytestring = "1.3.0" chitchat = "0.7" -chrono = { version = "0.4.23", default-features = false, features = ["clock", "std"] } +chrono = { version = "0.4.23", default-features = false, features = [ + "clock", + "std", +] } clap = { version = "4.4.1", features = ["env", "string"] } colored = "2.0.0" console-subscriber = "0.1.8" @@ -96,12 +99,19 @@ libz-sys = "1.1.8" lru = "0.12" lindera-core = "0.27.0" lindera-dictionary = "0.27.0" -lindera-tokenizer = { version = "0.27.0", features = ["ipadic", "ipadic-compress", "cc-cedict", "cc-cedict-compress", "ko-dic", "ko-dic-compress"] } +lindera-tokenizer = { version = "0.27.0", features = [ + "cc-cedict-compress", + "cc-cedict", + "ipadic-compress", + "ipadic", + "ko-dic-compress", + "ko-dic", +] } matches = "0.1.9" md5 = "0.7" mime_guess = "2.0.4" mockall = "0.11" -mrecordlog = { git = "https://github.com/quickwit-oss/mrecordlog", rev = "0d1a7aa" } +mrecordlog = { git = "https://github.com/quickwit-oss/mrecordlog", rev = "ebee0fd" } new_string_template = "1.4.0" nom = "7.1.3" num_cpus = "1" @@ -113,7 +123,9 @@ opentelemetry = { version = "0.19", features = ["rt-tokio"] } opentelemetry-otlp = "0.12.0" pin-project = "1.1.0" pnet = { version = "0.33.0", features = ["std"] } -postcard = { version = "1.0.4", features = ["use-std"], default-features = false} +postcard = { version = "1.0.4", features = [ + "use-std", +], default-features = false } predicates = "3" prettyplease = "0.2.0" proc-macro2 = "1.0.50" @@ -124,7 +136,11 @@ prost = { version = "0.11.6", default-features = false, features = [ ] } prost-build = "0.11.6" prost-types = "0.11.6" -pulsar = { git = "https://github.com/quickwit-oss/pulsar-rs.git", rev = "f9eff04", default-features = false, features = ["compression", "tokio-runtime", "auth-oauth2"] } +pulsar = { git = "https://github.com/quickwit-oss/pulsar-rs.git", rev = "f9eff04", default-features = false, features = [ + "auth-oauth2", + "compression", + "tokio-runtime", +] } quote = "1.0.23" rand = "0.8" rand_distr = "0.4" @@ -143,7 +159,10 @@ reqwest = { version = "0.11", default-features = false, features = [ ] } rust-embed = "6.8.1" sea-query = { version = "0" } -sea-query-binder = { version = "0", features = ["sqlx-postgres", "runtime-tokio-rustls",] } +sea-query-binder = { version = "0", features = [ + "runtime-tokio-rustls", + "sqlx-postgres", +] } # ^1.0.184 due to serde-rs/serde#2538 serde = { version = "1.0.184", features = ["derive", "rc"] } serde_json = "1.0" @@ -152,12 +171,12 @@ serde_with = "3.4.0" serde_yaml = "0.9" siphasher = "0.3" sqlx = { version = "0.7", features = [ - "runtime-tokio-rustls", - "postgres", "migrate", + "postgres", + "runtime-tokio-rustls", "time", ] } -syn = { version = "2.0.11", features = [ "extra-traits", "full", "parsing" ]} +syn = { version = "2.0.11", features = ["extra-traits", "full", "parsing"] } sync_wrapper = "0.1.2" tabled = { version = "0.8", features = ["color"] } tempfile = "3" @@ -173,14 +192,20 @@ tokio-util = { version = "0.7", features = ["full"] } toml = "0.7.6" tonic = { version = "0.9.0", features = ["gzip"] } tonic-build = "0.9.0" -tower = { version = "0.4.13", features = ["balance", "buffer", "load", "retry", "util"] } +tower = { version = "0.4.13", features = [ + "balance", + "buffer", + "load", + "retry", + "util", +] } tower-http = { version = "0.4.0", features = ["compression-gzip", "cors"] } tracing = "0.1.37" tracing-opentelemetry = "0.19.0" tracing-subscriber = { version = "0.3.16", features = [ - "time", - "std", "env-filter", + "std", + "time", ] } ttl_cache = "0.5" typetag = "0.2" @@ -199,7 +224,9 @@ whichlang = { git = "https://github.com/quickwit-oss/whichlang", rev = "fe406416 wiremock = "0.5" aws-config = "0.55.0" -aws-credential-types = { version = "0.55.0", features = ["hardcoded-credentials"] } +aws-credential-types = { version = "0.55.0", features = [ + "hardcoded-credentials", +] } aws-sdk-kinesis = "0.28.0" aws-sdk-s3 = "0.28.0" aws-smithy-async = "0.55.0" @@ -209,8 +236,12 @@ aws-smithy-types = "0.55.0" aws-types = "0.55.0" azure_core = { version = "0.13.0", features = ["enable_reqwest_rustls"] } -azure_storage = { version = "0.13.0", default-features = false, features = ["enable_reqwest_rustls"] } -azure_storage_blobs = { version = "0.13.0", default-features = false, features = ["enable_reqwest_rustls"] } +azure_storage = { version = "0.13.0", default-features = false, features = [ + "enable_reqwest_rustls", +] } +azure_storage_blobs = { version = "0.13.0", default-features = false, features = [ + "enable_reqwest_rustls", +] } quickwit-actors = { version = "0.6.3", path = "./quickwit-actors" } quickwit-aws = { version = "0.6.3", path = "./quickwit-aws" } @@ -242,10 +273,10 @@ quickwit-storage = { version = "0.6.3", path = "./quickwit-storage" } quickwit-telemetry = { version = "0.6.3", path = "./quickwit-telemetry" } tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "927b443", default-features = false, features = [ - "mmap", "lz4-compression", - "zstd-compression", + "mmap", "quickwit", + "zstd-compression", ] } # This is actually not used directly the goal is to fix the version diff --git a/quickwit/quickwit-config/src/node_config/mod.rs b/quickwit/quickwit-config/src/node_config/mod.rs index 53bc51d908b..c381d396b14 100644 --- a/quickwit/quickwit-config/src/node_config/mod.rs +++ b/quickwit/quickwit-config/src/node_config/mod.rs @@ -188,7 +188,7 @@ pub struct IngestApiConfig { pub max_queue_memory_usage: ByteSize, pub max_queue_disk_usage: ByteSize, pub replication_factor: usize, - pub content_length_limit: u64, + pub content_length_limit: ByteSize, } impl Default for IngestApiConfig { @@ -197,7 +197,7 @@ impl Default for IngestApiConfig { max_queue_memory_usage: ByteSize::gib(2), // TODO maybe we want more? max_queue_disk_usage: ByteSize::gib(4), // TODO maybe we want more? replication_factor: 1, - content_length_limit: ByteSize::mib(10).as_u64(), + content_length_limit: ByteSize::mib(10), } } } diff --git a/quickwit/quickwit-ingest/Cargo.toml b/quickwit/quickwit-ingest/Cargo.toml index 48d95f92664..99f2c3438d9 100644 --- a/quickwit/quickwit-ingest/Cargo.toml +++ b/quickwit/quickwit-ingest/Cargo.toml @@ -9,6 +9,7 @@ description = "Quickwit is a cost-efficient search engine." anyhow = { workspace = true } async-trait = { workspace = true } bytes = { workspace = true } +bytesize = { workspace = true } dyn-clone = { workspace = true } flume = { workspace = true } futures = { workspace = true } @@ -35,7 +36,6 @@ quickwit-config = { workspace = true } quickwit-proto = { workspace = true } [dev-dependencies] -bytesize = { workspace = true } itertools = { workspace = true } mockall = { workspace = true } rand = { workspace = true } diff --git a/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs b/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs index a92755d3c2f..5d2465a5f01 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs @@ -26,6 +26,7 @@ use std::sync::Arc; use std::time::Duration; use async_trait::async_trait; +use bytesize::ByteSize; use futures::stream::FuturesUnordered; use futures::StreamExt; use mrecordlog::error::{CreateQueueError, TruncateError}; @@ -37,17 +38,17 @@ use quickwit_proto::ingest::ingester::{ IngesterService, IngesterServiceClient, IngesterServiceStream, OpenFetchStreamRequest, OpenReplicationStreamRequest, OpenReplicationStreamResponse, PersistFailure, PersistFailureReason, PersistRequest, PersistResponse, PersistSuccess, PingRequest, - PingResponse, ReplicateRequest, ReplicateSubrequest, SynReplicationMessage, TruncateRequest, - TruncateResponse, + PingResponse, ReplicateFailureReason, ReplicateRequest, ReplicateSubrequest, + SynReplicationMessage, TruncateRequest, TruncateResponse, }; use quickwit_proto::ingest::{CommitTypeV2, IngestV2Error, IngestV2Result, ShardState}; use quickwit_proto::types::{NodeId, Position, QueueId}; use tokio::sync::RwLock; -use tracing::{error, info}; +use tracing::{error, info, warn}; use super::fetch::FetchTask; use super::models::{IngesterShard, PrimaryShard}; -use super::mrecord::{is_eof_mrecord, MRecord}; +use super::mrecordlog_utils::{append_eof_record_if_necessary, check_enough_capacity}; use super::replication::{ ReplicationStreamTask, ReplicationStreamTaskHandle, ReplicationTask, ReplicationTaskHandle, SYN_REPLICATION_STREAM_CAPACITY, @@ -55,7 +56,7 @@ use super::replication::{ use super::IngesterPool; use crate::ingest_v2::models::SoloShard; use crate::metrics::INGEST_METRICS; -use crate::{FollowerId, LeaderId}; +use crate::{estimate_size, FollowerId, LeaderId, MRecord}; /// Duration after which persist requests time out with /// [`quickwit_proto::ingest::IngestV2Error::Timeout`]. @@ -70,6 +71,8 @@ pub struct Ingester { self_node_id: NodeId, ingester_pool: IngesterPool, state: Arc>, + disk_capacity: ByteSize, + memory_capacity: ByteSize, replication_factor: usize, } @@ -95,6 +98,8 @@ impl Ingester { self_node_id: NodeId, ingester_pool: Pool, wal_dir_path: &Path, + disk_capacity: ByteSize, + memory_capacity: ByteSize, replication_factor: usize, ) -> IngestV2Result { let mrecordlog = MultiRecordLog::open_with_prefs( @@ -114,6 +119,8 @@ impl Ingester { self_node_id, ingester_pool, state: Arc::new(RwLock::new(inner)), + disk_capacity, + memory_capacity, replication_factor, }; info!( @@ -245,8 +252,6 @@ impl IngesterService for Ingester { self.self_node_id, persist_request.leader_id, ))); } - let mut state_guard = self.state.write().await; - let mut persist_successes = Vec::with_capacity(persist_request.subrequests.len()); let mut persist_failures = Vec::new(); let mut replicate_subrequests: HashMap> = HashMap::new(); @@ -255,6 +260,8 @@ impl IngesterService for Ingester { let force_commit = commit_type == CommitTypeV2::Force; let leader_id: NodeId = persist_request.leader_id.into(); + let mut state_guard = self.state.write().await; + for subrequest in persist_request.subrequests { let queue_id = subrequest.queue_id(); let follower_id_opt: Option = subrequest.follower_id.map(Into::into); @@ -283,10 +290,44 @@ impl IngesterService for Ingester { persist_failures.push(persist_failure); continue; } - let doc_batch = subrequest - .doc_batch - .expect("router should not send empty persist subrequests"); + let doc_batch = match subrequest.doc_batch { + Some(doc_batch) if !doc_batch.is_empty() => doc_batch, + _ => { + warn!("received empty persist request"); + + let persist_success = PersistSuccess { + subrequest_id: subrequest.subrequest_id, + index_uid: subrequest.index_uid, + source_id: subrequest.source_id, + shard_id: subrequest.shard_id, + replication_position_inclusive: Some( + shard.replication_position_inclusive(), + ), + }; + persist_successes.push(persist_success); + continue; + } + }; + let requested_capacity = estimate_size(&doc_batch); + if let Err(error) = check_enough_capacity( + &state_guard.mrecordlog, + self.disk_capacity, + self.memory_capacity, + requested_capacity, + ) { + warn!("failed to persist records: {error}"); + + let persist_failure = PersistFailure { + subrequest_id: subrequest.subrequest_id, + index_uid: subrequest.index_uid, + source_id: subrequest.source_id, + shard_id: subrequest.shard_id, + reason: PersistFailureReason::ResourceExhausted as i32, + }; + persist_failures.push(persist_failure); + continue; + } let current_position_inclusive: Position = if force_commit { let encoded_mrecords = doc_batch .docs() @@ -396,12 +437,31 @@ impl IngesterService for Ingester { }; persist_successes.push(persist_success); } + for replicate_failure in replicate_response.failures { + // TODO: If the replica shard is closed, close the primary shard if it is not + // already. + let persist_failure_reason = match replicate_failure.reason() { + ReplicateFailureReason::Unspecified => PersistFailureReason::Unspecified, + ReplicateFailureReason::ShardClosed => PersistFailureReason::ShardClosed, + ReplicateFailureReason::ResourceExhausted => { + PersistFailureReason::ResourceExhausted + } + }; + let persist_failure = PersistFailure { + subrequest_id: replicate_failure.subrequest_id, + index_uid: replicate_failure.index_uid, + source_id: replicate_failure.source_id, + shard_id: replicate_failure.shard_id, + reason: persist_failure_reason as i32, + }; + persist_failures.push(persist_failure); + } } let leader_id = self.self_node_id.to_string(); let persist_response = PersistResponse { leader_id, successes: persist_successes, - failures: Vec::new(), // TODO + failures: persist_failures, }; Ok(persist_response) } @@ -446,6 +506,8 @@ impl IngesterService for Ingester { self.state.clone(), syn_replication_stream, ack_replication_stream_tx, + self.disk_capacity, + self.memory_capacity, ); entry.insert(replication_task_handle); Ok(ack_replication_stream) @@ -513,7 +575,7 @@ impl IngesterService for Ingester { Position::Offset(offset) => offset.as_u64(), Position::Eof => state_guard .mrecordlog - .current_position(&queue_id) + .last_position(&queue_id) .ok() .flatten(), }; @@ -525,7 +587,7 @@ impl IngesterService for Ingester { { Ok(_) | Err(TruncateError::MissingQueue(_)) => {} Err(error) => { - error!("failed to truncate queue `{}`: {}", queue_id, error); + error!("failed to truncate queue `{queue_id}`: {error}"); } } } @@ -558,32 +620,6 @@ impl IngesterService for Ingester { } } -/// Appends an EOF record to the queue if the it is empty or the last record is not an EOF -/// record. -/// -/// # Panics -/// -/// Panics if the queue does not exist. -async fn append_eof_record_if_necessary(mrecordlog: &mut MultiRecordLog, queue_id: &QueueId) { - let mut should_append_eof_record = true; - - if let Some(current_position) = mrecordlog.current_position(queue_id).expect("TODO") { - let mrecords = mrecordlog - .range(queue_id, current_position..) - .expect("TODO"); - - if let Some((_, last_mecord)) = mrecords.last() { - should_append_eof_record = !is_eof_mrecord(&last_mecord); - } - } - if should_append_eof_record { - mrecordlog - .append_record(queue_id, None, MRecord::Eof.encode()) - .await - .expect("TODO"); - } -} - #[cfg(test)] mod tests { use std::net::SocketAddr; @@ -599,6 +635,7 @@ mod tests { use super::*; use crate::ingest_v2::fetch::FetchRange; + use crate::ingest_v2::mrecord::is_eof_mrecord; use crate::ingest_v2::test_utils::{IngesterShardTestExt, MultiRecordLogTestExt}; #[tokio::test] @@ -607,11 +644,15 @@ mod tests { let self_node_id: NodeId = "test-ingester-0".into(); let ingester_pool = IngesterPool::default(); let wal_dir_path = tempdir.path(); + let disk_capacity = ByteSize::mb(256); + let memory_capacity = ByteSize::mb(1); let replication_factor = 2; let mut ingester = Ingester::try_new( self_node_id.clone(), ingester_pool, wal_dir_path, + disk_capacity, + memory_capacity, replication_factor, ) .await @@ -708,11 +749,15 @@ mod tests { let self_node_id: NodeId = "test-ingester-0".into(); let ingester_pool = IngesterPool::default(); let wal_dir_path = tempdir.path(); + let disk_capacity = ByteSize::mb(256); + let memory_capacity = ByteSize::mb(1); let replication_factor = 1; let mut ingester = Ingester::try_new( self_node_id.clone(), ingester_pool, wal_dir_path, + disk_capacity, + memory_capacity, replication_factor, ) .await @@ -740,7 +785,30 @@ mod tests { }, ], }; - ingester.persist(persist_request).await.unwrap(); + let persist_response = ingester.persist(persist_request).await.unwrap(); + assert_eq!(persist_response.leader_id, "test-ingester-0"); + assert_eq!(persist_response.successes.len(), 2); + assert_eq!(persist_response.failures.len(), 0); + + let persist_success_0 = &persist_response.successes[0]; + assert_eq!(persist_success_0.subrequest_id, 0); + assert_eq!(persist_success_0.index_uid, "test-index:0"); + assert_eq!(persist_success_0.source_id, "test-source"); + assert_eq!(persist_success_0.shard_id, 1); + assert_eq!( + persist_success_0.replication_position_inclusive, + Some(Position::from(1u64)) + ); + + let persist_success_1 = &persist_response.successes[1]; + assert_eq!(persist_success_1.subrequest_id, 1); + assert_eq!(persist_success_1.index_uid, "test-index:1"); + assert_eq!(persist_success_1.source_id, "test-source"); + assert_eq!(persist_success_1.shard_id, 1); + assert_eq!( + persist_success_1.replication_position_inclusive, + Some(Position::from(2u64)) + ); let state_guard = ingester.state.read().await; assert_eq!(state_guard.shards.len(), 2); @@ -780,11 +848,15 @@ mod tests { let self_node_id: NodeId = "test-follower".into(); let ingester_pool = IngesterPool::default(); let wal_dir_path = tempdir.path(); + let disk_capacity = ByteSize::mb(256); + let memory_capacity = ByteSize::mb(1); let replication_factor = 1; let mut ingester = Ingester::try_new( self_node_id.clone(), ingester_pool, wal_dir_path, + disk_capacity, + memory_capacity, replication_factor, ) .await @@ -821,11 +893,17 @@ mod tests { let leader_id: NodeId = "test-leader".into(); let ingester_pool = IngesterPool::default(); let wal_dir_path = tempdir.path(); + + let disk_capacity = ByteSize::mb(256); + let memory_capacity = ByteSize::mb(1); let replication_factor = 2; + let mut leader = Ingester::try_new( leader_id.clone(), ingester_pool.clone(), wal_dir_path, + disk_capacity, + memory_capacity, replication_factor, ) .await @@ -834,11 +912,13 @@ mod tests { let tempdir = tempfile::tempdir().unwrap(); let follower_id: NodeId = "test-follower".into(); let wal_dir_path = tempdir.path(); - let replication_factor = 2; + let follower = Ingester::try_new( follower_id.clone(), ingester_pool.clone(), wal_dir_path, + disk_capacity, + memory_capacity, replication_factor, ) .await @@ -876,6 +956,26 @@ mod tests { assert_eq!(persist_response.successes.len(), 2); assert_eq!(persist_response.failures.len(), 0); + let persist_success_0 = &persist_response.successes[0]; + assert_eq!(persist_success_0.subrequest_id, 0); + assert_eq!(persist_success_0.index_uid, "test-index:0"); + assert_eq!(persist_success_0.source_id, "test-source"); + assert_eq!(persist_success_0.shard_id, 1); + assert_eq!( + persist_success_0.replication_position_inclusive, + Some(Position::from(1u64)) + ); + + let persist_success_1 = &persist_response.successes[1]; + assert_eq!(persist_success_1.subrequest_id, 1); + assert_eq!(persist_success_1.index_uid, "test-index:1"); + assert_eq!(persist_success_1.source_id, "test-source"); + assert_eq!(persist_success_1.shard_id, 1); + assert_eq!( + persist_success_1.replication_position_inclusive, + Some(Position::from(2u64)) + ); + let leader_state_guard = leader.state.read().await; assert_eq!(leader_state_guard.shards.len(), 2); @@ -943,11 +1043,15 @@ mod tests { let leader_id: NodeId = "test-leader".into(); let ingester_pool = IngesterPool::default(); let wal_dir_path = tempdir.path(); + let disk_capacity = ByteSize::mb(256); + let memory_capacity = ByteSize::mb(1); let replication_factor = 2; let mut leader = Ingester::try_new( leader_id.clone(), ingester_pool.clone(), wal_dir_path, + disk_capacity, + memory_capacity, replication_factor, ) .await @@ -970,11 +1074,15 @@ mod tests { let tempdir = tempfile::tempdir().unwrap(); let follower_id: NodeId = "test-follower".into(); let wal_dir_path = tempdir.path(); + let disk_capacity = ByteSize::mb(256); + let memory_capacity = ByteSize::mb(1); let replication_factor = 2; let follower = Ingester::try_new( follower_id.clone(), ingester_pool.clone(), wal_dir_path, + disk_capacity, + memory_capacity, replication_factor, ) .await @@ -1028,6 +1136,26 @@ mod tests { assert_eq!(persist_response.successes.len(), 2); assert_eq!(persist_response.failures.len(), 0); + let persist_success_0 = &persist_response.successes[0]; + assert_eq!(persist_success_0.subrequest_id, 0); + assert_eq!(persist_success_0.index_uid, "test-index:0"); + assert_eq!(persist_success_0.source_id, "test-source"); + assert_eq!(persist_success_0.shard_id, 1); + assert_eq!( + persist_success_0.replication_position_inclusive, + Some(Position::from(0u64)) + ); + + let persist_success_1 = &persist_response.successes[1]; + assert_eq!(persist_success_1.subrequest_id, 1); + assert_eq!(persist_success_1.index_uid, "test-index:1"); + assert_eq!(persist_success_1.source_id, "test-source"); + assert_eq!(persist_success_1.shard_id, 1); + assert_eq!( + persist_success_1.replication_position_inclusive, + Some(Position::from(1u64)) + ); + let leader_state_guard = leader.state.read().await; assert_eq!(leader_state_guard.shards.len(), 2); @@ -1081,17 +1209,147 @@ mod tests { ); } + #[tokio::test] + async fn test_ingester_persist_shard_closed() { + let tempdir = tempfile::tempdir().unwrap(); + let self_node_id: NodeId = "test-ingester-0".into(); + let ingester_pool = IngesterPool::default(); + let wal_dir_path = tempdir.path(); + let disk_capacity = ByteSize::mib(256); + let memory_capacity = ByteSize::mib(1); + let replication_factor = 1; + let mut ingester = Ingester::try_new( + self_node_id.clone(), + ingester_pool, + wal_dir_path, + disk_capacity, + memory_capacity, + replication_factor, + ) + .await + .unwrap(); + + let queue_id_01 = queue_id("test-index:0", "test-source", 1); + + let solo_shard = SoloShard::new(ShardState::Closed, Position::Beginning); + let shard = IngesterShard::Solo(solo_shard); + + ingester + .state + .write() + .await + .shards + .insert(queue_id_01.clone(), shard); + + let persist_request = PersistRequest { + leader_id: self_node_id.to_string(), + commit_type: CommitTypeV2::Force as i32, + subrequests: vec![PersistSubrequest { + subrequest_id: 0, + index_uid: "test-index:0".to_string(), + source_id: "test-source".to_string(), + shard_id: 1, + follower_id: None, + doc_batch: Some(DocBatchV2::for_test(["test-doc-010"])), + }], + }; + let persist_response = ingester.persist(persist_request).await.unwrap(); + assert_eq!(persist_response.leader_id, "test-ingester-0"); + assert_eq!(persist_response.successes.len(), 0); + assert_eq!(persist_response.failures.len(), 1); + + let persist_failure = &persist_response.failures[0]; + assert_eq!(persist_failure.subrequest_id, 0); + assert_eq!(persist_failure.index_uid, "test-index:0"); + assert_eq!(persist_failure.source_id, "test-source"); + assert_eq!(persist_failure.shard_id, 1); + assert_eq!(persist_failure.reason(), PersistFailureReason::ShardClosed); + + let state_guard = ingester.state.read().await; + assert_eq!(state_guard.shards.len(), 1); + + let solo_shard_01 = state_guard.shards.get(&queue_id_01).unwrap(); + solo_shard_01.assert_is_solo(); + solo_shard_01.assert_is_closed(); + solo_shard_01.assert_replication_position(Position::Beginning); + } + + #[tokio::test] + async fn test_ingester_persist_resource_exhausted() { + let tempdir = tempfile::tempdir().unwrap(); + let self_node_id: NodeId = "test-ingester-0".into(); + let ingester_pool = IngesterPool::default(); + let wal_dir_path = tempdir.path(); + let disk_capacity = ByteSize(0); + let memory_capacity = ByteSize(0); + let replication_factor = 1; + let mut ingester = Ingester::try_new( + self_node_id.clone(), + ingester_pool, + wal_dir_path, + disk_capacity, + memory_capacity, + replication_factor, + ) + .await + .unwrap(); + + let persist_request = PersistRequest { + leader_id: self_node_id.to_string(), + commit_type: CommitTypeV2::Force as i32, + subrequests: vec![PersistSubrequest { + subrequest_id: 0, + index_uid: "test-index:0".to_string(), + source_id: "test-source".to_string(), + shard_id: 1, + follower_id: None, + doc_batch: Some(DocBatchV2::for_test(["test-doc-010"])), + }], + }; + let persist_response = ingester.persist(persist_request).await.unwrap(); + assert_eq!(persist_response.leader_id, "test-ingester-0"); + assert_eq!(persist_response.successes.len(), 0); + assert_eq!(persist_response.failures.len(), 1); + + let persist_failure = &persist_response.failures[0]; + assert_eq!(persist_failure.subrequest_id, 0); + assert_eq!(persist_failure.index_uid, "test-index:0"); + assert_eq!(persist_failure.source_id, "test-source"); + assert_eq!(persist_failure.shard_id, 1); + assert_eq!( + persist_failure.reason(), + PersistFailureReason::ResourceExhausted + ); + + let state_guard = ingester.state.read().await; + assert_eq!(state_guard.shards.len(), 1); + + let queue_id_01 = queue_id("test-index:0", "test-source", 1); + let solo_shard_01 = state_guard.shards.get(&queue_id_01).unwrap(); + solo_shard_01.assert_is_solo(); + solo_shard_01.assert_is_open(); + solo_shard_01.assert_replication_position(Position::Beginning); + + state_guard + .mrecordlog + .assert_records_eq(&queue_id_01, .., &[]); + } + #[tokio::test] async fn test_ingester_open_fetch_stream() { let tempdir = tempfile::tempdir().unwrap(); let self_node_id: NodeId = "test-ingester-0".into(); let ingester_pool = IngesterPool::default(); let wal_dir_path = tempdir.path(); + let disk_capacity = ByteSize::mb(256); + let memory_capacity = ByteSize::mb(1); let replication_factor = 1; let mut ingester = Ingester::try_new( self_node_id.clone(), ingester_pool, wal_dir_path, + disk_capacity, + memory_capacity, replication_factor, ) .await @@ -1185,11 +1443,15 @@ mod tests { let self_node_id: NodeId = "test-ingester-0".into(); let ingester_pool = IngesterPool::default(); let wal_dir_path = tempdir.path(); + let disk_capacity = ByteSize::mb(256); + let memory_capacity = ByteSize::mb(1); let replication_factor = 1; let mut ingester = Ingester::try_new( self_node_id.clone(), ingester_pool, wal_dir_path, + disk_capacity, + memory_capacity, replication_factor, ) .await @@ -1275,11 +1537,15 @@ mod tests { let self_node_id: NodeId = "test-ingester-0".into(); let ingester_pool = IngesterPool::default(); let wal_dir_path = tempdir.path(); + let disk_capacity = ByteSize::mb(256); + let memory_capacity = ByteSize::mb(1); let replication_factor = 1; let mut ingester = Ingester::try_new( self_node_id.clone(), ingester_pool, wal_dir_path, + disk_capacity, + memory_capacity, replication_factor, ) .await diff --git a/quickwit/quickwit-ingest/src/ingest_v2/mod.rs b/quickwit/quickwit-ingest/src/ingest_v2/mod.rs index 16e4317ab70..ec950ada275 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/mod.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/mod.rs @@ -21,6 +21,7 @@ mod fetch; mod ingester; mod models; mod mrecord; +mod mrecordlog_utils; mod replication; mod router; mod shard_table; @@ -28,12 +29,15 @@ mod shard_table; mod test_utils; mod workbench; +use bytesize::ByteSize; use quickwit_common::tower::Pool; use quickwit_proto::ingest::ingester::IngesterServiceClient; +use quickwit_proto::ingest::DocBatchV2; use quickwit_proto::types::NodeId; pub use self::fetch::{FetchStreamError, MultiFetchStream}; pub use self::ingester::Ingester; +use self::mrecord::MRECORD_HEADER_LEN; pub use self::mrecord::{decoded_mrecords, MRecord}; pub use self::router::IngestRouter; @@ -45,3 +49,28 @@ pub type ClientId = String; pub type LeaderId = NodeId; pub type FollowerId = NodeId; + +pub(super) fn estimate_size(doc_batch: &DocBatchV2) -> ByteSize { + let estimate = doc_batch.num_bytes() + doc_batch.num_docs() * MRECORD_HEADER_LEN; + ByteSize(estimate as u64) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_estimate_size() { + let doc_batch = DocBatchV2 { + doc_buffer: Vec::new().into(), + doc_lengths: Vec::new(), + }; + assert_eq!(estimate_size(&doc_batch), ByteSize(0)); + + let doc_batch = DocBatchV2 { + doc_buffer: vec![0u8; 100].into(), + doc_lengths: vec![10, 20, 30], + }; + assert_eq!(estimate_size(&doc_batch), ByteSize(106)); + } +} diff --git a/quickwit/quickwit-ingest/src/ingest_v2/mrecord.rs b/quickwit/quickwit-ingest/src/ingest_v2/mrecord.rs index 0d32ba4b52d..d73b7b83dc8 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/mrecord.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/mrecord.rs @@ -28,14 +28,17 @@ pub enum HeaderVersion { V0 = 0, } +/// Length of the header of a [`MRecord`] in bytes. +pub(super) const MRECORD_HEADER_LEN: usize = 2; + /// `Doc` header v0 composed of the header version and the `Doc = 0` record type. -const DOC_HEADER_V0: &[u8; 2] = &[HeaderVersion::V0 as u8, 0]; +const DOC_HEADER_V0: &[u8; MRECORD_HEADER_LEN] = &[HeaderVersion::V0 as u8, 0]; /// `Commit` header v0 composed of the header version and the `Commit = 1` record type. -const COMMIT_HEADER_V0: &[u8; 2] = &[HeaderVersion::V0 as u8, 1]; +const COMMIT_HEADER_V0: &[u8; MRECORD_HEADER_LEN] = &[HeaderVersion::V0 as u8, 1]; /// `Eof` header v0 composed of the header version and the `Eof = 2` record type. -const EOF_HEADER_V0: &[u8; 2] = &[HeaderVersion::V0 as u8, 2]; +const EOF_HEADER_V0: &[u8; MRECORD_HEADER_LEN] = &[HeaderVersion::V0 as u8, 2]; #[derive(Debug, Clone, Eq, PartialEq)] pub enum MRecord { diff --git a/quickwit/quickwit-ingest/src/ingest_v2/mrecordlog_utils.rs b/quickwit/quickwit-ingest/src/ingest_v2/mrecordlog_utils.rs new file mode 100644 index 00000000000..15cf05a0465 --- /dev/null +++ b/quickwit/quickwit-ingest/src/ingest_v2/mrecordlog_utils.rs @@ -0,0 +1,159 @@ +// Copyright (C) 2023 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use bytesize::ByteSize; +use mrecordlog::error::{AppendError, MissingQueue}; +use mrecordlog::MultiRecordLog; +use quickwit_proto::types::QueueId; +use tracing::warn; + +use super::mrecord::is_eof_mrecord; +use crate::MRecord; + +/// Appends an EOF record to the queue if it is empty or the last record is not an EOF +/// record. +pub(super) async fn append_eof_record_if_necessary( + mrecordlog: &mut MultiRecordLog, + queue_id: &QueueId, +) { + let should_append_eof_record = match mrecordlog.last_record(queue_id) { + Ok(Some((_, last_mrecord))) => !is_eof_mrecord(&last_mrecord), + Ok(None) => true, + Err(MissingQueue(_)) => { + warn!("failed to append EOF record to queue `{queue_id}`: queue does not exist"); + return; + } + }; + if should_append_eof_record { + match mrecordlog + .append_record(queue_id, None, MRecord::Eof.encode()) + .await + { + Ok(_) | Err(AppendError::MissingQueue(_)) => {} + Err(error) => { + warn!("failed to append EOF record to queue `{queue_id}`: {error}"); + } + } + } +} + +/// Error returned when the mrecordlog does not have enough capacity to store some records. +#[derive(Debug, Clone, Copy, thiserror::Error)] +pub(super) enum NotEnoughCapacityError { + #[error( + "write-ahead log is full, capacity: usage: {usage}, capacity: {capacity}, requested: \ + {requested}" + )] + Disk { + usage: ByteSize, + capacity: ByteSize, + requested: ByteSize, + }, + #[error( + "write-ahead log memory buffer is full, usage: {usage}, capacity: {capacity}, requested: \ + {requested}" + )] + Memory { + usage: ByteSize, + capacity: ByteSize, + requested: ByteSize, + }, +} + +/// Checks whether the log has enough capacity to store some records. +pub(super) fn check_enough_capacity( + mrecordlog: &MultiRecordLog, + disk_capacity: ByteSize, + memory_capacity: ByteSize, + requested_capacity: ByteSize, +) -> Result<(), NotEnoughCapacityError> { + let disk_usage = ByteSize(mrecordlog.disk_usage() as u64); + + if disk_usage + requested_capacity > disk_capacity { + return Err(NotEnoughCapacityError::Disk { + usage: disk_usage, + capacity: disk_capacity, + requested: requested_capacity, + }); + } + let memory_usage = ByteSize(mrecordlog.memory_usage() as u64); + + if memory_usage + requested_capacity > memory_capacity { + return Err(NotEnoughCapacityError::Memory { + usage: memory_usage, + capacity: memory_capacity, + requested: requested_capacity, + }); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_append_eof_record_if_necessary() { + let tempdir = tempfile::tempdir().unwrap(); + let mut mrecordlog = MultiRecordLog::open(tempdir.path()).await.unwrap(); + + append_eof_record_if_necessary(&mut mrecordlog, &"queue-not-found".to_string()).await; + + mrecordlog.create_queue("test-queue").await.unwrap(); + append_eof_record_if_necessary(&mut mrecordlog, &"test-queue".to_string()).await; + + let (last_position, last_record) = mrecordlog.last_record("test-queue").unwrap().unwrap(); + assert_eq!(last_position, 0); + assert!(is_eof_mrecord(&last_record)); + + append_eof_record_if_necessary(&mut mrecordlog, &"test-queue".to_string()).await; + let (last_position, last_record) = mrecordlog.last_record("test-queue").unwrap().unwrap(); + assert_eq!(last_position, 0); + assert!(is_eof_mrecord(&last_record)); + + mrecordlog.truncate("test-queue", 0).await.unwrap(); + + append_eof_record_if_necessary(&mut mrecordlog, &"test-queue".to_string()).await; + let (last_position, last_record) = mrecordlog.last_record("test-queue").unwrap().unwrap(); + assert_eq!(last_position, 1); + assert!(is_eof_mrecord(&last_record)); + } + + #[tokio::test] + async fn test_check_enough_capacity() { + let tempdir = tempfile::tempdir().unwrap(); + let mrecordlog = MultiRecordLog::open(tempdir.path()).await.unwrap(); + + let disk_error = + check_enough_capacity(&mrecordlog, ByteSize(0), ByteSize(0), ByteSize(12)).unwrap_err(); + + assert!(matches!(disk_error, NotEnoughCapacityError::Disk { .. })); + + let memory_error = + check_enough_capacity(&mrecordlog, ByteSize::mb(256), ByteSize(11), ByteSize(12)) + .unwrap_err(); + + assert!(matches!( + memory_error, + NotEnoughCapacityError::Memory { .. } + )); + + check_enough_capacity(&mrecordlog, ByteSize::mb(256), ByteSize(12), ByteSize(12)).unwrap(); + } +} diff --git a/quickwit/quickwit-ingest/src/ingest_v2/replication.rs b/quickwit/quickwit-ingest/src/ingest_v2/replication.rs index 1feb4a5e1a0..67d19cdb7b1 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/replication.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/replication.rs @@ -22,22 +22,26 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::Duration; +use bytesize::ByteSize; use futures::{Future, StreamExt}; use quickwit_common::ServiceStream; use quickwit_proto::ingest::ingester::{ - ack_replication_message, syn_replication_message, AckReplicationMessage, ReplicateRequest, - ReplicateResponse, ReplicateSuccess, SynReplicationMessage, + ack_replication_message, syn_replication_message, AckReplicationMessage, ReplicateFailure, + ReplicateFailureReason, ReplicateRequest, ReplicateResponse, ReplicateSuccess, + SynReplicationMessage, }; use quickwit_proto::ingest::{CommitTypeV2, IngestV2Error, IngestV2Result}; use quickwit_proto::types::{NodeId, Position}; use tokio::sync::mpsc::error::TryRecvError; use tokio::sync::{mpsc, oneshot, RwLock}; use tokio::task::JoinHandle; -use tracing::error; +use tracing::{error, warn}; use super::ingester::IngesterState; use super::models::{IngesterShard, ReplicaShard}; use super::mrecord::MRecord; +use super::mrecordlog_utils::check_enough_capacity; +use crate::estimate_size; use crate::metrics::INGEST_METRICS; pub(super) const SYN_REPLICATION_STREAM_CAPACITY: usize = 5; @@ -265,6 +269,8 @@ pub(super) struct ReplicationTask { syn_replication_stream: ServiceStream, ack_replication_stream_tx: mpsc::UnboundedSender>, current_replication_seqno: ReplicationSeqNo, + disk_capacity: ByteSize, + memory_capacity: ByteSize, } impl ReplicationTask { @@ -274,6 +280,8 @@ impl ReplicationTask { state: Arc>, syn_replication_stream: ServiceStream, ack_replication_stream_tx: mpsc::UnboundedSender>, + disk_capacity: ByteSize, + memory_capacity: ByteSize, ) -> ReplicationTaskHandle { let mut replication_task = Self { leader_id, @@ -282,6 +290,8 @@ impl ReplicationTask { syn_replication_stream, ack_replication_stream_tx, current_replication_seqno: 0, + disk_capacity, + memory_capacity, }; let join_handle = tokio::spawn(async move { replication_task.run().await }); ReplicationTaskHandle { join_handle } @@ -314,7 +324,9 @@ impl ReplicationTask { let commit_type = replicate_request.commit_type(); let force_commit = commit_type == CommitTypeV2::Force; + let mut replicate_successes = Vec::with_capacity(replicate_request.subrequests.len()); + let mut replicate_failures = Vec::new(); let mut state_guard = self.state.write().await; @@ -341,15 +353,57 @@ impl ReplicationTask { .expect("replica shard should be initialized") }; if replica_shard.is_closed() { - // TODO + let replicate_failure = ReplicateFailure { + subrequest_id: subrequest.subrequest_id, + index_uid: subrequest.index_uid, + source_id: subrequest.source_id, + shard_id: subrequest.shard_id, + reason: ReplicateFailureReason::ShardClosed as i32, + }; + replicate_failures.push(replicate_failure); + continue; } if replica_shard.replication_position_inclusive() != from_position_exclusive { // TODO } - let doc_batch = subrequest - .doc_batch - .expect("leader should not send empty replicate subrequests"); + let doc_batch = match subrequest.doc_batch { + Some(doc_batch) if !doc_batch.is_empty() => doc_batch, + _ => { + warn!("received empty replicate request"); + let replicate_success = ReplicateSuccess { + subrequest_id: subrequest.subrequest_id, + index_uid: subrequest.index_uid, + source_id: subrequest.source_id, + shard_id: subrequest.shard_id, + replication_position_inclusive: Some( + replica_shard.replication_position_inclusive(), + ), + }; + replicate_successes.push(replicate_success); + continue; + } + }; + let requested_capacity = estimate_size(&doc_batch); + + if let Err(error) = check_enough_capacity( + &state_guard.mrecordlog, + self.disk_capacity, + self.memory_capacity, + requested_capacity, + ) { + warn!("failed to replicate records: {error}"); + + let replicate_failure = ReplicateFailure { + subrequest_id: subrequest.subrequest_id, + index_uid: subrequest.index_uid, + source_id: subrequest.source_id, + shard_id: subrequest.shard_id, + reason: ReplicateFailureReason::ResourceExhausted as i32, + }; + replicate_failures.push(replicate_failure); + continue; + } let current_position_inclusive: Position = if force_commit { let encoded_mrecords = doc_batch .docs() @@ -404,7 +458,7 @@ impl ReplicationTask { let replicate_response = ReplicateResponse { follower_id, successes: replicate_successes, - failures: Vec::new(), + failures: replicate_failures, replication_seqno: replicate_request.replication_seqno, }; Ok(replicate_response) @@ -463,7 +517,7 @@ mod tests { use mrecordlog::MultiRecordLog; use quickwit_proto::ingest::ingester::{ReplicateSubrequest, ReplicateSuccess}; - use quickwit_proto::ingest::DocBatchV2; + use quickwit_proto::ingest::{DocBatchV2, ShardState}; use quickwit_proto::types::queue_id; use super::*; @@ -627,12 +681,18 @@ mod tests { ServiceStream::new_bounded(SYN_REPLICATION_STREAM_CAPACITY); let (ack_replication_stream_tx, mut ack_replication_stream) = ServiceStream::new_unbounded(); + + let disk_capacity = ByteSize::mb(256); + let memory_capacity = ByteSize::mb(1); + let _replication_task_handle = ReplicationTask::spawn( leader_id, follower_id, state.clone(), syn_replication_stream, ack_replication_stream_tx, + disk_capacity, + memory_capacity, ); let replicate_request = ReplicateRequest { leader_id: "test-leader".to_string(), @@ -767,4 +827,151 @@ mod tests { &[(0, "\0\0test-doc-foo"), (1, "\0\0test-doc-moo")], ); } + + #[tokio::test] + async fn test_replication_task_shard_closed() { + let leader_id: NodeId = "test-leader".into(); + let follower_id: NodeId = "test-follower".into(); + let tempdir = tempfile::tempdir().unwrap(); + let mrecordlog = MultiRecordLog::open(tempdir.path()).await.unwrap(); + let state = Arc::new(RwLock::new(IngesterState { + mrecordlog, + shards: HashMap::new(), + replication_streams: HashMap::new(), + replication_tasks: HashMap::new(), + })); + let (syn_replication_stream_tx, syn_replication_stream) = + ServiceStream::new_bounded(SYN_REPLICATION_STREAM_CAPACITY); + let (ack_replication_stream_tx, mut ack_replication_stream) = + ServiceStream::new_unbounded(); + + let disk_capacity = ByteSize::mb(256); + let memory_capacity = ByteSize::mb(1); + + let _replication_task_handle = ReplicationTask::spawn( + leader_id.clone(), + follower_id, + state.clone(), + syn_replication_stream, + ack_replication_stream_tx, + disk_capacity, + memory_capacity, + ); + + let queue_id_01 = queue_id("test-index:0", "test-source", 1); + + let mut replica_shard = ReplicaShard::new(leader_id); + replica_shard.shard_state = ShardState::Closed; + let shard = IngesterShard::Replica(replica_shard); + + state + .write() + .await + .shards + .insert(queue_id_01.clone(), shard); + + let replicate_request = ReplicateRequest { + leader_id: "test-leader".to_string(), + follower_id: "test-follower".to_string(), + commit_type: CommitTypeV2::Auto as i32, + subrequests: vec![ReplicateSubrequest { + subrequest_id: 0, + index_uid: "test-index:0".to_string(), + source_id: "test-source".to_string(), + shard_id: 1, + doc_batch: Some(DocBatchV2::for_test(["test-doc-foo"])), + from_position_exclusive: Position::from(0u64).into(), + to_position_inclusive: Some(Position::from(1u64)), + }], + replication_seqno: 0, + }; + let syn_replication_message = + SynReplicationMessage::new_replicate_request(replicate_request); + syn_replication_stream_tx + .send(syn_replication_message) + .await + .unwrap(); + let ack_replication_message = ack_replication_stream.next().await.unwrap().unwrap(); + let replicate_response = into_replicate_response(ack_replication_message); + + assert_eq!(replicate_response.follower_id, "test-follower"); + assert_eq!(replicate_response.successes.len(), 0); + assert_eq!(replicate_response.failures.len(), 1); + + let replicate_failure = &replicate_response.failures[0]; + assert_eq!(replicate_failure.index_uid, "test-index:0"); + assert_eq!(replicate_failure.source_id, "test-source"); + assert_eq!(replicate_failure.shard_id, 1); + assert_eq!( + replicate_failure.reason(), + ReplicateFailureReason::ShardClosed + ); + } + + #[tokio::test] + async fn test_replication_task_resource_exhausted() { + let leader_id: NodeId = "test-leader".into(); + let follower_id: NodeId = "test-follower".into(); + let tempdir = tempfile::tempdir().unwrap(); + let mrecordlog = MultiRecordLog::open(tempdir.path()).await.unwrap(); + let state = Arc::new(RwLock::new(IngesterState { + mrecordlog, + shards: HashMap::new(), + replication_streams: HashMap::new(), + replication_tasks: HashMap::new(), + })); + let (syn_replication_stream_tx, syn_replication_stream) = + ServiceStream::new_bounded(SYN_REPLICATION_STREAM_CAPACITY); + let (ack_replication_stream_tx, mut ack_replication_stream) = + ServiceStream::new_unbounded(); + + let disk_capacity = ByteSize(0); + let memory_capacity = ByteSize(0); + + let _replication_task_handle = ReplicationTask::spawn( + leader_id, + follower_id, + state.clone(), + syn_replication_stream, + ack_replication_stream_tx, + disk_capacity, + memory_capacity, + ); + let replicate_request = ReplicateRequest { + leader_id: "test-leader".to_string(), + follower_id: "test-follower".to_string(), + commit_type: CommitTypeV2::Auto as i32, + subrequests: vec![ReplicateSubrequest { + subrequest_id: 0, + index_uid: "test-index:0".to_string(), + source_id: "test-source".to_string(), + shard_id: 1, + doc_batch: Some(DocBatchV2::for_test(["test-doc-foo"])), + from_position_exclusive: None, + to_position_inclusive: Some(Position::from(0u64)), + }], + replication_seqno: 0, + }; + let syn_replication_message = + SynReplicationMessage::new_replicate_request(replicate_request); + syn_replication_stream_tx + .send(syn_replication_message) + .await + .unwrap(); + let ack_replication_message = ack_replication_stream.next().await.unwrap().unwrap(); + let replicate_response = into_replicate_response(ack_replication_message); + + assert_eq!(replicate_response.follower_id, "test-follower"); + assert_eq!(replicate_response.successes.len(), 0); + assert_eq!(replicate_response.failures.len(), 1); + + let replicate_failure_0 = &replicate_response.failures[0]; + assert_eq!(replicate_failure_0.index_uid, "test-index:0"); + assert_eq!(replicate_failure_0.source_id, "test-source"); + assert_eq!(replicate_failure_0.shard_id, 1); + assert_eq!( + replicate_failure_0.reason(), + ReplicateFailureReason::ResourceExhausted + ); + } } diff --git a/quickwit/quickwit-ingest/src/queue.rs b/quickwit/quickwit-ingest/src/queue.rs index df5e80e593c..cb3e7e9eacb 100644 --- a/quickwit/quickwit-ingest/src/queue.rs +++ b/quickwit/quickwit-ingest/src/queue.rs @@ -209,11 +209,11 @@ impl Queues { } pub(crate) fn disk_usage(&self) -> usize { - self.record_log.on_disk_size() + self.record_log.disk_usage() } pub(crate) fn memory_usage(&self) -> usize { - self.record_log.in_memory_size() + self.record_log.memory_usage() } } diff --git a/quickwit/quickwit-proto/protos/quickwit/ingester.proto b/quickwit/quickwit-proto/protos/quickwit/ingester.proto index be924bc950b..109888c60f8 100644 --- a/quickwit/quickwit-proto/protos/quickwit/ingester.proto +++ b/quickwit/quickwit-proto/protos/quickwit/ingester.proto @@ -145,13 +145,19 @@ message ReplicateSuccess { quickwit.ingest.Position replication_position_inclusive = 5; } +enum ReplicateFailureReason { + REPLICATE_FAILURE_REASON_UNSPECIFIED = 0; + REPLICATE_FAILURE_REASON_SHARD_CLOSED = 1; + reserved 2; // REPLICATE_FAILURE_REASON_RATE_LIMITED = 2; + REPLICATE_FAILURE_REASON_RESOURCE_EXHAUSTED = 3; +} + message ReplicateFailure { uint32 subrequest_id = 1; string index_uid = 2; string source_id = 3; uint64 shard_id = 4; - // ingest.DocBatchV2 doc_batch = 4; - // ingest.IngestError error = 5; + ReplicateFailureReason reason = 5; } message TruncateRequest { diff --git a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.ingester.rs b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.ingester.rs index 26322f37715..53857e8ac79 100644 --- a/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.ingester.rs +++ b/quickwit/quickwit-proto/src/codegen/quickwit/quickwit.ingest.ingester.rs @@ -194,10 +194,10 @@ pub struct ReplicateFailure { pub index_uid: ::prost::alloc::string::String, #[prost(string, tag = "3")] pub source_id: ::prost::alloc::string::String, - /// ingest.DocBatchV2 doc_batch = 4; - /// ingest.IngestError error = 5; #[prost(uint64, tag = "4")] pub shard_id: u64, + #[prost(enumeration = "ReplicateFailureReason", tag = "5")] + pub reason: i32, } #[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] #[allow(clippy::derive_partial_eq_without_eq)] @@ -320,6 +320,43 @@ impl PersistFailureReason { } } } +#[derive(serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ReplicateFailureReason { + Unspecified = 0, + ShardClosed = 1, + ResourceExhausted = 3, +} +impl ReplicateFailureReason { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + ReplicateFailureReason::Unspecified => "REPLICATE_FAILURE_REASON_UNSPECIFIED", + ReplicateFailureReason::ShardClosed => { + "REPLICATE_FAILURE_REASON_SHARD_CLOSED" + } + ReplicateFailureReason::ResourceExhausted => { + "REPLICATE_FAILURE_REASON_RESOURCE_EXHAUSTED" + } + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "REPLICATE_FAILURE_REASON_UNSPECIFIED" => Some(Self::Unspecified), + "REPLICATE_FAILURE_REASON_SHARD_CLOSED" => Some(Self::ShardClosed), + "REPLICATE_FAILURE_REASON_RESOURCE_EXHAUSTED" => { + Some(Self::ResourceExhausted) + } + _ => None, + } + } +} /// BEGIN quickwit-codegen #[allow(unused_imports)] use std::str::FromStr; diff --git a/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs b/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs index d698143d0ec..5a65411d262 100644 --- a/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs +++ b/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs @@ -79,7 +79,7 @@ fn ingest_filter( warp::path!(String / "ingest") .and(warp::post()) .and(warp::body::content_length_limit( - config.content_length_limit, + config.content_length_limit.as_u64(), )) .and(warp::body::bytes()) .and(serde_qs::warp::query::( @@ -103,7 +103,7 @@ fn ingest_v2_filter( warp::path!(String / "ingest-v2") .and(warp::post()) .and(warp::body::content_length_limit( - config.content_length_limit, + config.content_length_limit.as_u64(), )) .and(warp::body::bytes()) .and(serde_qs::warp::query::( @@ -355,7 +355,7 @@ pub(crate) mod tests { #[tokio::test] async fn test_ingest_api_return_413_if_above_content_limit() { let config = IngestApiConfig { - content_length_limit: 1, + content_length_limit: ByteSize(1), ..Default::default() }; let (universe, _temp_dir, ingest_service, _) = diff --git a/quickwit/quickwit-serve/src/lib.rs b/quickwit/quickwit-serve/src/lib.rs index e903211a632..fcd3618df9e 100644 --- a/quickwit/quickwit-serve/src/lib.rs +++ b/quickwit/quickwit-serve/src/lib.rs @@ -572,6 +572,8 @@ async fn setup_ingest_v2( self_node_id.clone(), ingester_pool.clone(), &wal_dir_path, + config.ingest_api_config.max_queue_disk_usage, + config.ingest_api_config.max_queue_memory_usage, replication_factor, ) .await?; From a43ec93c20a75aa09fadd32681fa2a3728c71ea2 Mon Sep 17 00:00:00 2001 From: Adrien Guillo Date: Fri, 10 Nov 2023 05:56:44 -0500 Subject: [PATCH 21/27] Rate limit ingestion per shard to 5 MB/s (#4105) --- quickwit/quickwit-common/src/tower/rate.rs | 9 +- .../quickwit-ingest/src/ingest_v2/fetch.rs | 4 + .../quickwit-ingest/src/ingest_v2/ingester.rs | 128 ++++++++++++- quickwit/quickwit-ingest/src/ingest_v2/mod.rs | 2 + .../src/ingest_v2/rate_limiter.rs | 179 ++++++++++++++++++ .../src/ingest_v2/replication.rs | 3 + quickwit/quickwit-serve/src/lib.rs | 3 +- 7 files changed, 319 insertions(+), 9 deletions(-) create mode 100644 quickwit/quickwit-ingest/src/ingest_v2/rate_limiter.rs diff --git a/quickwit/quickwit-common/src/tower/rate.rs b/quickwit/quickwit-common/src/tower/rate.rs index a46a50293a5..81e7f72fe11 100644 --- a/quickwit/quickwit-common/src/tower/rate.rs +++ b/quickwit/quickwit-common/src/tower/rate.rs @@ -41,18 +41,21 @@ impl ConstantRate { /// /// # Panics /// - /// This function panics if `work` is equal to zero or `period` is < 1ms. + /// This function panics if `period` is < 1ms. pub fn new(work: u64, period: Duration) -> Self { - assert!(work > 0); assert!(period.as_millis() > 0); Self { work, period } } - pub fn from_bytes(bytes: ByteSize, period: Duration) -> Self { + pub fn bytes_per_period(bytes: ByteSize, period: Duration) -> Self { let work = bytes.as_u64(); Self::new(work, period) } + + pub fn bytes_per_sec(bytes: ByteSize) -> Self { + Self::bytes_per_period(bytes, Duration::from_secs(1)) + } } impl Rate for ConstantRate { diff --git a/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs b/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs index 97efd81d0d2..a26c2452d48 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs @@ -628,6 +628,7 @@ mod tests { let state = Arc::new(RwLock::new(IngesterState { mrecordlog, shards: HashMap::new(), + rate_limiters: HashMap::new(), replication_streams: HashMap::new(), replication_tasks: HashMap::new(), })); @@ -797,6 +798,7 @@ mod tests { let state = Arc::new(RwLock::new(IngesterState { mrecordlog, shards: HashMap::new(), + rate_limiters: HashMap::new(), replication_streams: HashMap::new(), replication_tasks: HashMap::new(), })); @@ -836,6 +838,7 @@ mod tests { let state = Arc::new(RwLock::new(IngesterState { mrecordlog, shards: HashMap::new(), + rate_limiters: HashMap::new(), replication_streams: HashMap::new(), replication_tasks: HashMap::new(), })); @@ -902,6 +905,7 @@ mod tests { let state = Arc::new(RwLock::new(IngesterState { mrecordlog, shards: HashMap::new(), + rate_limiters: HashMap::new(), replication_streams: HashMap::new(), replication_tasks: HashMap::new(), })); diff --git a/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs b/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs index 5d2465a5f01..0b372e8ebbc 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/ingester.rs @@ -49,6 +49,7 @@ use tracing::{error, info, warn}; use super::fetch::FetchTask; use super::models::{IngesterShard, PrimaryShard}; use super::mrecordlog_utils::{append_eof_record_if_necessary, check_enough_capacity}; +use super::rate_limiter::{RateLimiter, RateLimiterSettings}; use super::replication::{ ReplicationStreamTask, ReplicationStreamTaskHandle, ReplicationTask, ReplicationTaskHandle, SYN_REPLICATION_STREAM_CAPACITY, @@ -73,6 +74,7 @@ pub struct Ingester { state: Arc>, disk_capacity: ByteSize, memory_capacity: ByteSize, + rate_limiter_settings: RateLimiterSettings, replication_factor: usize, } @@ -87,6 +89,7 @@ impl fmt::Debug for Ingester { pub(super) struct IngesterState { pub mrecordlog: MultiRecordLog, pub shards: HashMap, + pub rate_limiters: HashMap, // Replication stream opened with followers. pub replication_streams: HashMap, // Replication tasks running for each replication stream opened with leaders. @@ -100,6 +103,7 @@ impl Ingester { wal_dir_path: &Path, disk_capacity: ByteSize, memory_capacity: ByteSize, + rate_limiter_settings: RateLimiterSettings, replication_factor: usize, ) -> IngestV2Result { let mrecordlog = MultiRecordLog::open_with_prefs( @@ -112,6 +116,7 @@ impl Ingester { let inner = IngesterState { mrecordlog, shards: HashMap::new(), + rate_limiters: HashMap::new(), replication_streams: HashMap::new(), replication_tasks: HashMap::new(), }; @@ -121,6 +126,7 @@ impl Ingester { state: Arc::new(RwLock::new(inner)), disk_capacity, memory_capacity, + rate_limiter_settings, replication_factor, }; info!( @@ -152,7 +158,7 @@ impl Ingester { let solo_shard = SoloShard::new(ShardState::Closed, Position::Eof); let shard = IngesterShard::Solo(solo_shard); - state_guard.shards.insert(queue_id, shard); + state_guard.shards.insert(queue_id.clone(), shard); } Ok(()) } @@ -178,6 +184,9 @@ impl Ingester { }); } }; + let rate_limiter = RateLimiter::from_settings(self.rate_limiter_settings); + state.rate_limiters.insert(queue_id.clone(), rate_limiter); + let shard = if let Some(follower_id) = follower_id_opt { self.init_replication_stream(state, leader_id, follower_id) .await?; @@ -316,8 +325,10 @@ impl IngesterService for Ingester { self.memory_capacity, requested_capacity, ) { - warn!("failed to persist records: {error}"); - + warn!( + "failed to persist records to ingester `{}`: {error}", + self.self_node_id + ); let persist_failure = PersistFailure { subrequest_id: subrequest.subrequest_id, index_uid: subrequest.index_uid, @@ -328,6 +339,24 @@ impl IngesterService for Ingester { persist_failures.push(persist_failure); continue; } + let rate_limiter = state_guard + .rate_limiters + .get_mut(&queue_id) + .expect("rate limiter should be initialized"); + + if !rate_limiter.acquire(requested_capacity) { + warn!("failed to persist records to shard `{queue_id}`: rate limited"); + + let persist_failure = PersistFailure { + subrequest_id: subrequest.subrequest_id, + index_uid: subrequest.index_uid, + source_id: subrequest.source_id, + shard_id: subrequest.shard_id, + reason: PersistFailureReason::RateLimited as i32, + }; + persist_failures.push(persist_failure); + continue; + } let current_position_inclusive: Position = if force_commit { let encoded_mrecords = doc_batch .docs() @@ -625,6 +654,7 @@ mod tests { use std::net::SocketAddr; use bytes::Bytes; + use quickwit_common::tower::ConstantRate; use quickwit_proto::ingest::ingester::{ IngesterServiceGrpcServer, IngesterServiceGrpcServerAdapter, PersistSubrequest, TruncateSubrequest, @@ -646,6 +676,7 @@ mod tests { let wal_dir_path = tempdir.path(); let disk_capacity = ByteSize::mb(256); let memory_capacity = ByteSize::mb(1); + let rate_limiter_settings = RateLimiterSettings::default(); let replication_factor = 2; let mut ingester = Ingester::try_new( self_node_id.clone(), @@ -653,6 +684,7 @@ mod tests { wal_dir_path, disk_capacity, memory_capacity, + rate_limiter_settings, replication_factor, ) .await @@ -751,6 +783,7 @@ mod tests { let wal_dir_path = tempdir.path(); let disk_capacity = ByteSize::mb(256); let memory_capacity = ByteSize::mb(1); + let rate_limiter_settings = RateLimiterSettings::default(); let replication_factor = 1; let mut ingester = Ingester::try_new( self_node_id.clone(), @@ -758,6 +791,7 @@ mod tests { wal_dir_path, disk_capacity, memory_capacity, + rate_limiter_settings, replication_factor, ) .await @@ -850,6 +884,7 @@ mod tests { let wal_dir_path = tempdir.path(); let disk_capacity = ByteSize::mb(256); let memory_capacity = ByteSize::mb(1); + let rate_limiter_settings = RateLimiterSettings::default(); let replication_factor = 1; let mut ingester = Ingester::try_new( self_node_id.clone(), @@ -857,6 +892,7 @@ mod tests { wal_dir_path, disk_capacity, memory_capacity, + rate_limiter_settings, replication_factor, ) .await @@ -896,6 +932,7 @@ mod tests { let disk_capacity = ByteSize::mb(256); let memory_capacity = ByteSize::mb(1); + let rate_limiter_settings = RateLimiterSettings::default(); let replication_factor = 2; let mut leader = Ingester::try_new( @@ -904,6 +941,7 @@ mod tests { wal_dir_path, disk_capacity, memory_capacity, + rate_limiter_settings, replication_factor, ) .await @@ -919,6 +957,7 @@ mod tests { wal_dir_path, disk_capacity, memory_capacity, + rate_limiter_settings, replication_factor, ) .await @@ -1045,6 +1084,7 @@ mod tests { let wal_dir_path = tempdir.path(); let disk_capacity = ByteSize::mb(256); let memory_capacity = ByteSize::mb(1); + let rate_limiter_settings = RateLimiterSettings::default(); let replication_factor = 2; let mut leader = Ingester::try_new( leader_id.clone(), @@ -1052,6 +1092,7 @@ mod tests { wal_dir_path, disk_capacity, memory_capacity, + rate_limiter_settings, replication_factor, ) .await @@ -1076,6 +1117,7 @@ mod tests { let wal_dir_path = tempdir.path(); let disk_capacity = ByteSize::mb(256); let memory_capacity = ByteSize::mb(1); + let rate_limiter_settings = RateLimiterSettings::default(); let replication_factor = 2; let follower = Ingester::try_new( follower_id.clone(), @@ -1083,6 +1125,7 @@ mod tests { wal_dir_path, disk_capacity, memory_capacity, + rate_limiter_settings, replication_factor, ) .await @@ -1217,6 +1260,7 @@ mod tests { let wal_dir_path = tempdir.path(); let disk_capacity = ByteSize::mib(256); let memory_capacity = ByteSize::mib(1); + let rate_limiter_settings = RateLimiterSettings::default(); let replication_factor = 1; let mut ingester = Ingester::try_new( self_node_id.clone(), @@ -1224,6 +1268,7 @@ mod tests { wal_dir_path, disk_capacity, memory_capacity, + rate_limiter_settings, replication_factor, ) .await @@ -1243,7 +1288,7 @@ mod tests { let persist_request = PersistRequest { leader_id: self_node_id.to_string(), - commit_type: CommitTypeV2::Force as i32, + commit_type: CommitTypeV2::Auto as i32, subrequests: vec![PersistSubrequest { subrequest_id: 0, index_uid: "test-index:0".to_string(), @@ -1274,6 +1319,71 @@ mod tests { solo_shard_01.assert_replication_position(Position::Beginning); } + #[tokio::test] + async fn test_ingester_persist_rate_limited() { + let tempdir = tempfile::tempdir().unwrap(); + let self_node_id: NodeId = "test-ingester-0".into(); + let ingester_pool = IngesterPool::default(); + let wal_dir_path = tempdir.path(); + let disk_capacity = ByteSize::mib(256); + let memory_capacity = ByteSize::mib(1); + let rate_limiter_settings = RateLimiterSettings { + burst_limit: ByteSize(0), + rate_limit: ConstantRate::bytes_per_sec(ByteSize(0)), + refill_period: Duration::from_millis(100), + }; + let replication_factor = 1; + let mut ingester = Ingester::try_new( + self_node_id.clone(), + ingester_pool, + wal_dir_path, + disk_capacity, + memory_capacity, + rate_limiter_settings, + replication_factor, + ) + .await + .unwrap(); + + let persist_request = PersistRequest { + leader_id: self_node_id.to_string(), + commit_type: CommitTypeV2::Auto as i32, + subrequests: vec![PersistSubrequest { + subrequest_id: 0, + index_uid: "test-index:0".to_string(), + source_id: "test-source".to_string(), + shard_id: 1, + follower_id: None, + doc_batch: Some(DocBatchV2::for_test(["test-doc-010"])), + }], + }; + let persist_response = ingester.persist(persist_request).await.unwrap(); + assert_eq!(persist_response.leader_id, "test-ingester-0"); + assert_eq!(persist_response.successes.len(), 0); + assert_eq!(persist_response.failures.len(), 1); + + let persist_failure = &persist_response.failures[0]; + assert_eq!(persist_failure.subrequest_id, 0); + assert_eq!(persist_failure.index_uid, "test-index:0"); + assert_eq!(persist_failure.source_id, "test-source"); + assert_eq!(persist_failure.shard_id, 1); + assert_eq!(persist_failure.reason(), PersistFailureReason::RateLimited); + + let state_guard = ingester.state.read().await; + assert_eq!(state_guard.shards.len(), 1); + + let queue_id_01 = queue_id("test-index:0", "test-source", 1); + + let solo_shard_01 = state_guard.shards.get(&queue_id_01).unwrap(); + solo_shard_01.assert_is_solo(); + solo_shard_01.assert_is_open(); + solo_shard_01.assert_replication_position(Position::Beginning); + + state_guard + .mrecordlog + .assert_records_eq(&queue_id_01, .., &[]); + } + #[tokio::test] async fn test_ingester_persist_resource_exhausted() { let tempdir = tempfile::tempdir().unwrap(); @@ -1282,6 +1392,7 @@ mod tests { let wal_dir_path = tempdir.path(); let disk_capacity = ByteSize(0); let memory_capacity = ByteSize(0); + let rate_limiter_settings = RateLimiterSettings::default(); let replication_factor = 1; let mut ingester = Ingester::try_new( self_node_id.clone(), @@ -1289,6 +1400,7 @@ mod tests { wal_dir_path, disk_capacity, memory_capacity, + rate_limiter_settings, replication_factor, ) .await @@ -1296,7 +1408,7 @@ mod tests { let persist_request = PersistRequest { leader_id: self_node_id.to_string(), - commit_type: CommitTypeV2::Force as i32, + commit_type: CommitTypeV2::Auto as i32, subrequests: vec![PersistSubrequest { subrequest_id: 0, index_uid: "test-index:0".to_string(), @@ -1343,6 +1455,7 @@ mod tests { let wal_dir_path = tempdir.path(); let disk_capacity = ByteSize::mb(256); let memory_capacity = ByteSize::mb(1); + let rate_limiter_settings = RateLimiterSettings::default(); let replication_factor = 1; let mut ingester = Ingester::try_new( self_node_id.clone(), @@ -1350,6 +1463,7 @@ mod tests { wal_dir_path, disk_capacity, memory_capacity, + rate_limiter_settings, replication_factor, ) .await @@ -1445,6 +1559,7 @@ mod tests { let wal_dir_path = tempdir.path(); let disk_capacity = ByteSize::mb(256); let memory_capacity = ByteSize::mb(1); + let rate_limiter_settings = RateLimiterSettings::default(); let replication_factor = 1; let mut ingester = Ingester::try_new( self_node_id.clone(), @@ -1452,6 +1567,7 @@ mod tests { wal_dir_path, disk_capacity, memory_capacity, + rate_limiter_settings, replication_factor, ) .await @@ -1539,6 +1655,7 @@ mod tests { let wal_dir_path = tempdir.path(); let disk_capacity = ByteSize::mb(256); let memory_capacity = ByteSize::mb(1); + let rate_limiter_settings = RateLimiterSettings::default(); let replication_factor = 1; let mut ingester = Ingester::try_new( self_node_id.clone(), @@ -1546,6 +1663,7 @@ mod tests { wal_dir_path, disk_capacity, memory_capacity, + rate_limiter_settings, replication_factor, ) .await diff --git a/quickwit/quickwit-ingest/src/ingest_v2/mod.rs b/quickwit/quickwit-ingest/src/ingest_v2/mod.rs index ec950ada275..345de74ca08 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/mod.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/mod.rs @@ -22,6 +22,7 @@ mod ingester; mod models; mod mrecord; mod mrecordlog_utils; +mod rate_limiter; mod replication; mod router; mod shard_table; @@ -39,6 +40,7 @@ pub use self::fetch::{FetchStreamError, MultiFetchStream}; pub use self::ingester::Ingester; use self::mrecord::MRECORD_HEADER_LEN; pub use self::mrecord::{decoded_mrecords, MRecord}; +pub use self::rate_limiter::RateLimiterSettings; pub use self::router::IngestRouter; pub type IngesterPool = Pool; diff --git a/quickwit/quickwit-ingest/src/ingest_v2/rate_limiter.rs b/quickwit/quickwit-ingest/src/ingest_v2/rate_limiter.rs new file mode 100644 index 00000000000..1f310eca3b9 --- /dev/null +++ b/quickwit/quickwit-ingest/src/ingest_v2/rate_limiter.rs @@ -0,0 +1,179 @@ +// Copyright (C) 2023 Quickwit, Inc. +// +// Quickwit is offered under the AGPL v3.0 and as commercial software. +// For commercial licensing, contact us at hello@quickwit.io. +// +// AGPL: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::time::{Duration, Instant}; + +use bytesize::ByteSize; +use quickwit_common::tower::{ConstantRate, Rate}; + +#[derive(Debug, Clone, Copy)] +pub struct RateLimiterSettings { + // After a long period of inactivity, the rate limiter can accumulate some "credits" + // up to what we call a `burst_limit`. + // + // Until these credits are expired, the rate limiter may exceed temporarily its rate limit. + pub burst_limit: ByteSize, + pub rate_limit: ConstantRate, + // The refill period has an effect on the resolution at which the + // rate limiting is enforced. + // + // `Instant::now()` is guaranteed to be called at most once per refill_period. + pub refill_period: Duration, +} + +impl Default for RateLimiterSettings { + fn default() -> Self { + // 10 MB burst limit. + let burst_limit = ByteSize::mb(10); + // 5 MB/s rate limit. + let rate_limit = ConstantRate::bytes_per_sec(ByteSize::mb(5)); + // Refill every 100ms. + let refill_period = Duration::from_millis(100); + + Self { + burst_limit, + rate_limit, + refill_period, + } + } +} + +/// A bursty token-based rate limiter. +#[derive(Debug, Clone)] +pub(super) struct RateLimiter { + capacity: u64, + available: u64, + refill_amount: u64, + refill_period: Duration, + refill_period_micros: u64, + refill_at: Instant, +} + +impl RateLimiter { + pub fn from_settings(settings: RateLimiterSettings) -> Self { + let capacity = settings.burst_limit.as_u64(); + + let work = settings.rate_limit.work() as u128; + let refill_period = settings.refill_period; + let rate_limit_period = settings.rate_limit.period(); + let refill_amount = work * refill_period.as_nanos() / rate_limit_period.as_nanos(); + + Self { + capacity, + available: capacity, + refill_amount: refill_amount as u64, + refill_period, + refill_period_micros: refill_period.as_micros() as u64, + refill_at: Instant::now() + refill_period, + } + } + + pub fn acquire(&mut self, capacity: ByteSize) -> bool { + if self.acquire_inner(capacity.as_u64()) { + true + } else { + self.refill(Instant::now()); + self.acquire_inner(capacity.as_u64()) + } + } + + fn acquire_inner(&mut self, capacity: u64) -> bool { + if self.available >= capacity { + self.available -= capacity; + true + } else { + false + } + } + + fn refill(&mut self, now: Instant) { + if now < self.refill_at { + return; + } + let elapsed = (now - self.refill_at).as_micros() as u64; + // More than one refill period may have elapsed so we need to take that into account. + let refill = self.refill_amount + self.refill_amount * elapsed / self.refill_period_micros; + self.available = std::cmp::min(self.available + refill, self.capacity); + self.refill_at = now + self.refill_period; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_rate_limiter() { + let settings = RateLimiterSettings { + burst_limit: ByteSize::mb(2), + rate_limit: ConstantRate::bytes_per_sec(ByteSize::mb(1)), + refill_period: Duration::from_millis(100), + }; + let mut rate_limiter = RateLimiter::from_settings(settings); + assert_eq!(rate_limiter.capacity, ByteSize::mb(2).as_u64()); + assert_eq!(rate_limiter.available, ByteSize::mb(2).as_u64()); + assert_eq!(rate_limiter.refill_amount, ByteSize::kb(100).as_u64()); + assert_eq!(rate_limiter.refill_period, Duration::from_millis(100)); + + assert!(rate_limiter.acquire(ByteSize::mb(1))); + assert!(rate_limiter.acquire(ByteSize::mb(1))); + assert!(!rate_limiter.acquire(ByteSize::kb(1))); + + std::thread::sleep(Duration::from_millis(100)); + + assert!(rate_limiter.acquire(ByteSize::kb(100))); + assert!(!rate_limiter.acquire(ByteSize::kb(20))); + + std::thread::sleep(Duration::from_millis(250)); + + assert!(rate_limiter.acquire(ByteSize::kb(125))); + assert!(rate_limiter.acquire(ByteSize::kb(125))); + assert!(!rate_limiter.acquire(ByteSize::kb(20))); + } + + #[test] + fn test_rate_limiter_refill() { + let settings = RateLimiterSettings { + burst_limit: ByteSize::mb(2), + rate_limit: ConstantRate::bytes_per_sec(ByteSize::mb(1)), + refill_period: Duration::from_millis(100), + }; + let mut rate_limiter = RateLimiter::from_settings(settings); + + rate_limiter.available = 0; + let now = Instant::now(); + rate_limiter.refill(now); + assert_eq!(rate_limiter.available, 0); + + rate_limiter.available = 0; + let now = now + Duration::from_millis(100); + rate_limiter.refill(now); + assert_eq!(rate_limiter.available, ByteSize::kb(100).as_u64()); + + rate_limiter.available = 0; + let now = now + Duration::from_millis(110); + rate_limiter.refill(now); + assert_eq!(rate_limiter.available, ByteSize::kb(110).as_u64()); + + rate_limiter.available = 0; + let now = now + Duration::from_millis(210); + rate_limiter.refill(now); + assert_eq!(rate_limiter.available, ByteSize::kb(210).as_u64()); + } +} diff --git a/quickwit/quickwit-ingest/src/ingest_v2/replication.rs b/quickwit/quickwit-ingest/src/ingest_v2/replication.rs index 67d19cdb7b1..4124a6f480d 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/replication.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/replication.rs @@ -674,6 +674,7 @@ mod tests { let state = Arc::new(RwLock::new(IngesterState { mrecordlog, shards: HashMap::new(), + rate_limiters: HashMap::new(), replication_streams: HashMap::new(), replication_tasks: HashMap::new(), })); @@ -837,6 +838,7 @@ mod tests { let state = Arc::new(RwLock::new(IngesterState { mrecordlog, shards: HashMap::new(), + rate_limiters: HashMap::new(), replication_streams: HashMap::new(), replication_tasks: HashMap::new(), })); @@ -917,6 +919,7 @@ mod tests { let state = Arc::new(RwLock::new(IngesterState { mrecordlog, shards: HashMap::new(), + rate_limiters: HashMap::new(), replication_streams: HashMap::new(), replication_tasks: HashMap::new(), })); diff --git a/quickwit/quickwit-serve/src/lib.rs b/quickwit/quickwit-serve/src/lib.rs index fcd3618df9e..f244c12b88d 100644 --- a/quickwit/quickwit-serve/src/lib.rs +++ b/quickwit/quickwit-serve/src/lib.rs @@ -69,7 +69,7 @@ use quickwit_indexing::actors::IndexingService; use quickwit_indexing::start_indexing_service; use quickwit_ingest::{ start_ingest_api_service, GetMemoryCapacity, IngestApiService, IngestRequest, IngestRouter, - IngestServiceClient, Ingester, IngesterPool, + IngestServiceClient, Ingester, IngesterPool, RateLimiterSettings, }; use quickwit_janitor::{start_janitor_service, JanitorService}; use quickwit_metastore::{ @@ -574,6 +574,7 @@ async fn setup_ingest_v2( &wal_dir_path, config.ingest_api_config.max_queue_disk_usage, config.ingest_api_config.max_queue_memory_usage, + RateLimiterSettings::default(), replication_factor, ) .await?; From 9e95ec9e7a27bdb32c4addb5731d1c41bfd46caf Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Fri, 10 Nov 2023 21:52:43 +0900 Subject: [PATCH 22/27] Added an integration test for ingest v2 (#4071) - Adds an ingest_v2 method to the cluster sandbox. - Adds two simple integration tests. - Converts the ingest v2 error into rest api errors. - This PR suffers from the deadlock described in #4070 Closes #4065 --- .../src/test_utils/cluster_sandbox.rs | 10 +- .../src/tests/index_tests.rs | 146 ++++++++++++++---- quickwit/quickwit-rest-client/src/models.rs | 1 + .../quickwit-rest-client/src/rest_client.rs | 5 +- quickwit/quickwit-search/src/root.rs | 4 +- .../src/index_api/rest_handler.rs | 1 - .../src/ingest_api/rest_handler.rs | 49 +++++- 7 files changed, 174 insertions(+), 42 deletions(-) diff --git a/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs b/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs index dc08cbd2186..2d832c9d0b3 100644 --- a/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs +++ b/quickwit/quickwit-integration-tests/src/test_utils/cluster_sandbox.rs @@ -201,6 +201,10 @@ impl ClusterSandbox { }) } + pub fn enable_ingest_v2(&mut self) { + self.searcher_rest_client.enable_ingest_v2(); + } + // Starts one node that runs all the services. pub async fn start_standalone_node() -> anyhow::Result { let temp_dir = tempfile::tempdir()?; @@ -286,13 +290,13 @@ impl ClusterSandbox { pub async fn wait_for_splits( &self, index_id: &str, - split_states: Option>, + split_states_filter: Option>, required_splits_num: usize, ) -> anyhow::Result<()> { wait_until_predicate( || { let splits_query_params = ListSplitsQueryParams { - split_states: split_states.clone(), + split_states: split_states_filter.clone(), ..Default::default() }; async move { @@ -322,7 +326,7 @@ impl ClusterSandbox { } }, Duration::from_secs(10), - Duration::from_millis(100), + Duration::from_millis(500), ) .await?; Ok(()) diff --git a/quickwit/quickwit-integration-tests/src/tests/index_tests.rs b/quickwit/quickwit-integration-tests/src/tests/index_tests.rs index e887cb82281..688114cdb98 100644 --- a/quickwit/quickwit-integration-tests/src/tests/index_tests.rs +++ b/quickwit/quickwit-integration-tests/src/tests/index_tests.rs @@ -23,9 +23,11 @@ use std::time::Duration; use bytes::Bytes; use quickwit_common::test_utils::wait_until_predicate; use quickwit_config::service::QuickwitService; +use quickwit_config::ConfigFormat; use quickwit_indexing::actors::INDEXING_DIR_NAME; use quickwit_janitor::actors::DELETE_SERVICE_TASK_DIR_NAME; use quickwit_metastore::SplitState; +use quickwit_rest_client::error::{ApiError, Error}; use quickwit_rest_client::rest_client::CommitType; use quickwit_serve::SearchRequestQueryString; use serde_json::json; @@ -60,11 +62,7 @@ async fn test_restarting_standalone_server() { sandbox .indexer_rest_client .indexes() - .create( - index_config.clone(), - quickwit_config::ConfigFormat::Yaml, - false, - ) + .create(index_config.clone(), ConfigFormat::Yaml, false) .await .unwrap(); @@ -102,7 +100,7 @@ async fn test_restarting_standalone_server() { sandbox .indexer_rest_client .indexes() - .create(index_config, quickwit_config::ConfigFormat::Yaml, false) + .create(index_config, ConfigFormat::Yaml, false) .await .unwrap(); @@ -215,36 +213,124 @@ async fn test_restarting_standalone_server() { sandbox.shutdown().await.unwrap(); } +const TEST_INDEX_CONFIG: &str = r#" + version: 0.6 + index_id: test_index + doc_mapping: + field_mappings: + - name: body + type: text + indexing_settings: + commit_timeout_secs: 1 + merge_policy: + type: stable_log + merge_factor: 4 + max_merge_factor: 4 +"#; + +#[tokio::test] +async fn test_ingest_v2_index_not_found() { + // This tests checks what happens when we try to ingest into a non-existing index. + quickwit_common::setup_logging_for_tests(); + let nodes_services = &[ + HashSet::from_iter([QuickwitService::Indexer, QuickwitService::Janitor]), + HashSet::from_iter([QuickwitService::Indexer, QuickwitService::Janitor]), + HashSet::from_iter([ + QuickwitService::ControlPlane, + QuickwitService::Metastore, + QuickwitService::Searcher, + ]), + ]; + let mut sandbox = ClusterSandbox::start_cluster_nodes(&nodes_services[..]) + .await + .unwrap(); + sandbox.enable_ingest_v2(); + sandbox.wait_for_cluster_num_ready_nodes(3).await.unwrap(); + let missing_index_err: Error = sandbox + .indexer_rest_client + .ingest( + "missing_index", + ingest_json!({"body": "doc1"}), + None, + None, + CommitType::WaitFor, + ) + .await + .unwrap_err(); + let Error::Api(ApiError { message, code }) = missing_index_err else { + panic!("Expected an API error."); + }; + assert_eq!(code, 404u16); + let error_message = message.unwrap(); + assert_eq!(error_message, "index `missing_index` not found"); + sandbox.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn test_ingest_v2_happy_path() { + // This tests checks our happy path for ingesting one doc. + quickwit_common::setup_logging_for_tests(); + let nodes_services = &[ + HashSet::from_iter([QuickwitService::Indexer, QuickwitService::Janitor]), + HashSet::from_iter([QuickwitService::Indexer, QuickwitService::Janitor]), + HashSet::from_iter([ + QuickwitService::ControlPlane, + QuickwitService::Metastore, + QuickwitService::Searcher, + ]), + ]; + let mut sandbox = ClusterSandbox::start_cluster_nodes(&nodes_services[..]) + .await + .unwrap(); + sandbox.enable_ingest_v2(); + sandbox.wait_for_cluster_num_ready_nodes(3).await.unwrap(); + sandbox + .indexer_rest_client + .indexes() + .create(TEST_INDEX_CONFIG.into(), ConfigFormat::Yaml, false) + .await + .unwrap(); + sandbox + .indexer_rest_client + .sources("test_index") + .toggle("_ingest-source", true) + .await + .unwrap(); + sandbox + .indexer_rest_client + .ingest( + "test_index", + ingest_json!({"body": "doc1"}), + None, + None, + CommitType::WaitFor, + ) + .await + .unwrap(); + let search_req = SearchRequestQueryString { + query: "*".to_string(), + ..Default::default() + }; + let search_result = sandbox + .indexer_rest_client + .search("test_index", search_req) + .await + .unwrap(); + assert_eq!(search_result.num_hits, 1); + sandbox.shutdown().await.unwrap(); +} + #[tokio::test] async fn test_commit_modes() { quickwit_common::setup_logging_for_tests(); let sandbox = ClusterSandbox::start_standalone_node().await.unwrap(); - let index_id = "test_commit_modes_index"; + let index_id = "test_index"; // Create index sandbox .indexer_rest_client .indexes() - .create( - r#" - version: 0.6 - index_id: test_commit_modes_index - doc_mapping: - field_mappings: - - name: body - type: text - indexing_settings: - commit_timeout_secs: 1 - merge_policy: - type: stable_log - merge_factor: 4 - max_merge_factor: 4 - - "# - .into(), - quickwit_config::ConfigFormat::Yaml, - false, - ) + .create(TEST_INDEX_CONFIG.into(), ConfigFormat::Yaml, false) .await .unwrap(); @@ -391,7 +477,7 @@ async fn test_very_large_index_name() { "#, ) .into(), - quickwit_config::ConfigFormat::Yaml, + ConfigFormat::Yaml, false, ) .await @@ -447,7 +533,7 @@ async fn test_very_large_index_name() { "#, ) .into(), - quickwit_config::ConfigFormat::Yaml, + ConfigFormat::Yaml, false, ) .await @@ -484,7 +570,7 @@ async fn test_shutdown() { commit_timeout_secs: 1 "# .into(), - quickwit_config::ConfigFormat::Yaml, + ConfigFormat::Yaml, false, ) .await diff --git a/quickwit/quickwit-rest-client/src/models.rs b/quickwit/quickwit-rest-client/src/models.rs index 72e2fb1b804..6416a8ec286 100644 --- a/quickwit/quickwit-rest-client/src/models.rs +++ b/quickwit/quickwit-rest-client/src/models.rs @@ -26,6 +26,7 @@ use serde::de::DeserializeOwned; use crate::error::{ApiError, Error, ErrorResponsePayload}; +#[derive(Debug)] pub struct ApiResponse { inner: reqwest::Response, } diff --git a/quickwit/quickwit-rest-client/src/rest_client.rs b/quickwit/quickwit-rest-client/src/rest_client.rs index 52ffb9e00ee..6b333131b93 100644 --- a/quickwit/quickwit-rest-client/src/rest_client.rs +++ b/quickwit/quickwit-rest-client/src/rest_client.rs @@ -198,6 +198,10 @@ pub struct QuickwitClient { } impl QuickwitClient { + pub fn enable_ingest_v2(&mut self) { + self.ingest_v2 = true; + } + pub async fn search( &self, index_id: &str, @@ -288,7 +292,6 @@ impl QuickwitClient { timeout, ) .await?; - if response.status_code() == StatusCode::TOO_MANY_REQUESTS { if let Some(event_fn) = &on_ingest_event { event_fn(IngestEvent::Sleep) diff --git a/quickwit/quickwit-search/src/root.rs b/quickwit/quickwit-search/src/root.rs index 4d0f459e65a..2a916f00fa4 100644 --- a/quickwit/quickwit-search/src/root.rs +++ b/quickwit/quickwit-search/src/root.rs @@ -392,7 +392,7 @@ fn get_scroll_ttl_duration(search_request: &SearchRequest) -> crate::Result MetastoreResult> { - info!("get-indexes-metadatas"); metastore .list_indexes_metadata(ListIndexesMetadataRequest::all()) .await diff --git a/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs b/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs index 5a65411d262..9c0dd0ea912 100644 --- a/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs +++ b/quickwit/quickwit-serve/src/ingest_api/rest_handler.rs @@ -24,8 +24,8 @@ use quickwit_ingest::{ IngestServiceClient, IngestServiceError, TailRequest, }; use quickwit_proto::ingest::router::{ - IngestRequestV2, IngestResponseV2, IngestRouterService, IngestRouterServiceClient, - IngestSubrequest, + IngestFailureReason, IngestRequestV2, IngestResponseV2, IngestRouterService, + IngestRouterServiceClient, IngestSubrequest, }; use quickwit_proto::ingest::{DocBatchV2, IngestV2Error}; use quickwit_proto::types::IndexId; @@ -127,7 +127,7 @@ async fn ingest_v2( body: Bytes, ingest_options: IngestOptions, mut ingest_router: IngestRouterServiceClient, -) -> Result { +) -> Result { let mut doc_buffer = BytesMut::new(); let mut doc_lengths = Vec::new(); @@ -135,6 +135,7 @@ async fn ingest_v2( doc_lengths.push(line.len() as u32); doc_buffer.put(line); } + let num_docs = doc_lengths.len(); let doc_batch = DocBatchV2 { doc_buffer: doc_buffer.freeze(), doc_lengths, @@ -149,8 +150,46 @@ async fn ingest_v2( commit_type: ingest_options.commit_type as i32, subrequests: vec![subrequest], }; - let response = ingest_router.ingest(request).await?; - Ok(response) + let response = ingest_router + .ingest(request) + .await + .map_err(|err: IngestV2Error| IngestServiceError::Internal(err.to_string()))?; + convert_ingest_response_v2(response, num_docs) +} + +fn convert_ingest_response_v2( + mut response: IngestResponseV2, + num_docs: usize, +) -> Result { + let num_responses = response.successes.len() + response.failures.len(); + if num_responses != 1 { + return Err(IngestServiceError::Internal(format!( + "Expected a single failure/success, got {}.", + num_responses + ))); + } + if response.successes.pop().is_some() { + return Ok(IngestResponse { + num_docs_for_processing: num_docs as u64, + }); + } + let ingest_failure = response.failures.pop().unwrap(); + Err(match ingest_failure.reason() { + IngestFailureReason::Unspecified => { + IngestServiceError::Internal("Unknown reason".to_string()) + } + IngestFailureReason::IndexNotFound => IngestServiceError::IndexNotFound { + index_id: ingest_failure.index_id, + }, + IngestFailureReason::SourceNotFound => IngestServiceError::Internal(format!( + "Ingest v2 source not found for index {}", + ingest_failure.index_id + )), + IngestFailureReason::Internal => IngestServiceError::Internal("Internal error".to_string()), + IngestFailureReason::NoShardsAvailable => IngestServiceError::Unavailable, + IngestFailureReason::RateLimited => IngestServiceError::RateLimited, + IngestFailureReason::ResourceExhausted => IngestServiceError::Unavailable, + }) } #[utoipa::path( From 0b5ddafadcc5736de28fca3f823d5f8cd04a9d10 Mon Sep 17 00:00:00 2001 From: Eugene Tolbakov Date: Fri, 10 Nov 2023 16:23:51 +0000 Subject: [PATCH 23/27] =?UTF-8?q?fix:=20replace=20actor=20heartbeat=20dura?= =?UTF-8?q?tion=20with=20emit=20timeout=20as=20the=20loop=20i=E2=80=A6=20(?= =?UTF-8?q?#4112)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: replace actor heartbeat duration with emit timeout as the loop interval for sources * chore: remove dead code * chore: use Bytesize for BATCH_NUM_BYTES_LIMIT const --- .../src/source/gcp_pubsub_source.rs | 5 ++--- .../src/source/ingest/mod.rs | 5 ++--- .../src/source/kafka_source.rs | 20 +++++-------------- .../src/source/kinesis/kinesis_source.rs | 11 +++++----- quickwit/quickwit-indexing/src/source/mod.rs | 16 +++++++++++++++ .../src/source/pulsar_source.rs | 16 ++------------- 6 files changed, 33 insertions(+), 40 deletions(-) diff --git a/quickwit/quickwit-indexing/src/source/gcp_pubsub_source.rs b/quickwit/quickwit-indexing/src/source/gcp_pubsub_source.rs index 389ec96d366..a22e5e2c454 100644 --- a/quickwit/quickwit-indexing/src/source/gcp_pubsub_source.rs +++ b/quickwit/quickwit-indexing/src/source/gcp_pubsub_source.rs @@ -37,11 +37,10 @@ use serde_json::{json, Value as JsonValue}; use tokio::time; use tracing::{debug, info, warn}; -use super::SourceActor; +use super::{SourceActor, BATCH_NUM_BYTES_LIMIT, EMIT_BATCHES_TIMEOUT}; use crate::actors::DocProcessor; use crate::source::{BatchBuilder, Source, SourceContext, SourceRuntimeArgs, TypedSourceFactory}; -const BATCH_NUM_BYTES_LIMIT: u64 = 5_000_000; const DEFAULT_MAX_MESSAGES_PER_PULL: i32 = 1_000; pub struct GcpPubSubSourceFactory; @@ -168,7 +167,7 @@ impl Source for GcpPubSubSource { ) -> Result { let now = Instant::now(); let mut batch: BatchBuilder = BatchBuilder::default(); - let deadline = time::sleep(*quickwit_actors::HEARTBEAT / 2); + let deadline = time::sleep(EMIT_BATCHES_TIMEOUT); tokio::pin!(deadline); // TODO: ensure we ACK the message after being commit: at least once // TODO: ensure we increase_ack_deadline for the items diff --git a/quickwit/quickwit-indexing/src/source/ingest/mod.rs b/quickwit/quickwit-indexing/src/source/ingest/mod.rs index 1d7aee86ed1..53aa50a4b1e 100644 --- a/quickwit/quickwit-indexing/src/source/ingest/mod.rs +++ b/quickwit/quickwit-indexing/src/source/ingest/mod.rs @@ -45,12 +45,11 @@ use ulid::Ulid; use super::{ Assignment, BatchBuilder, Source, SourceContext, SourceRuntimeArgs, TypedSourceFactory, + BATCH_NUM_BYTES_LIMIT, EMIT_BATCHES_TIMEOUT, }; use crate::actors::DocProcessor; use crate::models::{NewPublishLock, NewPublishToken, PublishLock}; -const EMIT_BATCHES_TIMEOUT: Duration = Duration::from_millis(if cfg!(test) { 100 } else { 1_000 }); - pub struct IngestSourceFactory; #[async_trait] @@ -309,7 +308,7 @@ impl Source for IngestSource { Ok(Ok(fetch_payload)) => { self.process_fetch_response(&mut batch_builder, fetch_payload)?; - if batch_builder.num_bytes >= 5 * 1024 * 1024 { + if batch_builder.num_bytes >= BATCH_NUM_BYTES_LIMIT { break; } } diff --git a/quickwit/quickwit-indexing/src/source/kafka_source.rs b/quickwit/quickwit-indexing/src/source/kafka_source.rs index 845a1a59b72..52c97722a47 100644 --- a/quickwit/quickwit-indexing/src/source/kafka_source.rs +++ b/quickwit/quickwit-indexing/src/source/kafka_source.rs @@ -49,20 +49,10 @@ use tracing::{debug, info, warn}; use crate::actors::DocProcessor; use crate::models::{NewPublishLock, PublishLock}; -use crate::source::{BatchBuilder, Source, SourceContext, SourceRuntimeArgs, TypedSourceFactory}; - -/// Number of bytes after which we cut a new batch. -/// -/// We try to emit chewable batches for the indexer. -/// One batch = one message to the indexer actor. -/// -/// If batches are too large: -/// - we might not be able to observe the state of the indexer for 5 seconds. -/// - we will be needlessly occupying resident memory in the mailbox. -/// - we will not have a precise control of the timeout before commit. -/// -/// 5MB seems like a good one size fits all value. -const BATCH_NUM_BYTES_LIMIT: u64 = 5_000_000; +use crate::source::{ + BatchBuilder, Source, SourceContext, SourceRuntimeArgs, TypedSourceFactory, + BATCH_NUM_BYTES_LIMIT, EMIT_BATCHES_TIMEOUT, +}; type GroupId = String; @@ -486,7 +476,7 @@ impl Source for KafkaSource { ) -> Result { let now = Instant::now(); let mut batch = BatchBuilder::default(); - let deadline = time::sleep(*quickwit_actors::HEARTBEAT / 2); + let deadline = time::sleep(EMIT_BATCHES_TIMEOUT); tokio::pin!(deadline); loop { diff --git a/quickwit/quickwit-indexing/src/source/kinesis/kinesis_source.rs b/quickwit/quickwit-indexing/src/source/kinesis/kinesis_source.rs index 7b3ba961781..5df7892a39f 100644 --- a/quickwit/quickwit-indexing/src/source/kinesis/kinesis_source.rs +++ b/quickwit/quickwit-indexing/src/source/kinesis/kinesis_source.rs @@ -43,9 +43,10 @@ use super::shard_consumer::{ShardConsumer, ShardConsumerHandle, ShardConsumerMes use crate::actors::DocProcessor; use crate::models::RawDocBatch; use crate::source::kinesis::helpers::get_kinesis_client; -use crate::source::{Source, SourceContext, SourceRuntimeArgs, TypedSourceFactory}; - -const TARGET_BATCH_NUM_BYTES: u64 = 5_000_000; +use crate::source::{ + Source, SourceContext, SourceRuntimeArgs, TypedSourceFactory, BATCH_NUM_BYTES_LIMIT, + EMIT_BATCHES_TIMEOUT, +}; type ShardId = String; @@ -215,7 +216,7 @@ impl Source for KinesisSource { let mut docs = Vec::new(); let mut checkpoint_delta = SourceCheckpointDelta::default(); - let deadline = time::sleep(*quickwit_actors::HEARTBEAT / 2); + let deadline = time::sleep(EMIT_BATCHES_TIMEOUT); tokio::pin!(deadline); loop { @@ -278,7 +279,7 @@ impl Source for KinesisSource { ).context("failed to record partition delta")?; } } - if batch_num_bytes >= TARGET_BATCH_NUM_BYTES { + if batch_num_bytes >= BATCH_NUM_BYTES_LIMIT { break; } } diff --git a/quickwit/quickwit-indexing/src/source/mod.rs b/quickwit/quickwit-indexing/src/source/mod.rs index 94bf12fbb94..d3325fc2195 100644 --- a/quickwit/quickwit-indexing/src/source/mod.rs +++ b/quickwit/quickwit-indexing/src/source/mod.rs @@ -77,6 +77,7 @@ use std::time::Duration; use async_trait::async_trait; use bytes::Bytes; +use bytesize::ByteSize; pub use file_source::{FileSource, FileSourceFactory}; #[cfg(feature = "gcp-pubsub")] pub use gcp_pubsub_source::{GcpPubSubSource, GcpPubSubSourceFactory}; @@ -109,6 +110,21 @@ use crate::models::RawDocBatch; use crate::source::ingest::IngestSourceFactory; use crate::source::ingest_api_source::IngestApiSourceFactory; +/// Number of bytes after which we cut a new batch. +/// +/// We try to emit chewable batches for the indexer. +/// One batch = one message to the indexer actor. +/// +/// If batches are too large: +/// - we might not be able to observe the state of the indexer for 5 seconds. +/// - we will be needlessly occupying resident memory in the mailbox. +/// - we will not have a precise control of the timeout before commit. +/// +/// 5MB seems like a good one size fits all value. +const BATCH_NUM_BYTES_LIMIT: u64 = ByteSize::mib(5).as_u64(); + +const EMIT_BATCHES_TIMEOUT: Duration = Duration::from_millis(if cfg!(test) { 100 } else { 1_000 }); + /// Runtime configuration used during execution of a source actor. pub struct SourceRuntimeArgs { pub pipeline_id: IndexingPipelineId, diff --git a/quickwit/quickwit-indexing/src/source/pulsar_source.rs b/quickwit/quickwit-indexing/src/source/pulsar_source.rs index 4ee7718841e..23063c554ff 100644 --- a/quickwit/quickwit-indexing/src/source/pulsar_source.rs +++ b/quickwit/quickwit-indexing/src/source/pulsar_source.rs @@ -42,21 +42,9 @@ use tracing::{debug, info, warn}; use crate::actors::DocProcessor; use crate::source::{ BatchBuilder, Source, SourceActor, SourceContext, SourceRuntimeArgs, TypedSourceFactory, + BATCH_NUM_BYTES_LIMIT, EMIT_BATCHES_TIMEOUT, }; -/// Number of bytes after which we cut a new batch. -/// -/// We try to emit chewable batches for the indexer. -/// One batch = one message to the indexer actor. -/// -/// If batches are too large: -/// - we might not be able to observe the state of the indexer for 5 seconds. -/// - we will be needlessly occupying resident memory in the mailbox. -/// - we will not have a precise control of the timeout before commit. -/// -/// 5MB seems like a good one size fits all value. -const BATCH_NUM_BYTES_LIMIT: u64 = 5_000_000; - type PulsarConsumer = Consumer; pub struct PulsarSourceFactory; @@ -225,7 +213,7 @@ impl Source for PulsarSource { ) -> Result { let now = Instant::now(); let mut batch = BatchBuilder::default(); - let deadline = time::sleep(*quickwit_actors::HEARTBEAT / 2); + let deadline = time::sleep(EMIT_BATCHES_TIMEOUT); tokio::pin!(deadline); loop { From 831ca60750e130ecb5a9a4db3d46b310cc37815e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Nov 2023 15:41:26 -0500 Subject: [PATCH 24/27] Bump proptest from 1.3.1 to 1.4.0 in /quickwit (#4115) Bumps [proptest](https://github.com/proptest-rs/proptest) from 1.3.1 to 1.4.0. - [Release notes](https://github.com/proptest-rs/proptest/releases) - [Changelog](https://github.com/proptest-rs/proptest/blob/master/CHANGELOG.md) - [Commits](https://github.com/proptest-rs/proptest/compare/v1.3.1...v1.4.0) --- updated-dependencies: - dependency-name: proptest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- quickwit/Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 1fadf51bb17..afd62109f63 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -4955,9 +4955,9 @@ dependencies = [ [[package]] name = "proptest" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c003ac8c77cb07bb74f5f198bce836a689bcd5a42574612bf14d17bfd08c20e" +checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf" dependencies = [ "bit-set", "bit-vec", @@ -4967,7 +4967,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "rand_xorshift", - "regex-syntax 0.7.5", + "regex-syntax 0.8.2", "rusty-fork", "tempfile", "unarray", From 3f449ee56b5b929cb74ef638e2fc836a2ead5bd7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Nov 2023 23:25:41 +0100 Subject: [PATCH 25/27] Bump axios from 1.5.1 to 1.6.1 in /quickwit/quickwit-ui (#4111) Bumps [axios](https://github.com/axios/axios) from 1.5.1 to 1.6.1. - [Release notes](https://github.com/axios/axios/releases) - [Changelog](https://github.com/axios/axios/blob/v1.x/CHANGELOG.md) - [Commits](https://github.com/axios/axios/compare/v1.5.1...v1.6.1) --- updated-dependencies: - dependency-name: axios dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- quickwit/quickwit-ui/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/quickwit/quickwit-ui/yarn.lock b/quickwit/quickwit-ui/yarn.lock index 0502aec3e7b..44c7b899f12 100644 --- a/quickwit/quickwit-ui/yarn.lock +++ b/quickwit/quickwit-ui/yarn.lock @@ -4301,9 +4301,9 @@ axe-core@^4.4.3: integrity sha512-lCZN5XRuOnpG4bpMq8v0khrWtUOn+i8lZSb6wHZH56ZfbIEv6XwJV84AAueh9/zi7qPVJ/E4yz6fmsiyOmXR4w== axios@^1.4.0: - version "1.5.1" - resolved "https://registry.yarnpkg.com/axios/-/axios-1.5.1.tgz#11fbaa11fc35f431193a9564109c88c1f27b585f" - integrity sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A== + version "1.6.1" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.1.tgz#76550d644bf0a2d469a01f9244db6753208397d7" + integrity sha512-vfBmhDpKafglh0EldBEbVuoe7DyAavGSLWhuSm5ZSEKQnHhBf0xAAwybbNH1IkrJNGnS/VG4I5yxig1pCEXE4g== dependencies: follow-redirects "^1.15.0" form-data "^4.0.0" From 1d6753bb20ed38d75054470091f62323b0b02877 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Massot?= Date: Sat, 11 Nov 2023 10:53:09 +0100 Subject: [PATCH 26/27] Use TantivyFastFieldRangeQuery for datetime range query. (#4116) * Use TantivyFastFieldRangeQuery for datetime range query. * Add/fix tests. --- .../quickwit-doc-mapper/src/query_builder.rs | 74 ++++++++++++++---- .../src/query_ast/range_query.rs | 45 +++++++++-- .../es_compatibility/0007-range_queries.yaml | 28 +++++++ .../es_compatibility/_setup.quickwit.yaml | 1 + .../es_compatibility/gharchive-bulk.json.gz | Bin 33652 -> 33673 bytes 5 files changed, 128 insertions(+), 20 deletions(-) diff --git a/quickwit/quickwit-doc-mapper/src/query_builder.rs b/quickwit/quickwit-doc-mapper/src/query_builder.rs index d2cf5edc3c7..9532313178f 100644 --- a/quickwit/quickwit-doc-mapper/src/query_builder.rs +++ b/quickwit/quickwit-doc-mapper/src/query_builder.rs @@ -256,16 +256,19 @@ fn extract_prefix_term_ranges( #[cfg(test)] mod test { + use quickwit_datetime::{parse_date_time_str, DateTimeInputFormat}; use quickwit_query::create_default_quickwit_tokenizer_manager; use quickwit_query::query_ast::query_ast_from_user_text; + use tantivy::columnar::MonotonicallyMappableToU64; use tantivy::schema::{Schema, FAST, INDEXED, STORED, TEXT}; + use tantivy::{DateOptions, DateTime, DateTimePrecision}; use super::build_query; use crate::{DYNAMIC_FIELD_NAME, SOURCE_FIELD_NAME}; - enum TestExpectation { - Err(&'static str), - Ok(&'static str), + enum TestExpectation<'a> { + Err(&'a str), + Ok(&'a str), } fn make_schema(dynamic_mode: bool) -> Schema { @@ -279,7 +282,10 @@ mod test { schema_builder.add_ip_addr_field("ip", FAST | STORED); schema_builder.add_ip_addr_field("ips", FAST); schema_builder.add_ip_addr_field("ip_notff", STORED); - schema_builder.add_date_field("dt", FAST); + let date_options = DateOptions::default() + .set_fast() + .set_precision(tantivy::DateTimePrecision::Milliseconds); + schema_builder.add_date_field("dt", date_options); schema_builder.add_u64_field("u64_fast", FAST | STORED); schema_builder.add_i64_field("i64_fast", FAST | STORED); schema_builder.add_f64_field("f64_fast", FAST | STORED); @@ -477,16 +483,56 @@ mod test { #[test] fn test_datetime_range_query() { - check_build_query_static_mode( - "dt:[2023-01-10T15:13:35Z TO 2023-01-10T15:13:40Z]", - Vec::new(), - TestExpectation::Ok("RangeQuery { field: \"dt\", value_type: Date"), - ); - check_build_query_static_mode( - "dt:<2023-01-10T15:13:35Z", - Vec::new(), - TestExpectation::Ok("RangeQuery { field: \"dt\", value_type: Date"), - ); + let input_formats = [DateTimeInputFormat::Rfc3339]; + { + // Check range on datetime in millisecond, precision has no impact as it is in + // milliseconds. + let start_date_time_str = "2023-01-10T08:38:51.150Z"; + let start_date_time = parse_date_time_str(start_date_time_str, &input_formats).unwrap(); + let start_date_time_u64 = start_date_time.to_u64(); + let end_date_time_str = "2023-01-10T08:38:51.160Z"; + let end_date_time: DateTime = + parse_date_time_str(end_date_time_str, &input_formats).unwrap(); + let end_date_time_u64 = end_date_time.to_u64(); + let expectation_with_lower_and_upper_bounds = format!( + r#"FastFieldRangeWeight {{ field: "dt", lower_bound: Included({start_date_time_u64}), upper_bound: Included({end_date_time_u64}), column_type_opt: Some(DateTime) }}"#, + ); + check_build_query_static_mode( + &format!("dt:[{start_date_time_str} TO {end_date_time_str}]"), + Vec::new(), + TestExpectation::Ok(&expectation_with_lower_and_upper_bounds), + ); + let expectation_with_upper_bound = format!( + r#"FastFieldRangeWeight {{ field: "dt", lower_bound: Unbounded, upper_bound: Excluded({end_date_time_u64}), column_type_opt: Some(DateTime) }}"#, + ); + check_build_query_static_mode( + &format!("dt:<{end_date_time_str}"), + Vec::new(), + TestExpectation::Ok(&expectation_with_upper_bound), + ); + } + + // Check range on datetime in microseconds and truncation to milliseconds. + { + let start_date_time_str = "2023-01-10T08:38:51.000150Z"; + let start_date_time = parse_date_time_str(start_date_time_str, &input_formats) + .unwrap() + .truncate(DateTimePrecision::Milliseconds); + let start_date_time_u64 = start_date_time.to_u64(); + let end_date_time_str = "2023-01-10T08:38:51.000151Z"; + let end_date_time: DateTime = parse_date_time_str(end_date_time_str, &input_formats) + .unwrap() + .truncate(DateTimePrecision::Milliseconds); + let end_date_time_u64 = end_date_time.to_u64(); + let expectation_with_lower_and_upper_bounds = format!( + r#"FastFieldRangeWeight {{ field: "dt", lower_bound: Included({start_date_time_u64}), upper_bound: Included({end_date_time_u64}), column_type_opt: Some(DateTime) }}"#, + ); + check_build_query_static_mode( + &format!("dt:[{start_date_time_str} TO {end_date_time_str}]"), + Vec::new(), + TestExpectation::Ok(&expectation_with_lower_and_upper_bounds), + ); + } } #[test] diff --git a/quickwit/quickwit-query/src/query_ast/range_query.rs b/quickwit/quickwit-query/src/query_ast/range_query.rs index ef5d654af8a..16ada6047fe 100644 --- a/quickwit/quickwit-query/src/query_ast/range_query.rs +++ b/quickwit/quickwit-query/src/query_ast/range_query.rs @@ -24,6 +24,7 @@ use tantivy::query::{ FastFieldRangeWeight as TantivyFastFieldRangeQuery, RangeQuery as TantivyRangeQuery, }; use tantivy::schema::Schema as TantivySchema; +use tantivy::DateTime; use super::QueryAst; use crate::json_literal::InterpretUserInput; @@ -266,11 +267,19 @@ impl BuildTantivyAst for RangeQuery { field_name: field_entry.name().to_string(), }); } - tantivy::schema::FieldType::Date(_) => { + tantivy::schema::FieldType::Date(date_options) => { let (lower_bound, upper_bound) = convert_bounds(&self.lower_bound, &self.upper_bound, field_entry.name())?; - TantivyRangeQuery::new_date_bounds(self.field.clone(), lower_bound, upper_bound) - .into() + let truncate_datetime = + |date: &DateTime| date.truncate(date_options.get_precision()); + let truncated_lower_bound = map_bound(&lower_bound, truncate_datetime); + let truncated_upper_bound = map_bound(&upper_bound, truncate_datetime); + TantivyFastFieldRangeQuery::new::( + self.field.clone(), + truncated_lower_bound, + truncated_upper_bound, + ) + .into() } tantivy::schema::FieldType::Facet(_) => { return Err(InvalidQuery::RangeQueryNotSupportedForField { @@ -333,11 +342,20 @@ impl BuildTantivyAst for RangeQuery { } } +fn map_bound(bound: &Bound, transform: impl Fn(&TFrom) -> TTo) -> Bound { + match bound { + Bound::Excluded(ref from_val) => Bound::Excluded(transform(from_val)), + Bound::Included(ref from_val) => Bound::Included(transform(from_val)), + Bound::Unbounded => Bound::Unbounded, + } +} + #[cfg(test)] mod tests { use std::ops::Bound; use tantivy::schema::{Schema, FAST, STORED, TEXT}; + use tantivy::DateOptions; use super::RangeQuery; use crate::query_ast::tantivy_query_ast::TantivyBoolQuery; @@ -352,6 +370,10 @@ mod tests { schema_builder.add_u64_field("my_u64_field", FAST); schema_builder.add_f64_field("my_f64_field", FAST); schema_builder.add_text_field("my_str_field", FAST); + let date_options = DateOptions::default() + .set_fast() + .set_precision(tantivy::DateTimePrecision::Milliseconds); + schema_builder.add_date_field("my_date_field", date_options); schema_builder.add_u64_field("my_u64_not_fastfield", STORED); if dynamic_mode { schema_builder.add_json_field("_dynamic", TEXT | STORED | FAST); @@ -359,12 +381,17 @@ mod tests { schema_builder.build() } - fn test_range_query_typed_field_util(field: &str, expected: &str) { + fn test_range_query_typed_field_util( + field: &str, + lower_value: JsonLiteral, + upper_value: JsonLiteral, + expected: &str, + ) { let schema = make_schema(false); let range_query = RangeQuery { field: field.to_string(), - lower_bound: Bound::Included(JsonLiteral::String("1980".to_string())), - upper_bound: Bound::Included(JsonLiteral::String("1989".to_string())), + lower_bound: Bound::Included(lower_value), + upper_bound: Bound::Included(upper_value), }; let tantivy_ast = range_query .build_tantivy_ast_call( @@ -384,17 +411,23 @@ mod tests { fn test_range_query_typed_field() { test_range_query_typed_field_util( "my_i64_field", + JsonLiteral::String("1980".to_string()), + JsonLiteral::String("1989".to_string()), "FastFieldRangeWeight { field: \"my_i64_field\", lower_bound: \ Included(9223372036854777788), upper_bound: Included(9223372036854777797), \ column_type_opt: Some(I64) }", ); test_range_query_typed_field_util( "my_u64_field", + JsonLiteral::String("1980".to_string()), + JsonLiteral::String("1989".to_string()), "FastFieldRangeWeight { field: \"my_u64_field\", lower_bound: Included(1980), \ upper_bound: Included(1989), column_type_opt: Some(U64) }", ); test_range_query_typed_field_util( "my_f64_field", + JsonLiteral::String("1980".to_string()), + JsonLiteral::String("1989".to_string()), "FastFieldRangeWeight { field: \"my_f64_field\", lower_bound: \ Included(13879794984393113600), upper_bound: Included(13879834566811713536), \ column_type_opt: Some(F64) }", diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/0007-range_queries.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/0007-range_queries.yaml index a66284f6754..c3c625395c4 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/0007-range_queries.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/0007-range_queries.yaml @@ -159,3 +159,31 @@ expected: total: value: 86 relation: "eq" +--- +# Timestamp field with milliseconds precision 2015-02-01T00:00:00.001 +json: + query: + range: + created_at: + gte: "2015-02-01T00:00:00.001Z" + lt: "2015-02-01T00:00:00.002Z" +expected: + hits: + total: + value: 1 + relation: "eq" +--- +# Timestamp field with range in microseconds. +# Datetime will be truncated at milliseconds as +# defined in the doc mapper. +json: + query: + range: + created_at: + gte: "2015-02-01T00:00:00.001999Z" + lte: "2015-02-01T00:00:00.001999Z" +expected: + hits: + total: + value: 1 + relation: "eq" diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.quickwit.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.quickwit.yaml index 24089348d94..5eea75dd277 100644 --- a/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.quickwit.yaml +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/_setup.quickwit.yaml @@ -27,6 +27,7 @@ json: - name: created_at type: datetime fast: true + fast_precision: milliseconds dynamic_mapping: expand_dots: true tokenizer: default diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/gharchive-bulk.json.gz b/quickwit/rest-api-tests/scenarii/es_compatibility/gharchive-bulk.json.gz index 459830d8abb97201031acc85f9b6bc8807fb8b2e..9b4bb284afc52de8f71bd6b3811eecbcb563f050 100644 GIT binary patch delta 32929 zcmV(_K-9nVh60I)0tX+92nfzcPmu>Bf0Lc;F0$C{4U)+&u$P6=Pl;nn)RL4Pk1_xK zs!2U4iMA=)nf;PM5K9!Rx>)b(uHwTaTRQ*C^{`?;glgz<2 z!daRMj0l>%nbh|y57*w6Uf2E)w|-f}MQ!V%jCkVxSPDWpe>2I8S+<0Gzg_F4fB%y% zjW6NeS*dSzt;_2MH(XekIeeJc^{Tp>PW39goMrWVZ7yxGn64{dR?|Za8~7u*=3=#h zjbOG|)<9m1X1by3>&fRglhUt>EwvP3q6ATICQH5Whty6N#d20G^zu0>BfgFRwS>uQuP|2j?qWW~&DD zYVu98ESyi)b(UxKeNq=ml`U3z7VeV;{E#Pl>5`Ady8Q11sL?aOXeG8%EX%ucTAa_!$tY$IcLt?Q@+2=fa>E)f2J&{gx_C4ztWACpE=&eJ5af7hhJ(`CRO$cD8isYSyg9CTVL<4v}y_eucmXaT{YclTl@+? zS+9a8++eH(b14Zt=v4C5e;P`>k`33KQI64;Ww~l zS0uN-ypJ}W<+@DjIozDwfp#P-ST=qEC!!l8uja)aTmayr&O&Az=Tcox_VV7^^=s(c zbR)9qU_pKU=NJbwf6l$o0q#YmOi?QL?xoD^e5Kd%%5`0r@Flq^#+mM;lHHl?V1|q{Su)0CX(zY*uBz7l9S{p<(QCR?5hk$Mo9@S= zsA}hL%dF9~e=)XugkqHOZsmH-Uu4?gsHP;1;>xx0d3DUAap6t!04d-Tf!;m&^Oj8sM$4WMpg^)k1%L`#)oY@W z2RNRV-N`c39pDZLBue9u0N}&4sAKIEZ;6^Z10Q`hNe$3L^9Pt0(Nw_45mdZW>v0&5!1qDxr>d!!s;u_c+AX5-1)W#^^VUy7M)J=_%5}WGSov7_=Y_oi z=iJ^3CRtV4Z0X^KB{&917qT4ex>)Wm?A%yPf3|vJF}|Sq1EE(OUt!Uv&tQ}*_hihD z_t?CsHn+4ZMbC6y`hS6=t6vy2BC2i7c(I#wSqS41Tt@J?{h@0+e*u~XK89RVxclh( zKJR4nIuUj&=&2->QGvvZdH1^p^9Jul2W@lxb;98$#B7nDhNGh_>mZqXQOY$w9vzYN ze>9ViZaym_bz#Scq#i6UiAe|OUKErLFuh(>Iz;=@uyla!+_-cAe12d$0DqmxbVSkl zq3OADZfN+}xWOx^@yq(!-}~w+fhzCZ>hT|~Om`;I!sy4`SMCmT*|z|OU_S~;XxYPIA+D9somR%y?TNFAnJ2x2sL2ir}A z1WOR*Y089FLQ=vg6G(GvQv>KUMN|jUyWPYTQ(B@RxsAJ1LNkoCr%pQLI5A3Ee?}#C zymz}vTf=n90<6Or<|r7I1|*%52I15b!mOm4$hXxh?(*J8wstm~K#J=sc@GLu70bi@ zrf-}}zAZ14zh`;ePo6Ck2gyL$xX!r!^v7IRa}Y=Zu6yQ3ZW|wocDLg+m?xj#>~`%~ zz*JLDgwycEgah9LHpD1Y_!%u>o5nLD!W*R0Z>eIroEiYwHsWDN9yGrTYYRO;Uu! z|FVMO5a=({9}=^!p+c;ZfB$?x`)M_O zKl>I^uK#*vRu1ONh#limyTyE1CjycxMco9Vf+}8rY!2PWMAZ)X$lbG+puR2y3csP9 zI&4>(-gy%fw_&061gfX|pe^q^Ee8!~25h?@-gfXx9R#CfF9zgNOfbbeXL}!g2*m<| zogG^xb`gw{eM?eLP6gS8f0TG9`Ta_rMYjZH#V|#94#_h)7k^o|fOA5clsZK{_DCQn zjWb3BZJ40QnP8;gH1*w_iy>x-Q-nBS2oB1?1;#_lb%0b$8zLwsEzD`ob;g3Hew%s@ zw&zLUSXtwoBvNDUt<^dnHU{pIQC9M|vj}kLwf_vvP5$EjQI|mLf1)8Z$l+QNz?SZl zP&8^f!Ye-`%$=7W%!M!IrR9(=pA>P~X%@THRhH|UtO_8UakhGdVqAkyLsn0JkEAqZ z`(yYUNN4=4D3^K~kKAtk^&Il(>(b}j=B@z(eh$eq*1GHMA>*u-Qbn{QsURu1B1oAW z!-60y8RRaSreFjffBtl>6*8Qluwe#6)Ymq(o_i`KgJb|{ZBmO+Af|yiwN@?(69G%5 zlwjUturp*0b^*+vOU*QgQk(Gp19{M9hrvP_kq9B91CMQe4toj-SI`72A|#~5eb)NN zEYDWJ@Z@h>4LPlmKn+4RTepSLn?%E8{@e1?X6S%H1;OYpOC zhdbFERV2UTNhpg&(qTsc*ER(~ULVNm5#c>4@NxcosALASd6fS?*jXcjW764b+LJhL zZ|wknl*4T)e;T*DmJZonYxInC+P(CAoiAtWRXw(q^bo_if}o|UzaC&>zth%PYdXIh z*fn|#^h_L)=Vg5+*YkZ8vB6SLKwVAT<>8JI14TCSMB_jWWKbrmvg8N*Y?MP?95ger zIn)HpnF)@_fGAYyC`h4%uvqs-kuw%NbxJq{rZBMqe>;h=CY%UMm2v`et5a}H2Ama? z?|n|GEB3>^q1KBxDtoz_uf}9nZItDZ05ykpwaci_RRt8GO-s70Q|XFgV)X5pjXd*^3yW= zrD+KMe^XW`+vO8oC-35*z#o0wOR8R{ejnAG%R0UPe~-uL{q9;(h0tDcTw(73dsU1b zAvr)W&Q~`i^^^oDhoWF#5`0tEKWdXL>&o9w-I2(pwn6G#cTNvg?44lH7)#f8N=jl*QhIjp()uO{*2Tuy^bNxWsrolI^2REA_KjYl~=K$;B3Ae075gp$|_!M(x`11Z$@##|-5(kYc* ze+TNAz>w+slnN@mw-#9qgfI;s-ZmlEn43#&FV{Ew!2A6gRDCtq$?qepKV_c|!|(gD z_~>mNk7L%W#sj{KS$p!okTfsbiHo)(9}$#$rMurAAIg?}NfjW`aDfI*y+%gbkTOVr zQm-+^R0*A?L3rl-=UaYlPC{7}=TGj6e_i=n??0}zy@A`VTE4zXew=5krjpu#{22mJ z-SneEA>5R{o9@&j$PaL>g^r(o4Y3U7aYroN672;e*jlZO(k2ZO8H=TW6hUYym;`A3 zZxqYI!@8~!i7D!N!@)+|TRngGl1lX%ehy{Mo}VeP;BGwlCMzT}+*xHc2lJ5He{w}# z^iqsU_HX3p9~P_n{ztD{1^+epHkb(=Jsj@YzdYaGHP4l-T7UbiXxh-nKfl%)mGy8m zAnQ@_B&VD8d@L$&tNGXyrycaTL9&*bzI0UznWp>mrgoj_pT7Igh}ezwktf zsFZ{EF;WCh)6{W_kn=vI5>YA)f8jCIa(U3j!*Oa6O|2maQRTtX(l}?7LO5pFXrq|X z8l|o`(>Y_o9s`U!s+99MP?efM5f~g4>_%u5Pnm=eUZ=|To+xd_e(0%ez1Dg5NxLHP zA3@Za%?bzsuwWub9x5FPaA$jipZrVN9v)zOoDrhB2mPz;F1xt^(G(Z$e}i2>upHY< z_Xw_W8xSp}Cw7VB_BiX#nA`(jD5avK3pp0ApIanYHbXbP+DlO z1OhcT2pcD$uyBYU?EWE5SP%|1!<7tENY|9ogi~)+FdR98AerF(_IbvFJ)Jr_Fe|Wg z)u{)gG8|nv8udKRCzJnY% z({Z#rG3zXM%~?YG6v?EsR9pue`LJl4kEUoqinOK zW7HW0AF?lk5#gN5&biCGd6`w;`q+;-c5w9w#mK&-mas?Oj&iWMWwp!nyHbbxyxyc< z$Y;FLnI{sb!{$C7qA7==YN0yin1Nk8ZUtOX!NvrM(%_Kn`X%rT5;<*|#TN1gL#0I3XsDT^uP-vg|1i!`^vQ1LCZpm|-*`7l1 zv;!gNWoZplPNtY*s5mX8#h4S3a_@w7HduiO(F!V3cF-E*bU@5m!2(B6#8N0jq!UIG zgPAanDM_Mzs9yJR#)3Vsy$Mj{D-S|(60&w0pw=LUdC1yRE1B^!7y}RXw$XsgCD6Od z?>qdSoXG6>f9*HL$V1I1MA#V;82Iex`;k$vq!0&%N1p%Cc9=Y3oFL4E9wZHpr^ktVp$5(n_rtb>!)FZLe;%RP&e|Sw_35ed?du~p{R!W# zz5_)mgSP%htCyD_L6s(_qq+{4w0k`0R%{B=8~2eFA@%MZKBz zaohL1f30)3{HjijPjm?0#uvz$`}k70>YxYm^5^pBBe;XAzL^>Mh7UT4xb^hnRDCv=4(xo(p z5>V+e=(eC!#Id&8U_~j^(4NL2I7C@&{!JxPnsbb>Yj)n-MEjwAM@l#9Ts4vMBAsm0 z^!@i%f6iFsx6t-mX!|X+{d+>&;Y%zDMJd~#aqNqZ5bJ`?)ehTszc*t2&4z!o;oofd zf6Lo&vvw>nHwEQD{qC6W$@Fk2%9}IxkNR*%6ob-~6X0nB3Mg*;{YVTE@*)0zqww*y z2HM2;X%x>H^J6FvP&^!bjVsh%FGgZ+K%-wOim@DPKzI8V!(X9!ifNN{ZuNSf$i4*W zPMNo*!bm(j@Q3NvAGQauV-aqlA2y#Fe@W`uSV8K>1^qvJSNa^ejotqhmNr#RDwc#B zq`r83ZS1Y}t}`=^6&hAESy|gm(YU!`W*Aj<)kC zyvI7A+vEB__N*mwZvjUEx^;I9>T^jSXYp8DR*KlepfIV4$Bhc*={REfREYm-Z|x8D z3wRWpSr+fn=Xb=nA9G>PBR4lUe;=LMz`>Swr9dt#TDNCZE}$@!j|XadrdWwxd1^I| zHDyn+&LtAaT|<7D>!|IvyK+iXQW#iC+Qn)taiMe2(05UbZU>7n;2y-sueOUq>MzA| z`K4V*yM6janRd-<$CXG_}UMjoWZgG z(P7KB(0{7DW)EWfwi2=Se|BU)YNfZn7Vj3lZ&krchfk{eOg$KMvbEW)YePn5 z=626ekJe$Ty39C=&fj}aQ2wH{5&Pw?O(<3N(7be+sm=jm*Im~PTUmDR{H-lNr<^FBO9C^LLBT0y#%P8z!gQKy$Q7!>7LIMP)kw7we+MJPQdb2ltQK%Z z9bv?2n3>~D_BG$s*79K6O^-gta8mhF@+jM&5;q|qf9+5{IO^@&{ zWidU1^){Ct(TCbff5*M^F^1A3#4s!A5lEzo^aw4+K6*TDp2pE5@;J-r2w1RLbObBR zCOXDcut9Vx*dxdqdIT133LSD4ZU-G`mlz}HF=J5{&~dN;^XK3Zqimle>nq0aIl^4D z)pJ~@#+p1wR!X?NbNJZN#?B+i5KHF~#AVE!N1$Og&e16ie==~6#n^>gH;-TXWlWn# zsGq{FIZ(#oM$IFra~93g6baaC1n&^KBY4`P3135Mzi@}ZDT^k&!|aZtGZxK0ljqEt zqu)Mb%N!r!lp%9)ME9~{VupT~R!lI{iisL4COEQUA|ET}+vTEgARn)`dTHGMA1a%r zw-&fXE~&*`f73=#Ca3L?R`Oe?*=hg>O^!#~RBjsy)kgUKHi&SJdrv2jqkxB7^lzPn zxoD0T6~l*9Y#^e4#Pe^z9irnKi79^-xl7Thq^$Yi?IEv6S~Xf+HF5R%hhaZRFfy;8 z^Tu3cWSs|akff!qf8+LhRi?pPmwn*A?cKYJg{0W=e@-8d6SW{Ifq-?Ae&4N@?*om9 zHECFoXvN1MrH%lE7;UYs_hR!Q-*1#%GtN78j0w~73 zTh&0n|3z#zi{NUVeT1;0j}=!_tyWMcZ@wAp_GeM3^+x=lmydd8KkjMrPt|``Wof_S*c6zy5 z6k->=kopK=jay2lhE3fW!4$SNgP-32vQYQh?pXFGo&Tp*#os<`)~m&SHE$uE0=T?3 z=+#x=83d?PJ*i>E`>mHH6}seMvRm>KR81kBf7c!ep%jDmfxi*ky$mg-&=3dq1uA(N zDb$v_e?harq}bSl?)LTvv3=L9=o#y3gzik$1Qb^_0dSp3E^>PgN;4oe(}1GXP%bgW zL~EWi*D+6y-Q0+}It0WE$jJkkly7Fc~Y>R?mE|NP@B7(oXLtx=0@FZm#8`)ZzYB6NWR_ z6ib->$o|C=g7kaV(eHn>SGn;Di5Vrl+l_6v(V>;#542&$5lco!Y9kPav$XBSRVK<8 zV)?zh!&2a~CrXujz4~x%*6-)Z-#L)Je=xdxu+~S1mA3xUfPt<dx@8xeX*%iP~RF%H${a>wAB{|HSySz5iqMf8H?w zqae|d0HdgviU$}4_l^n}MG1`!7zMjPgup0nSe(EpRAjWkC}vE|!1&a9Mh=XE+(*R^ zOhN`n5lkY6#S%<27#vYB3K-inDd7-upn-ry8uK-9s+Lr29P zOs>b62!u&SqvH@x>F?NRgvm7(e;$)C^&+Ap6GnkU;uA)JE)%6N3LO@!aAMknA{IvR zBI6dur{prx3!~VdB8Fi?@rOq;i~`PxXXsi|=*f5na!1G=aVaf@eE_Eo{MTY zL31Xy;nZlSA{-_zBW;{RtmJWsRhX-VF05nmolaH_2`_W z8OB(S*DCiIaHNB(2ASrme?f=>1)Z&2XDK#X*w5{!l0$+rp7AlWc{TApt2v`2**&-h z*xR}rc9_Y0IBSi`eD>phE|WQ#A#fQ^3GcQo7OVBPSm;oHXooyV>}r07rgJNDkdU_O z_hBIW`qj)10ezA)!WP-EBqLBMlxdV>1Q{_okU7r@Vk|c);!vf+e=uZvmAz8{v6VGV zfX2YCcm^R(L7rJ6%z#mfwxP>4_f0hWNNT+vi0l_@~D;taBsfQ$)A z8Ia<|ld!LUy)JGFy=&#|>!nx}$xrpn_dBRB3fGyWjQ$CexZbR_JhF88+s%z#8KpcUcuglS zG!Q&AL-2AXm|!VP8Hd2CUXeqJHIW$>DW}LFPASg&f+w+3iWv;FhN+Rr{uv92jLMMW zK)6p`=brIh4y38cXl9_&Im?k`!la5D#Tc|JQ%VhmG#a8&f5GF_7$K0#5zJHyGpbXg zbHzB92uaLX3N(d;zA%gZjaK_s=KkG<;{2O9at>FWI2@C%q;FY`cTqgWCe@Y8l>ruo{l-q;>p&m!*c7*3PlvM&MK z`wN3c7>H3WHDDA;Dosv_O1UgMD-Tti$ABXTc|`?be@aq2oPv-zC}Ed$4g|Fyb1I4C z0-@1ng@@J9lgg^kfO=EBDe51?r18WIwoOCkKKkbp%4~*EB8oHAy+uE*j8*WPMQDe@ zE`5+#)*KDQl97ypPOPjmm#nl}h}-hN?$>Mm-U4mwD1te0j;Ars$R>=FO^WOh6akun z93ZWke__yGS91Y#tKRK)Xn$4!5!97ULsE#TGLqRFB;rVLV=OTuNpLfYe|T`76}fpA(AMeDOH*w*JssL z_XpIFQXzb;RPu$DL5rlX9gklmD`ydNK(BuO?r-*NR5v9T>m;d$4~uRFP3-z^mPCI;VHg#f%d!YAG*xA5x*_(Lf1d4 zf4k&IjqJngr`6$d%Zz2OuwBLghs{9UO_=Bt``SPnLUDa z)Y5@LZ?#l~0e8s5dZjvcLZX1}`D~6FW}Kznyprv5zj?g2zlHW<>;Z%+;)8qhV((yeZ-{syB;Mf4%k+$L&@u>}HJ(Gu$3Elm^IZPymcTDFFgu zo$IdZwHF16Gn(6rf>nR^JJO0Bv+@ThGgc%Oa0b0}{8;2M;K;EEn2cnANJxaFhS|Y1 z%biWtC~7#NA_daw;TPSPc~h)!N^4g=ORwC!J!<7#caD`wF66uYWAe{id-YTMf2CIc z5~fBbX0y>BvtPsKIU+&OySJR~tLFM%GSum-_OMp>3mqz-Ep?bas<;`m$WUz02yLkT zdi6*s1J~r`bsYh9N))R9gW29=;bJcbaqd`5{{FtdTFP(j+ z)3Xhn3!e-aq#WaPPq!e{nLXdaf3UrK!g&r;hndj+thc$P^@^D28rP`tlM-cp9gmJcvu0?a#LQBmd{oXdrpcR^$0 z%{y4WLFXM(|H$(W@nyo#d)x#!iyIF==~VWGQ{^PbezZdNB7PdRZ^?-{QGq^`w(3s=VhM(*~ z51UVDk2%lbn($$+a)q5cHsvW{oZ4%IHk?wKq4H9pKyC;tgI?&#ggx|x-|b<+LPAh+^au%89{S+GC9VEbie zPResXH}gm#_+?*Cjxjs`avZ@s19Jihg*f0vy(0UDm4In8Bw zhUTPJi^~TAv}k30&-SW zY7AZjdI!@xe4@`@e`RV<@6fqEdud&``hGf#bDY_8d7LMy&SY|)9`RHz=j7F9Jev~) z)u#UWho7%Pk~!Poe;2u&W9{|M8@10^p0m^_1U1bGGb{r+LlAQ5Lg|qtc@7mz?Vp(( zvpk<2yEa@wz)6N(?pau*cTFB+x|+jkYz%?7sq4H$BHRQve-}H-G&aYh5@RwukAFc^ z+Kj=XO>F1K@HV}jhYT>uod>;;scsJU6DGXBcHaI>8+|Xd>PT|CHhSNntD)`wSv5YZ z#%I;|tQvo}s?pQS;j0nPdq3)NFCVY!;}-D?3GsH9_`%1!QT)ccn0v)a98>69#L*A&j*WhZv_n&;`Z`=g>mjMfBA}Xllebfy!$L@PmXr0 z`Ab5CyUqKx#=GJCh75Ag`3-R)x1Hahf$qGJB^l=Ce+vl8k8uBaE{Y4e1^q_-B<@1r z%{;=5=r`1C_aeT7xb9Ya!o5gf?SdS-7Y8Xg_b1$oUJr5B?HGb~&K()V;Yl~8uL28UjAXD66-+X^2w5pEJH2VI5t#k;~PqN`qC;g2Jw7J8K!qADkZ zJG@$MPTRcO?|HedYO-+uR39M533Wh<@-!VlF6T~a5MzBt2OwX^(1Fh1`*WbYjEkq` z!#_)n0eu8ihwayD9`a8LdvkIHR0)tJ6HKi+fpCNDR_(EqPe7V8zd{iXUKzbI`<4*!ziD+bhxIVrIdp zF2Ph`nQEb3b-!9L$xG=bIVjMKZ>T^RJ|tVQdR*#6uC|H22qY!YSIJ0JU}x#|R(8}njM(j4cFt_~?HB#_ z`b#!(2-~ZEA zUUC=mt3lnRS%pxl^8%hO0Ryu;l-{AKe-1>0GeD;VOf^bfW#-33t&1;qiMuHzxn(4I z+&G;tb)``jg|vFiD0y?Iv> zxHEzMTqjRDcQ5g)vzc_ab1oQyW*02y>qHDDO;jxmE7+a zO5fbvyxF`digl~<-0{t-N2dpOu2Qo7N}`0-wZ(du>^2`<>w}WWeZ%e)$*ohi?uQ5Q zt4@?&txJsEIyqRLze@i1pjt~vfA=@BOPuFo=iqZ)a#;1BUYkTDZ`@7F(%9kbr-dC~ zy}9ac30u9rx%w=<%IrT^=ga$2N@$jXPI+G~itDYtUOxnCw;n*49jxMM*glD;H08W) zf6xwfTW^MME34qV3)?jL=^DQ{vb?&?bS97x8mF8wpa>!oNJID}fuPt@haj4>E zD6AQQ8RW284)r*4;AdH?{)^~#TjF_T0G37eTsa82QsiAp}r7%n(*FR8a zPP;-yAe0Dq#WcmlNV|}7kzuKkD;>>+NI5hNLxm|;24M{OsF->Te>fCVDkw{Fh85zN z8*NY~dB#;ran2}U(10wpX!u1r>oIMO7pH%W>%eN znj3{srkKnX0QOx_nKO-Y$_>*z&$S-kJUs>+N+nAL%5V-9S0YmiS>2EkI~JBE6UJ^l zFmrto)r*JqZrC}fUJL!w_~q`ovJd7?qxRmSb+NV7iP;yWf0_BqL~N5MZSVi~|L)+r zR_<)NTC0Wm;101ry!!K-;`PHuTe77}7Dai8eYLWO*zd&lAz#}=>rI$=Pp^tb#Am5Td2DnIPfPoTgx)_J`cZ8YAa}Eqa z3TvHVmO{x$f64?*-MQZ6DF^O&-&11qs~LD9Z_N&34?RQ?KJ}njsOBU&4oSp zJHX0Ro|h+#UeZPcXM-F5#TU=f8`l+(i~G}N1QYrAw6b%Hx*QfKKV zWzQlhz%y-E6+f~l@*UrMeAI|R3@vmVow%Cq+BOxMI$W`&`P4^2DmFrDR*v01SMS;W zg|?8&f1*Inj#6D|q5Q3-gjxEabgGu{RSVZWHS72F=6Lml?V%_BpU3mHH5WlqFT&8e z%JdKEElU248XCg)WcCrn3|>B}5rda56U%MO7I*r(K4zpSo<8uqP}ixC>E;WX!l^Zd zJJ1yFnrkxQ5@^#$U_nxG?_f_0%)kdf(;_+*f5cc*17GWLMc8u*++9SMgRkrKlhpsD zrY?0xHQ!T-rw6`O-7qxN6l#cVhxh^91|3lAIMiKRHQ_QfJyadn52?fN5@TFbn6yLN zGCa+q-lk(96S*d~T;0JIF$1J(wrV5G55Hp5nP2{Zf^X9F{_;;*dKISuiZA-lOQ{w| ze@}Q7anAI^Ls^i|$|w4y6#9gs&+J92ZE*T2S0V_KL7WNItiFKgntbT(-N0*StJMUs zQIQSTiBd}#yQc4inr%`^7W$!vYz-lkzF>`$aj!DL-^K(bR2P>=C60wv|Z{*hM{IThKG@<2f9T3x})ksL)cf$BjcVv9sU~%&f8j#ontW^f zy8azv$NZu)d{`+ulJ9q`!n2O6~=>Yq!17X01jv*sPjn88$UDn^n;Cbp2gP;vdMm z4LWGG>v4rs^g2_M1yk1_S(t{AI<_za8;BfTn890ld|`IJm~1quf0<@N#z%dGK{MME z3^*lu0!e9wLj9p9y8e`pX_P1^qlV|jFd4SEBUE%@$b7r2OLf|oqpPZLMl5eFkr8`3 zriBSR`ShmT*j{7t1lchdnJn2po>Z!AA8NmR*%MsZq}e{)*38*{>+R{Y{r3Ch(4LM- z;sAu0pykc4{qnZ8e-rp0K^lX;yP*EHr&IsAbHHNiYU}oh#q+3Q>}gIZEhiy2I^$U> zWH-x;W|GBa#R-IzoM^D*WJCkAqkL#FrDa0{wOKASEd^vk({?9$&>72iSuk^?$(lIQ zq!~~zM^_J)$dRUnto-OfDRY(te@E)bXH7%)>M<^Ff5Gxduo9p|;bY{6NUr#0 zp?ta{Nn@ozl00=)=v*mL0Y5o<%9a1V0H};LQ<78(Gg3obLDv5Tr0{P))-+{g1t({l zTnTe+^n8!oXVLsBqIu`@PVeucT)6`gNvL=X87W(3Wma;Q z6DCSkG!0Hmf8@vLHZ7Q_^~?MFsCcbYMfz#&hy2W`Y~oaEbNfuGoGFzvrE;cJ&XmfT zQaRi0XS@Aux1TAMUzJjM75BkdlR@FBj_N?>bS)&`r5QScpb!*l-S$FqwHnb#nz`p$ zs@tBcY8d#>C%rfe2L=Y9a`r{%mK{jPMF5Ff82?$WAd5RwySa{kQyM>bPd&Y z0>gDY)ORw?!<-;hq)7)X6pv+2Dv)&5gmpZdcJ3Sj!#Yd0EGw=vV1rE>g$-Ucbv|qNVV&u zmrC)3f2ik8Alvz4d5bbdr@J#CIh=^0BBg-BO)w2YX%#cixyh;VY5!6Yz{FtZ7WgI01X!zPn_$NiZQm@PrP6AW&kiXzzzRX({+@7;&PuAqYyNk6^D%HcF*y=yfA%FoL`__(O^|X!&FiFyg2Umv`n@M5i0Oi4 zrSssw|DfMCEn$#RuJG#WZ>qj7{TK!Jtl}0wp*j2x%_won3YZKV@)D`FS6qDwI`%iz z)s2EXB`B%9ON#iZ!K#iX4tu*&SZM8-@qMaf`H=()xY;>B?4FfLLVJ3{Jn1x{f0$7? zHT$*7rGPN^Rx=^*>F1fr0B_+mC}C?743z>$R!9>0)Rg^&N|9k7jpd=T=^F=lK(I_w zOdhWXlh?*Ms;^FekOHM+QI?iaQ-MC*n0>=XltpQoE9H@*!L_weXi;Hh4s#_>S*b;+ zn7BRFVuOiwntkEzDgZ|FF}yALe;AwX>Q0V^H?+wbFQ)4s$#^v4wpcTi`RQ&xw#okD zt;8*_UV9mvs{)d;wknMM?5rA4*21cAw%S%zv+Y(@)o>qssu{(vo_&GQLvp2j)iv;x zoN72IjirF*$SPgO?*IAScAnHZDL0}+X+qCvnUPT`vw2^$0wYsULPDk*f2dGN5o1jy zOHFeu>H~LXf@Dt*(F41iRz#|G+l!S^;UgO1c^%KOOts~T>YlFsG`+tGGDIgnTS*U% zw_RX&CF|xnwyyb}PK*Fks{`A%6X3wnG&sBJsJ7*srmJD3cG+FkwTv*ZH3R8pfK04; z$gnNL);!P0o~~;iRdd9hf2d};zQd8@uShjQf?e#|wi+OdXin&a*z@VXI1EDKt15Ew z=*hTt5%kDhZmLqF5tF(s9xL^%{7;FXI&`Q!lyMOq5tQDKAjm@PrHmA1mQ?T1H!_mb zU#T&iz>!!e_s=6MQO-+6Lpf<^EXE0sFO0VA)Y=){P6sRk}{S*!-r_@|Nu8iSKH(%J}j@%wc z9#nnREfm#+v8s+R8u46 zn`$-mHage+Vmom1$+a<=wO?6i6ZQ-R+cXWw?rffRbYj*)uf1S>SvDjglfdkRe&14; zbW~ipwx)Lsf`_N%7eP@`H5K}rY!t&xQQf8A#BrKc_&}(vf2JZo0XvrN#RNT4yOu?A zR;{*~_I*JC_tWyUAuduiLv=c}%|?0;fvKy9NoQhggv^ua`lr>Db)JL7?1}rP)iCBg z)9OQ;{8Ba%!d_{owTw^7V5B@!hOnPM%Hd?aQHHYB7p+$2R!@{Q+s6;B_r_)~lywU% ze#O}@uhy|Ge~3vl>$Kq?o3}2hI=fTerzyoQSoxKAP;2diUsb(9p-pwnZ^{-0xaJpT zzqz<4PZB85`WgOT?SX)9F0k+rRR|KWJ(%a-pU_JIY^OLnYYyUxn&=WRta-ylt`A_3 zqg5niy(j8xpSMKD$AOz2QOBetuo{Tx>so5^o781Me`|#2>Noa;(XGq_MoftLaq3H; z(2rd+i05l+D{~D|e2^O%U;;u968r%BA%%}w9W6=Y03 z`H~v@5||A{rFw7CFN;RR!Wb(ICdZN@i~LenjI}7OUiamO*@B#y3euRzO0!gslFavJ z0KG*ne+>%j@)USWZqoEnPbDrF=+!&Y61|C#J-0nn*0$u*;;8=srPQm*IHGJ-RBUoC z0O0!BN7O*Ny1IG>Rsx_IaV{}h-X3%i_btVdo@)fC8j1>LHHR?7`o=ejQ8p0u8jx1! z-Bra>B*Xz(^DbVKQX=zQ8jLP-k`_@I1#3Kqe>w0T%=7Q^C<(|L6vuDqA}gXGqU)A= zVO$%m&1IZY1^tx8>XfYnVvX>8UD>rOc%PAom3FCn?^WkuSqDw!={Nb^=|@!Oz3Lux z{a_gXRS8QzZu;8_9|XPTkFcvm)hN2`C9Per|3TF{(WQy zMBR+LRBcZW5ma^NUz7I*cNtfmbw0;^Dfq4PesB2f4@^Q-#>EG2<^o$&v%g)Eacgu^ z(?pnmvlHF>mf??I8K7H+Zg(P`A5-#le@{1(ATiWwF2-K+@r+zR!w5CIA=K)Qu5pl1m5!sm;D{`#SsLe@B<6ZZs?nLXB#$K2-%Shxu(IV*Hu2dLl_kit7w zZs}L~8|Ba6|M(Z>o1fmj`%5i9vbtB6o$@-%#kg2gGr+Sw&+V)hgriclVrt!1f9$jm z8_4jyfU9Y$+Nr=v8y4tz{nLFJQ;C0*Cbtl`TenmuTTdJe0NJqlT)as z?hBcioV*i7RJ9Z7!A-#>X?v8?kg(^;(O-_fe6jk5J4WF3DME z2%I)r5&3N+&}~oMfN%YMI|CWSdOLH{n{ufM64x@F;mj+V&IdQ?nr2Rf)y)ZljK*o8 zq?AsBvY*~*A!SuhgS1uqe>4rZE1;(7K02r~l9!GTkL=jKyZ0ylIy=yjC?vU~-XX!t z_Ya6_3*A&8*NFuS;D`!4?o;J3KcIV}Jf&r(kk3^1M})TH=Vf?XKZT<+Dd-y@tx&>G z4DnZBpHRVawLR`U@`YB1oKM?sjy4Yfqyx7S3k_UJ)QRLwx+!#O}0LaCTN8yCS#-nIr5##ZJ2?vV@aB^|t0i1Ynl(jDNwL^O9qV2cNG2VfgxxKs3xc4=tuHSb{P z2i2R3;p#4gc{TN5mYBJONh!_$M7={la-p4z+lCM|Imq5S%`jE|QQ+_mhN@w%lk)p#j;q|Qlz$}DtIwn!vbTjsT(HrF;9EX!Ev1>M~?GK*X4UW)F37O$n~et7Gx z6ir*HODRrKB=4kX;bgC*Xpwfkk)i{;!-W(rn!tS&ZR731z+Us+ZlmaqXnRnwO7-}O z6@C9Xf69UVKtHoROb~_|BtOINOj`U5Q9D-j9SXC3e`8;Nf8$*L{^mcg-$MPrygVM$ ztWX{!q}WAjT`6rnItUVC%p!%Az$ns6UQj2TWsh{l#?>;t;jPGS@kX;-7@;?Kp`vj= zurd<-^;Ow8E1BOVHC3~~Y#BU8(^|B~`0(C3`1VRaW*+ z=5ffI!j0W)*!3~;-3o@_y>a!IjukTaJwy}Wy0cV| zHC1y|kM%1nE|1qat4`A9+vG~&mD%5_du`?TI3xnO?cN_s`0d_%Q)GK-Ep1&(UoZjH zfa)EE2@)y|Fbh%%e8~{oOzM=rQgB3nlh$?rDmkxAvGuH9>>#As(CmTT>PlNO=$VU$ zW_2Pc$7Td>66u+O=+ZfvKuI!%nSv>jWtm=|*vc1Xc}EGCXnw(qS7`Rb$TunA)L8-S zAT+5_{~#xYonPw?v|BTrBC+CM{INlKQMajq|6}*MWt`NJwni@#r4LMCirlsK%I^kC zUzkdHo_#xCt0Ltevpvy-U*>zyk`P&E0yK>F4wjI`kTZEwWH2+lbv_H==AQ21paYtu z#5qzNAe~7n7J61$itGpqOQP7u+wy#c@}$49HoUeuGQ~HbT2%q zjvN=R(WLlRCDaLC?sr~~VYX>XU5Mj?m)=%nQ1fe9K^Bl?ghCsHkk@p>J996gWaL2) z7t^^}`febDEx4l$Qvz@@vyOEY9zzQ*lS3`hHt2@SO1j2D`gR><9&Fq8HquGv#?hqi zE!P%mWR!nQ*EZt97`k1d2?2xEE<`L$Ls~G6_?0XDNySTAW!2tVn~vP+D=pihk{22Unfk`;cErP_t+((|?ySJ!pF?oouPS2wYE0hH;u%o6 zz&nxPxdXdX{_;TK$a^r7Butkts7)A8yuoDSUh@4DaXX0!Uc`sI!2n`$x7mDXr){S^ zBaxR&hG5O>;UzM{2rp>Wtlb>UQjT_B+9Q*+tMPoP`3;yd;H>Az)>NXV)0J}AOkQ>! zODZqH^U3=)Cm7kRp`3IiOK>k}?Vz5VfsN4493gb#np5-uL~Gv6M=Ci@dA@v~*K^9( zRD#OVuKJLcPt&Gf-r{tri#kj?} zqER_+U01B0K^)#M!Wd)$kDf6}oPl1qtQfRZBKym8n3o*!TR;lFjuXX|bSZe_RN2O& zfqqA^m>aOo!)q@rJm)2AF0;?&;_zI_>q2{o^C&&i-Hn3wW5e1e0%N={x;u_qHc453F5Lo}RLb2j(lSlx1dj_9Bf@OEy56YXr%-xjZIT zgQKRZ`@+Ra6EiA)pJcWw*u>IR;VD|$KUBVZ-mde98fzD*W;omoBQq-50kZu579~!Q zN&&?e*xHovb^NWovNtowwi&|v%Bm1(3v_awCzCqiB!Nk&0Bq$O@GA z?DQTFDq15Fet$|quUNH(N_r9uB6mI8XFhP*v3c{1QKW`uz&8d1L6IG zj01&H0cbbQBj8C_*Hl$iO%(P_NwJ*4P0X`M>e9g4+R{3_2w$UM8>1-U5+-)R&y(G%f z9cqkxEo=*8PAxUw)4FiWI9U!jKW%;3qL(teGD)5WWTyX908b6*(IXi$Wm{Tnp%g?= z072v${S|v6vC*?5k#$%AX9rkjY=)l6(Q;}U+ZNDCXqwr`-AuXzXCP~=)C*jC;hvn| zDW1u@8K3*g^U+4+-8@dp%$xQsxgDk{<_^X(J(DTcb&rOOzCW%5E(9vvJR@rljLI`n zw2c-rZS#EK`)#6$Y}$tV_zjeGzoa7<0i90`Jc&P;Fbh9Gm3QmVE6P#paFp)S?$Bc#e<>K8s^d}(*FH^@E$tX!7u z+O`|1))YXDM;qusnBxtST2-Re5^o0wfKH}}m8MSS!f)t<4Ut8u{S9YuwEku&cuK&<+87MX z?gnZ|a+k}ZG@a`OS*qHJYf$qMa4TE4VX=*k=b5NYc2{v~qf2@LVelW^OE@Y+ZYlbF z_iF3vz=@TP*=@pk=?1QC8hAQYZmaN3sN>0qrH)v6PY}DuHxML$7N{*hzN{Q?8Y-L^ptSs(VR|Jjp)Yuzx zBU|(GYY&Jcprf9TJPlb*nDvw+v$SPtH(FMj*@nZCeUV(lx?~Q|#%Xf4 z9VqdD9Dj;8>&zEY-&$q&`7MpKd6qxRPCF=MUAEhfPyDnmT#gHysTBJJAA7A)`FnmJ zy>-k^al@2(-9JgbwB^Nxby2=Q)^`RCf8bPcd}_?R=F(2LWB33HaA>IH?8bmi?mI^* zE3VbB;vw+?#Sc9e9dKX z9us;b_G}K)`h7N8Tj=nq2*O)s{At-RUVRW7chz$QzvpU~shm03N^yNhDw$2T0zQNJ zv<4>2KfGZ-?{lkG_n$vU)#n zvxV&ZU6|d}o@3svpt9d-wERh&w>K$-uYbO$|86^Wcl-d%OwfEE7rFFhb=!F8efyMd zlH7f^hSzUL_WHwb!JDm*`>;jmxZtN1UGA`^EaN~~bt>*)v zZpOEDTdt8Rj#{0fl(dE0W)z(1+XSpMvD*a6IRku06ue$O2K*r;HJLJS6Pj-EQWEO; zG`0#Q9WH>J5R!83D7YDUxAn>V^0Sq7iCHOpI(K`a4nYP|^lbwBY!>H06D{x#W9-C) zSvW3`hZ$K#ee&b2yf@N@X^0Ru-uW$pCE`&XvQQ{b!8;ftx#Kh_pZ+(r3Z8YF%zb9g z6Ryb9g-P6AFS5oh(JvLHH-fV3%Zz9SNvlk_)Br$cQjp`*OD$3E)-s(Jf<;dmajeT! zJW52GnjSPANPH4uR9TCJfoVW&0yp@NdVm;GIie-ROG<~>3>QD|LqaMM z-fw}dC{LUOi~w(yCh&ax-{}{G1gsgtig6K3#S#*%_l_(@SYEgyMKhB~Dn~c0KhT(&eK``x$uJ?*I++pYL^C?!v|mo=6&kTK3xvL-n=FcIKRA(UScj7JvHRo;HQ? z0Y3AU^pEr6=&5W$e0R@;`6$l0R@;wgbG%fkiC+9KboxDAXAf^&*FmSadXRvSEr z%(^*udu~o%RgEyPvhG~!_quI8MD`Z0KAN+Y|pnZR+ z#%~yILv>tfMA+1%(mz+|f4j3wA)K}A2aewP(-ZOPL^)82TM{x&Vy}f-`BYzo`bM1SKEY0m;zgdKWYH#xha#~R^H7#d(-KbMGzfawa)p#F^QKrEK3Mfns_SP z0}$un+Dy8T%SwGbZ*298_J@-`>Y=}xZ z4VRGPbxq+$e-i;DOZ!PVIXL+Bw|V@b1iZvj$GY&@bHyuIzS!KSi$Gb*Qb)OIp#I~J zKa!kkGg)wxYd?0)hC|#*04WSpqda~&R-1C-H?(3Nu=c7ljKCw_f z72wTQP9SgNp0Hv2W?4x~mu2ohX!G(lYBTGbR!fm!k8f5t_?6V;^%pZ4=Pyk^24iVZ z-GXo(Wp%@dJ0{38l3Z7VphIx+9}ekJ9@R9b>Tbg|Ld)i1P;T<9k};z{mXK3za>-o= z1MudSWe3pfS2mq!bW^l2ze&k)?4f^n|l!iImD> zfYW7XJUV%2yGWCB&;+&a zCL}|AiZ+`{FdA`3h!sJj0v>2Mj>W$QMvxH_Mg&bNeG#{N>wzvj685)YinrI<>iPeC zrj0Z}Frnhl1lz)^RSUHm;uRB#@FBGZ_~(C^m98#o7FxN~e2U33nsKjAy(;)VnM5<% z4Q|!oSfA8a5rbkP-X%B`_{-%#)1l?FD*{Z4>sEs(jaQ`nxSZf^--qv$7gr*QH~Tk$ z*dNyDecd$vDS@cNR)onjsDBCsFa;aM&+q3OA0yYE7quGzYiDazj#Q&1!gNXlMO4`_qPb!6uIc`wumh&f7Aci` zrc^?XUHiqH6XYdQO-Dc86g%VQVf@o)++g^fIZL`;$#^q#HSUgueb5I~IL|%NObZ4Q z&i)p`dGZjDu`!WgPt7D652j91AQ^NM=7G#K0R;juU9gY=96HV%Cvqns`w|SL#YK}L zX20XCdryN&#A0s?)l3}>UDv@JeM^h5Q|M_Gv7BCPH^vt_5uC%0NEHZ+tWS}dh&1bf!6Z!49BoYR16q-@?3=15u(`! zg)OFFKeZLb;&%D_zo}G-tJ7Ur8vpvMcbKq^EKPwJq^z(G=RgSiCM+PnCK<;~zSAd*O)ss}G9`A~#~%N^{2c6Y=I>o!0QON* zY?`8&agx*~M&A`>ZQZ;MAxW0gl)n{Qf{VYTXdLzLFixE2;vAS#4&fC`p+aKC)ZjfE z+8%>}7PLXQS&CZ#PZS4aZ$D7IK~Z!MBwRnbCc4Yr#7NiFYO6<^rQPlc)4Ix5Z*Xkk zeMLLdLnu0N-K+>xv4YW^>`!&|aiZtMQTb!nb9sYuY&C{x)+*}Pj{AdGh0zUoX?Hc1 z=9w=7v8inAq$Zi$^~+UbRL=PZ)^o95R~PNfU#leZMwd4LOWjzEx4v(qtx8uN)|bgi zq2t*b6+W-r^xvxsH};>NQSc6g*Zw%a*A^~vyOi~9{wbQfJe9&l8Q*ak^pP@(g=I;9 zOJ9Uab~`d<3^?!SUTBvX9^>I*0cyFbkiKvtDncgnx`95z1*ReDfv~WUbcG>q7i#!a zOVcFd38M-C;hf(^SmK(56pe-^8JwVu!f;2m`wtYiRg?3O!E#qpkiAG`kdDHZMRZ#@ zT7khWohZ@kG02~%!t`yrfr^G>yGWgY6KMc5jP-w+?#uY5k@4O#&gQ9p_Fe%xW}Ndq-xZ;=#*Ka>|U5nkkm z20@(w{51bxiAK0_hiTk|-at>5q6CNs^o>tck@(Y>zaTWtLeKF_bCqiKmhn_$L*D6X zq;{G7p+-ph{PL1Wy2@tfg({fDT7bl_o0+yOtaDOJuZyD4<2e2N>t0 zpaY_Vox-8VsDt=R{zpgv+Q__YA1=Y4hmx4SKa zhbP8<(*afO7+1%q57^15i4hT6W02w%k<`+yeNde~wkNW3Sr__3GEA#)8q9z+5R1d zEA|L59Ls&ugszFDyF@kHKk26+0y)w|k}3uMW)Rp`nut0!VycaaxW`~5LQZISD|ywp z7S(nmhFO0z{qeO)Mr*p&f?%3r5wQMFGi@$gd_4}#HrBV7-igI;ux6`#;-#}m*h8zK zGqo`kTpVM;gJ`S;R--!wl|p4D9)09^n&W>m)Q`uwSNukSan}#SlR9Km6b=RdDSe{T zYe{X%kk@T6%Xu4OIuqp|X^v4BDKlg>X;>+>0}!HQLKhh=iblZD!Y?DD1cZ+@g){j_ z6h)98AD`j4(gx6_J~VcN7TxS;9!#H9>b7K-Rsy4hr3_bXJ zHpeMiz{x{UGjgqCXJ*&Otnc*;MI?EO{rqZR#n~`}BY0tf%L)DyXgR0&NyLzm%))r+ z5s<9aXkuS0e%seQI{}_)1n4z=3# z&5m>=gd9ZWY}(BN+{MZ;H;fCv`mk`dKs({tO_RbrCFK!R;T`tB?+|LwD1kq2{ehN( z1__8~NDo9FBEPj4O!w>j5Ra>df}l#dX4;pd{(QeoFJnsr)vk~P7)gN3hXgdE2JRcw zb1B${EBF%{_U->$K*-84VGXtUPl8{g$IxqPIa6lx6 z!9^MK>q0;~VnOEv!0#)485qMtqVwrIeVi=~Je#G{!~h*j+yvNd^Y0jGBmAX7aj_~D zIkIumcdWtERv%Xo9LwKo!hnWrk1_T%wCqbCwWBaBlbMvpK>t16c zA>^+qtLp~i^?tH2LDOu|V!Kc9PdG45r#hrjqz_`q0Ye=Khc#GxBQtSEVjcQ?*Fz zDP7&=VNI>yPfSUw%PM(7hfj+ZhfjDJaAbYbP!!xE0D*q7)(#Zt+wz>B^O2qCT{b-O zoQ-yz96IRYK8KBg_~tck>=C}prD;MGxo6cNNMkzc(~b8dq7ei+!@17~4CUJGDG0w3rt45kL@ z3?~8<04)3Wz(X`b<+P$8u=P~BI?WgIL8cA;ifSZ=M>zeEQ4CDfVbg&I7za{Ua(2GG;SOw|EZ;B?Q$e@fxo;&Wa-)6URHO>t-;Q3|pbY5N?dE;iSe)=i>^4_Y$%+2Vzklq z0ISTR2hQoq^`-!JTk3cjEiN|JK=A5)A6%ZkG;t$Ee5!#gVx%AntmYK13kqIf8?WMi z`eqpk!v-{2&j&XDQ6zBKUswts0)N6FtEVh}5zkxY?m3wb#O!xD;qg07B`& zv_u7dy3(u`C~N!NFE?7GuX>Q_q=Bwl{w54SX#^4n&A>U$iWy7PWHJ8+G>eb{j*kJ+ zn><8<2m(~%xS znZ_iDi3T1}VTzgplbSHXfMm(I@5RW(?t?tr6CVbM%hQ@EAkGC)DwzR(m#&+Jn)~RG z-y%PqokNEu5~Vdo-apW(vuT@zk|X65jMbq-7nyiX@02J-|H^3hvfJt|^Wg9r_(sNDNhbMNzKb!^4Ug{r+FQr zokpCEc7UEyI(0Q}{&jx9GsL)=$IRB!&SnOXKI31Peu$k0SPOuTVFFNTdRT)3@JSqaZWVcSu7CpOOW`Nh#cC zm*`%I*4KXk4NL4KepgQne%VjNLYPTmtg^*;#p@zsfvzFPRbqh}y9H718Hs1pIv8Uj z0@_fZgZ9MpyN8DwP2ewFl6!YREOT0-`NOf~owBY@cBu!zLAN$i6f+ZM@$G5?;T9@a&p<$zY{IN3c?V?)~en;Ov5XZvcROmcG-C z)Ve<%4LY zQM;>K2N60NGh(>${e0fc{r-O5jQu>+_x(KiUUdyt zzMuv{dSkoUx+a(iN>kEuS(zSMJpDUq1cOJl-_*6LAR=;lVwJ=4i@wCwloOhbdV#IEoAW0MrQ z6%dE~_spHy#YYLqGGTxuC`bYymTj@B9Jalu5d08|kS4gfga=$M#Js8LKsO^xXCB4L zGZVUxGa)1cBj z7RIR9?}6<2r1TjwOhUHgheBG_0;uJfSwp130f7Nw-p@d~By7YJ1gwZogDx>vzeHhJRgf)9lk@LTNx+G)f2 zwckoroJF7c1sc)?PB2a|6MPBuomT*H#$z)vUUa4;N}(2qsa9gyUf9C{+dzo5>#^HD zVGfZLLkPc4N>OJXibEpVEsuT5dCWU0#z#X4aq=PpW+5TUH7-i7qkuh-&|pwn4B3+v z?$ql5VH}>lAIrY)8Dy796F9F{^aE7cNN{zWsdwS$CJ~kY8w*SeEI!#|)yfQqUf1muM5X z*-&5XJ*;(_zp&Q^;Q?K%4>O?lbd^~_+-p?nY5j2F@3m$>?ZC*KI{t9Phl@)e^abdh z%Lxk|yk%=(J(!EVQei>-P~2qK55G%?yb@1KRLt@Cdk|w~40d`FdlF=OI6O-=(0<1ysOfrDO2l0rO$& zsx=^*j;hnzJD#(r-d4cp=tU`?5SC1G>cVND0vY2}y6XUF<+&r<`kp{+)pY&hAcx1& z%z2z(`n--UW)t%gi;69uLODL0m!II(A@X-$!~;?iq}U-DFg*U}ag>S0*8q$umGKPC zWtk%goqyaEcrgnea3COELNjWp#2^}J%Kk9hV(-&SrF}y3{Ychz0SS{x8mso!qOU-A z&)%^vO!&Ol{)J`kUox{s@xa5SY90ejntvRVXg}jgjEGalBo#+54F5+>)kLI9mjhq0 zqwn=|)7wYo5#MjV?9NH#SEd@2`y-OlSwdEf*~!{EmA{$%Kkb0DWyXH|#>jIlJjh&r z`4$A9G(Z{v#E2Lz>@f<^k0qay`XGtfOJ}=+mx^f|X^9B~s6SB;5!~!;w3E(o7 zp{9nBu8-khL_uy(Llhhz0)ddPvZ|x$CWITBN{b-;@TvxZ;joWyIg1egQ=2P`w9 zHLh|>aD;CQ&qdRtU$jmWEsDwi*JO*+l)H7%NotUVG0GXfzhh8#wnhy_2Q>~9MJL}M zN)O@Lu@}uCmWSs~jC6?6zm-^}4K*l!&qm@DgD8qQyB;8QR+BEKG_i#w4sz{PN1{>G zKQV*}hGf`&+b`}rntuFm#iv<{Ay#P!Vhl&QO%;=kTNk4~MA)Bzm_~4WX{Drcmy=*q zp|g7#i_00TW1#PCPSf+Z{hqVf*;+ByI6J-~mp$^QHi*xl0B)u7Nsf?S^Db1}#?$(_ zGTl=c=msDbR>E5iW}3y=X=NpSYYRf@v7d>w^yt?Z+SDa&OW5w78EkDEaF^ylqYf!i zwqiu>*3cUer`X=Bvkg1?xM1>+IBSMH)^kbY&#sw`hd6xh?YMICH!Th8=i0}em24g_ zNy%}Jit_4DSMVyizVla{IT`qNQ8)M|o}1S#2=)x( zPe50`nJd&)k^%CcF`xol6U5xs8=otmzd8)*G3r5yuE%`Ystu&nozkZ5rblc$v_Ce_ zsR5&Mwxu>+;3p@?=!|c76407gmQq9srK(A;D`sKU+Mfhb;*7DQvP2D7>{(M^A>jutvP_tpf z_F;3D2MH-iq=z@?6gr@*&csU=aPA;9qRQSc@H?g*b+sxXVbGY1dz0)PwkOd((Eup0 zLBTeuJVFNjR`?eZ5FsEYqZqb%C(g=Z9mpp;aie%NZOvKFEi|u5hSis2&-ATtUp`=_ z1A<3XPs83|HEey^IPul1eVxuaS4Z238n322yKv-?wyeDFwACyC!EFzb~q$ zt0(@HuimWk$@-RUJ!;o%-?{p8^jMkLv-o9WWz}DwCy%+dYo%iHtQNJUS7xw` z9#uE0Zm3OA))pNKP5mY(RD*W|)Lpj;u=M*<34%Js87{e!<5!%e$*CF^uI^W5zq+aq8Y#n zsdF%VhPh}NwTv>6*1h;i6<22by>Y&;i=TkND4%A|uSm@l*Lv_sK{F4<_ug7txT4g0 zlzSs+bO_tFB$_26DQ-Nx4Oe6og;lYMUrJ3QO0=zhz@%HN*vYuS{s26UlYEH*c5rl9 zH=!F3`FC(wc_t+?r(3v}P-30E{@veMIpDk7#v%W%#2(S`p++ks>)9b_%axU2A!-^?***;ZThmqvVK0j7*M%Fu(Ycw6G>Lr7(tM2N|naMhUItjkA`b z?lOt1KlL5dwV3GFr?~n_;m+F-{1I=3Y5!oiHEq}0oX=RDi{<`3z2I8`$748N{U6+< zs=s39S?4)};aq!ju3K%!Ao!2R&n$P{LTW3VNA=Ezqx}LPe|D8E;3t*!MKG}G8?eRo zo@amED(78CY#zO?{7j=PewO4sYuMHUU%9{Mj)H60T^_niAg!Z()0Cri;+LWFp1*~< ztHdg<)bUk+g$L4Aci`GF#KLJh<;1pq;AvkOUI=M?U4=#@VN07HIanFhz)syMtYYNO zP#Xe^a4iB>iDMKAy{P~X=Hj~1?#@Z4Cuc13qr%V%r+9~poM{TdNxFJ5Ty{9XNHx)Z zm?DKSevlVPDG!YDvHE>mxjD!`qs+Ov6G#lI=E3Mq;@AZv8WsHN=ehVxkMGraB4?Pa z5*;+U(Nm?^MbRC}F&$yxry)Y@&xmfOn=|!3rlf!#RxNBV*rL#B2eB6|Rme@?QbarB z&I75pmQD0i*(Qfsd&aR|V1#j}iag_IUdy<=HTsvAYuB25_ z3on)-`QJjJE^etiLym2~K*-R=97nZamw z&@%(nH7N?Cn?s(?B1PF2pH*Z_OYJjK(`OhsU0qejMwVjQ;=#By@;aJ>{kfX0WcKw4 zy}9V8)1n3BozWJUvA$)-!N}3`y~*ZKav0sox(FTvMgdQ!mJZz)NUr^fkzqg0CO^YW zd#&GObaWVC`v)EU_Z|HX^B|!)1B_f#do=(vr(VkF5A5sJx9V={gl743ZTWy2OEE5e zLsuHTzutYYAq10G>3FW;F~V8dI{E>^6%dT1#VP%BQf5ePVJUY z&-t%{lx7Ti^8ZmhL-KFWGGi9GcW$Ut4*k(Xum7i`t+dwy1z!VFaofCl#?Qg;KN7rR zxi;S;Za&1i-zY#>)C+g)bm=}YK6=f&Vi~qt^qzWDB`%1o>8R1v(^U&&7JnMnKVFtu z$1(=Ka;b(t_P|ie_}i-J;;rMb8WZ1vP2bmg_BFDi?MFQbtGAEyvM>{W!yiX-3@l_2 zL{0P#V)U%8*69EQZtTxqHE)P8y%lu3)!M6DbKqT9Sz-+N57lyeep(u{U~Td>KoF=c zbf!!@M?in`oMf@U>8ZF@`c5vrWfmBX|MbDUll9(cX&iXmBB&;odF&Aus{Ele=~k(R ze{Er@<`XkU|LkZt+Sq1a2XnZ}*urQ1+fr)UJ{T6dhTR0H!1LeMssKg;P}y1W`*9## zpWOPME{+o9f{8j^?}e_b3*@lx8V;UoI#jVWV4f%Th>1Ype(d{?doq2v)%sScV>I|- z;rZwU@{|14AmXA}(Kx`MAJQP?VQJ7fIKx(;>3<(ZlJ0XD2z{b?7l(t1adEtB)#q2) zDpuAF5EB9bs;pp4ufpvb7Wb~qHBR>HFYh2c1@eHGeg{~aNQwgHTm8l40=h(1w@5P$ z$Di)^7Y*X=cq2$@CqnO5(H1I3f;u`It$ZWotA%R?*wb2(Ou#a=WdiIjw|rY5ZSj5D zqarJ~M!L-llz(L=X}>7a^%o%iK z@1b1lLzt6&HZsWRe+O1(14B9-VcV8dszNr`}^?>al`*({Nrd5 z$kGi+`(xp$e^p+I6%s(2#P+_EnXWh8m=CQF!b?M5nl|kUgEL`CQGaH*tv(#<ft zU6Yy63Vs)*r`{8t6QPUkRP5$%)7eSaiajiNzR64hVfklv@4T#tx0AnTiwcPdIf%s* zGZwr1^5f@ymmi|%GjH~qqPYwAeAOoB;u8pqCi-xT!oHi0?*3fzBfTv-=X2_1u+R( zgvC7ZO2f*x03T-fAMayyjQBhY7;Jd zY1j&9kvBGtKKUD-HTa7?7Oy!sdR!4?WLYL4h5YrGp$@kQm}!sbok08(#E*9PRaJ16 zQ}JYXL7h+=?KW1mm8>bZ#4r}!M#j4H#!!z?^+sVovJGxO;X&1aa~w@&TogHA%ql8W zsvcTVA4#qMDOLaX5ywDfx;S8*)iL^Z)yb5J6^z*%NM{)xH1`;2tsVNcNGl-yO4IF@ zVKyFDP0amv(+X&d0b~b)2Z!eFW|@UM5;7gM*TD3y_UAKo4+xVRBddp2WjqRf$KSrV zVI93clk!&j(>I?<8I?r6A2$R*dB04g;KuGtVI9ED?WRUw@68GO*P~(FyLck~2_E{< zm&hF4L|)HG`N!9+sD)8xlV|2%enA-x0Uwla5IT!C zxT+4Gzg8?3j3Z9<_nuWov8aol1|e*GpyDq-tMf^hf~>m{znAz|+Mx|Vwr}Gdb=51T zJ2S5Ry3j;^g`eoT;XH;}u&zY4n!R@bAdT`I;h!|Sm=58_Aqln^43&%~erv@V2u7>U z%z>~tI!oSY->d3_n+a+aWX@<#HC9Q07Ax>{=Ax3s*qaiBZMM^obf+y)+LctKE` zCE$7@yzCl751~)k#aD!+LjMbXh_=tbLlo!7^}ZYjRX4*rL=rW;P4$&mxzX8b|2I6U zw#}oRdRNQ$_(GA?)LCGe-f2bmtJGUO%TsA;}$9Pr$aCMQ&` zF{D)*gNRp3wUR=XMY4cgH)E4DpzT%4@99>EAkPMMPRi(8s#m`o2$FXc~%Td?E~t~DhG z8JO4#ga)rms6g?|JPP2B^Aj8fw*G75IaEtdQ>rSW0M*Ir-?4vlxTJfP1gbW8rgRz_ zy%B(mzGWRhEZyx-4*tA z*0p8^z>xQQ4vNq~w!Ee>HvGf#JD!=&Fh2q_UJ|)C>OTfc+20^XSraPLv<3>GCtM1o z&B8#M;Z>dmGwo`ACN=UxXE)2VRUP5Y+p0s9 z``PGKV2!wD0B>Jyj`$Ms{uh@L7GTM&7#YqyPzhTKFPuSAs4qvuiUhK8s6`mBFS`KJ z*J4~57zalN5t}=Zm983YzYj?mlf6?Ij~9H4Op?Jah>!%jU#KX&VZu=bD zMns4Ny4+dj+1|(`?XBUE=9GCP8(8#)3&v^X@I5TObnc^<;A4u*K8N0mO)|yJ(%9=*eED_m_rk|Gatnc{)HBi8-c09KI znP2i}#5(vBxtmqWnZA5WBVbx<)Hxa&o~v}XRQ&6r41oJoD_;(XR)x+x z(jUHZlBxUVMmvY;0_cJPzQ2mbOh%_{r?wF0KnF(|X1x`4K{d8CuPSd1L%2%XjBR_> z^1gmQP3=QxzQAS%R%j_#9JRx%9Bx4-RG^}z8MiJRTLz^pk9Je0}rELh&ggM2WQ2kCyMX0J6gZu38`nT273~v*Sd)7mSvw7S09DPJ>(cVuNeVF>7#KhY1DinH zSsFK;uM-xpP8fkm4*i4uU@YLT6Ku`pTKhO9L02#FS3&20@i*Z*&#*r*r}?^|d0{4S zVI!ayEU36#az8#+v*$*WUS686=4{xv|KPg{UF)37p=&RV4nKD}%sbYKZbCcQ_%+LR z3P;uN0}d*3CcfdS@GnSfpv(3Ttl+*89aM?9By^HFMo`p$4+syd{(U!U*Lt6AsUl2v*c2L?q&)E|1qd>3fmkd>lpY!B+Y>W?f7)DB%BL^EX6a(cb# z6U-yPqw%$>iY*9^Hcs)~4*QSjTi6nK9kwki;Hifh&Y)CvI%JP8)0nVm5BdYvij{pf zKOnpW?aN;IA_7?|>`w|mXeFivXc%nOV}ZnJ0gakNWC(ZJ>iSm9eoiEqsf_h;43^!Wi7D(@P-=@U|96&EcTgSD4gSL0;T&CXg%_p<=gqZ+L ze+|^~kiPLUn>?wu(uj@0TOnarUBly&pEd&ih`h&Pq6=$7`l#oW7cc3WT)uX7u(x2lM308bN{XEnmN0&Xw7T@coU!; zW|0RcU1h7SXfNAwiT+ZygqTmUP`!HtyGqF(A{9!Rt0PVGWv0lmgIG`l;SFn!OBp(7MG;Y4+E;+z+Dr zch;EexH%yTT>)8DvX%>Lvn9M1V9n#=g?Y)JtGmz#^7%nn>^wzP`!mjUFI4G5Aw)_G zSG-)}H)~K4v_$G=+SwXZ#>9~1&PiwrmsoN$)F%uof&wK)J5hZGNqFM`^o+(Oy}!-F zpayg$@<|&!CIYUD2q`K{B8x1X#z_9W+^EiY_GcW0WVRHhs6UEjwUI`!S$wz*#&Nlf zp;~<$lgIYPkmMcO`T5q{kgi@VX~`qOAK)k{0guKnoq=l{4HWe%

+Xk_VWrZYFib{h$r5Ur682^SEIa`WOKOpo0XpXfA96u z_$AysS?X(D>E&gO8!oJtIeeH_)uOx@kM$xupJdf^WzKCe8?Q>gEXVs8*6;^#^~G`x z8^LTbuYkM;&3H}K*Q3v`MoYgaHq=sti4sJ;8qM|0?^8RT74u0k)APruj0pFr+(E)} z3w@gx+Qk)@J^&WL^>>|TPFGnGe{sXr%QaukQqS!auFZ5=`Q_#M`^EY@{NQ|Pm)W8Q zy%@bo=7sags><@Lx=pGgDYMxk&%$jogCFul&t3AdSS|lM0c!Nb&l-s>R^=4XuUC!i zswrKWJhP*VYPs^SMz)r*yVO-w0)+V)B4-GHKw?&>}yyQ>}A75EIFbndo?PvPe2g{-O0)-o7?Jgd!{hzt7q{m{A9ff zo^XS)63nF}@Ss!4Q)?*ke?~e-kz@!dfG|E3%lPO@JB6KNB!oar3fEDpFvcW!O@gs9 zWf=QZ0t`IpEUU`V#sA%n%BenWLA|P`jn?>?&LS%Xvx;enu||dlk0jC5VdjWrP8h>d zrH#>Czn-n~Ducz(Yfb1BS!C?Ymn9g@*5SG2o2*>q`t}>plJ}q|e}0*S6>t=+QP&n& zt`>`8StY=V#A`cE{;Es=0Tli`d6Sn#vNo#Z7hi!WCS`qNyr(?>ZT{Q*ZytUFD|SV4 z?U%RFnzLLllWGb#CpVxS$pV&*U%-9n+PuqYaRV0sxTvy_+1jm?=cApxHx~UI`ZivR zY}{K=pa1nTu7&n+e=RCyic-0AEz8VK7kZW7Vmi>T>|+?@RMu1-I~5^GN?J~(wbnSB zgPhK);!JcSAtvTsjx_EL`2FAL4gFp9pRzVjhwLHf3X{p+@atM2T*~$1w9wW zQ=)Ln;6FpW)d(YDUzf&C^1|r+eD15ABY6)zPTo%aRA23T5#Uj+j>)u|<e+z_yNpzYmRfGvFc4qsrD9XzD>t$AJ z*$}%uKrzU0f7f&!8Lm{2ZX&jeEHg+7KZ19Sf6uacx$yPl^>fMh0PowZw(FE|EiAyD zMYhJ@hA%R%ag<|{MselZ_`E#i(J*x52HD*>vgq^LhJmxsg?@t$F#GFg#M>Z?o()X+ z0A~x|eV7)jJYREj?O+)LWdTiD#XDmniNG$hDvx^ee_`sAEi9z^H_1)0yegxoh`~6L zQRnU;W}1LQtM53_Cm-caOuaRWFWmfg`Z z<1OG82_#6vkO1J_xTs>a6mN-|IsyNDGD$ViUHu1`7txf!$0c~x6A`V!fT)`}&~sO- zZjEncf3~IJ$>i+;1F~W{(evz+9)t{l%TePIPlm05dK&F9bOUuaUM#a~ZEru2d^9UZB9lEm9C!nd}W4JYiyNzz|^Hwg;6J0k# z9!o+Q6-YdpcYmodukoI>(AL+VCls#3%Le&zC_2cn_7b@#rCY;e(E&*hGx^}=lcG@@ ze|C5{>cH}pc(jM^NdajO)AL27eY8&vNqg8%jY)gJrw64y@aKt22NazimL4zT`h}0R z8@zzpepy}mTVGxzP~n|kHP$lS?i&zJsi3W{Z{vYZw)80J@eX)c*4q&C^}299t+Hi3 z8c~>HE@--Y+C?#glz*64+rSQc)P|Ddf98{dx&$$~En@3c#xp{iYl&xmcn}R!fXUzn!vYde^OeV$DVuKU-MPsGH}0aa}y^Zn%4#rXZ?8%VkSf9si9*_$sf z)hXuv<_?flDQYJWB~aba9eB(__-1>#qBdCs9 zgQmQ14H(p*8L;iP|B%6-I4?j&k6elgrg&>l@1qZ)m_e|!L#xC#fMss3FM@4#)zN|6BIcUj1-)vzMXS1 z#4K@&5GM@5K^eHfcu2VpkZNf|1jVF*InKFGSn$|yQ_sQnJP8~tYn+orYRtX0TE|Vs zz&$d`O8$Bh0rtK255U~yNAC|N0~!|%sX-3ck^r`Jn}niP(*a)je-UABy>xFbyp)%g zeI|QS#ARzV>{^#uuCKB(fN+M{>H&&jLp?QF9Wy)E@@snaX*WLK9AkUV0o+j$-`%vvc`L_3lSl7cIOl*utH2(pqv?xJZ5M&RL3+gc&R z2?`r#FhqTAQ|q~>e^N3?29VY!wFm`bYM5hd<&rQFuvAJ3<~;^GL)Ksy!2G$?Omir; z3Gd#B2W@s3ER+$65HdROIK=0$r;u<3O|T+DLQ33at>0yNwg84Ff8S`xag79O5VFat zDU4nv8ZH%e)T_sGK%;41UCAV0V>$pX|{e<7QD7lD4QTPks$r>r5G z7E1y)=w^P;Nvt;oo~}e&FVUO(yR`TO4S89C{p6S6XJdyuS#MP&f8j}37PF+qjsUK0 z41&DelhZxIdr;uR{C8i;^k(xQ|Gn2)BZ6bnYBlXh9M)UggCFE@8;XW?*U~=SwOY?e zYs^beSNVLhe_B*S$4B=u3@Zp4s=Dg|CU%{+)>`A~P0vZuL!d|Eh&(OpGr6AbqKFNa zdIIWdVwZsLGP>_1P$gx;SWN;y|bgmNOF^kpWSt(ov8?31PAB zj3Os2cY3^*$&f8V>DQd{i%y`jd7*D8Cym@bBS zivywUhAaC%?+ayj4EL(RxlpaK?d7eWmxU>Yrgm-kfy3JBcv8EzHqLhw|J!B8tBJp? z2W+OrqNmYy!*)&mZ!@tcjQjsktibE$P0*{RKKXT?{az0Nf1g#!X8A-{$=i4=;5#3C zN#*m@fA4sZ-n8cZ{}H#*yWO>-3Zb3ixWe85_OcjyU}O)$FkfAh)KL2c~(N_jAvNT)ntGFBjfcQUozCov>b zY23=;0n%hJ2B9=iB9z2d2<{bj7)YVEGv+Gcl}@SjI#9<1hD_I|R8Zl)wa989glYKj zf4UC2#$26id%n8b1>T=lpz4dMPW~EM{VDsjAAa92i;vz`aXV(Us6F7@n6)SW3rX|5 zIc3pQQQD+IB4e=>kRk{z1(N`+|3tCOJgn<#k(i>I)*P(0z0vcx zPpMQN;b&jw?D&}y3+~2)Z?Zx%!<|)Db1)C7Emzb2eoh_={U%YMC(xpSYt$RLa5o7%2j$Y3evd$ax=9i6|9@ z@EB^j+?(RzIJJnT))0iK@?dFce>`K9LO5pFXrq|X8l|o?(>Y_%u5cbS9`UZ=`-?kH`*o4kWBDy`#fR6j!7LIm=)MLYau(=#A)N5M}Zoqh>e(Z$|!TM zYaX7*zWHuWCtgb4fHKUsf17hVL;V2im6aFCJIH}E9glM-W|ifx-b-knBKazIe60yS zo;+7nJ1mWmP>u;jkcHd;)B|Jg!&gZ$Pn!Nu@|E+~ne}JRSM|{S8U5-a3EA(Ez+`r4 zRnw&?yP~EqhX?tAz1;|rD)7(;#ye~yVgowJHXAwy?LqK9eG!ZZf9F)T_Fdjims$CZ zkMo#A53L@c7&w>I5O&YIQ1+TzR=Z5UUFuMsHk;H5`G{9KaYy3#khzbWXv$%zTBuGr zW?Dq1278E6~}>GVq{EO_jf z2<6&Ap@2zXkggz3e~3xr6i3R_cre@gI6>&t>ta>qMRB#CIn;`f{OIR;S?zlzT^BlU z|MupmZ-+UGF4=7zMej#`x}%7hI1%l^t~O=EJjMZrf%enTHOzIWUamueZ{jT{8ISY> z<;6g^(E=Yl$`}dia2Y5SH04{p+^n=+^3-9rhY&pMKnQwTe_F$olPRVcDozV&G3G?1 z+&f{N4OSpRw1SG1?TyAb9T0O?u)q-%u@uS>>4cHQU?z-XN|I0En<4rMeQ}Y24whjsO zeD>4*2pLj%e~7)pBhP2i2Zv8MbbEkibJljBs}G+j-@H0vJ)iJyfAJ0!Wf`>fzgRs#{|KryIzFn~ z*1$a#f08;5Kd0NK=V4j5?>y;*?iO(P@!UfcPdS^r!}X+txjQt^cPh7q^QlL2cc@N1 zkGq3B{V?th_IXa?wrM&281DM`?1w2N@FB~60)OX4wLa_PHt%#>B-)p%iIraIAHF>Sb)aXrB^mbSmNtwk)gb>RzwJ$ojp{r#qT#l8{>X z*O|YmY3=)tpa*cSRbU_KV4NF_eVr_OA3favQts@DN5^p7wngVf1o#|$4`>z}{y9uCY)*ob7*;1?D503=i1Mp6JjIQ+k z?rk_n4$o2d;G9K09L`ZUaJx_8@SAx0=4xMa6Ej$oDa-s6}Y^-#nbX7T{<*kHUMb1G+t~|6|Wu689Ex6rfvo z$Dlr!^l=uCwPmGmQRKFulCmdP``jjv6*G@9({gCeETsM_B?WP zbMw)e4IFG~R|@2^qIG*l2?N_e%u7uK_^?A&AK*ZRAz4X4E1OormD+~qv-s-_XOoH zN*l3X?%IS>We?3umznAu5O&>l&9Iea_wH}2$M5QZ!^RHV|Ew+$k#Wk2;<+R+Lm3pD zQf7>1C?ia#sfJvkDs17{7F&%}8*wl~EOk||!fF9m)DcFUhM76ee`H_tO>Hd?w%zpT zV+=PnP?XiC#<-lx<^ULLuQ~W6JdHIqnZKo`CcKQ9rr{#e&UzfYraM1Bm6tk1&v zES%56884iE2GoIE##&DYfU`}fM;=~ZyXg^mw9)hk?@|`iBUo>9=@EUXt#sTwA7dyz zLJYH#9)U!fNRQBBf9#{j)8=U$JtB{@jE;Z>n?*;k!fc{rOa&W6r-D6#tf5C>;ik|b zSK)Thfp&>8f*vy#WdR)r3ow5U9x=-HIkLWD44)&+MO!_`b!x21b7ZB2+dGGk9c}D9 zf()^A9zk5j%y|SFX5$>4(jWupSd3k`b@TY8U&gd~g!(D$f0_ej9B$M+f;wl>EKQMs zy+-g3p*w=7Et>E(r1lGU2%NHL!aL0FC^}=&>@#`JoH_dKGq%j}5l$I02S;=-D<)>> zcWK20Gp(4Yv0{QFD<<->V!mB23J3D>YO9yV{r{n|S$b=MTjY{j+%;_kWpdgMX(hjP z8m$I!(Bycue@*4Kkx*@f?{9+$=eYND0yzqJxJCcgNtlb~cu_HYIK>7c`bRwf_S+#k zzLA*nSCP9Eol44@58fW~dZbmO#Z?nmpMMzkf&?S;3OaAhMMliRcszgJ}% zymi?J?%UqIyI4qy9q;tG$1g`99Eye^`@-1&LOC3{vU{K#0-S+IlZG zAM*W1*)`+5Q|B0~R@O1mg^N7~O;6e>dvv12D<^cZj--{s&^(kXz;YlB{01)f2GI@OaJR=nSOSyG`( z9wxgbKS9+L(s}KH5K1vmkm(otW zF!uDMIDsrxX_k>(O9&Lqa$v5;swQ|0IINnGBcyYU6h=9fNKuY6!j0V_NTxxq1xbzK ze+-kcVr})z_lhJK`z`JCZla6i(eCD2E=nElUo~Mkb4{^?*^lgBEFnn0XC3|iM|+hU zuaKBg!n@trb{id93I0GERvfWpWTZ9%VK__MUR-6Od?A+KyE`leE_x!0=?*Jk~G zp8TBy=?kN~2Wx$FSZV7o4H)P;^vicLf8=rC)<3+v?wrKD$8ewifs@k?(ZjdAJo?qk z2zKaq$z%Na6;6~pzk;}&H^0V>$vzP|j!8ufMMS9F*0QU5nl4@x^AdDwt!)dDijJyf2h2e*%p^2ih=a zY{kj2j{>Ex5_4-dFGsmm*Y05HOt5lhjnfgjGiGxfXEui=4i=IU{V54NEct zl|q?DIYy8XlLMLaoFK+>lOhgPDhxxWSJ^uS5L;Q(1ZWKGif0hwe-z}ICBh6CrAUrb zTK2^q3Y*7(BMpTJ1rT5fn93DRg;JRUge%S#HTM5*&!P7s)GypphdG8T zZpI4i158jGc~FmYfBRSaEU*5)7rWiu*p*SrGlJK2@#9iR_=TkjSVEDGr4D)OGF|-{nA>nv7-!DxI?& zNhVCHxKWHjyE3KJP)MU88WlWFjS&K=9KlSbFrzv(I#-Nye~FO9jHN(RNazc**xzWi zZ)NV^T`11Ki6iH5)rrF~=}P*R)p!@h<9_$06+tcEB@2_d187oRxm+1wQP^+n^t}%B zA&*UQ>qVaXZraX-9`pCLAYQ^G~@e^5(gP*=j1JnYzB@iRQOrx~YqDYkv^>VcXO54_Cn8gdHYjEZBCT|->> zh*eP<5a_yz32x8HnaQz+R%sdG_CIE~@1BZ-QX=~ju)V)9XoP_nVF{hG9e=ZOjZB}?#4Lzx>`V6Qy#har3 zF-#gy%wXFzWbUJXE}_h32qmI8L)}~S)5=%{zgdKKDD2V)iDk{vKr9)_DCoq>I&;ZN ztA)5N|LcCe*6%IQwvHm06X$pu5b8m?|Tgy+I<51UJSKBcdd?#ljs}~ks(V)6~)}% zy)e%x#yL!hTjfxsL~HWGS(zM7zJA|tSNL)v*Bkqu>?vK0N@>ze!fiVVZ;HB^`rYWC zZ_8Ev=k@))+m#bTxt4_3Ymwjp8zPAkno^}1e{y|RU3Gsz4Jj4E*GeT{SQ)fP`r7gM zMY3`hF$eVO=kNYz&qjp2$T{a5Z1Zws$P3hkT|2cy(n1qXTKw@ z*fA@AfHGr6QUPbsOUI8z9s`aXi-5^U28e`2NNSiJOtakCRE?sB6Dm?5tsZ{SeVI4K z`lhsY)wA@Aq^N?zG@F^b-&P|^4U^{>7$C9F^dev_KeVm>aSOigfehVUS8J`P^)5hZE>%k zHrh4ky6yZtPiQ?cCo)E=Ry|j3_b?jNlg*ErI`h)mXF5IGz`5|rfI-SJPWN;RLY>+3 zEezYcC!FUnb(jh5&w86%TCa$Sf39(j8b7YwC?CXryRh2op`KR;>}4zEq0N^o>WEVN%^DoB{ zyfZK-V4`y{CvY#7g*gH5orgJr6Pk%R0e691%n9VMY|II;$b8HR)R>ISscH7i$((?> zkIKqC0UDf_c>*&mGxHRa!MT|epmEum6QJSwnbTZ`XJ}4pwU`{ue^A)MfHEC&0rp zH%~5spxn&~#K`Q;smZxa{^kVwr^w)(+#JGlI42-yRi(z@HK2Dey~8K^>{X`r^bVct zvzOL|tM8|?ILDbif0xI3lIlz*=jjno<#JA5ZN{@XK~Qb#pMUuIDkPb+{rz{5%Q@Cw z@4Qj_jO95?jY3eP;J0!wQU~{pfOk;CADlsOr^Y|AurOg;Df7--$ehhEZ+j+w`-&K4Z0fI?w?iTvub=+jnAs_cdHsby&S$8@x1q= z9{2L`sy=QJzmO1bcZnZ-yc@-Dyo+yZGS;yJLdFk)qr*ej^6DZ~O*{b?f+z z7Uk~oTz0;0e;_~j`#ATI|6t*6Bmc1i-AMrt74BvVjP46_Klu$7;g<4#st9+L@9q)h z#`1iyIQLf2U?FZV-&Yvt4)d3ohwcwf9LtRlMV^%ed&Qfi@Bi6N?TLb$`L<>s`_yZxS*>#8OT_fPc! zVw_M1q$p3*0pxP-qy{n8XLJDabqpQo{JlR1%FDQTT0Z=y!v&+UcYXFfvZt? z)Uz9-pswA$0#L1E+sh{&P#kI&Zk%^LsBl!;f6n8NZkPSGnw`<=G>kLKX}3Df)4I5q zWrf5b9oLeFB@0&k46FD7hByaJ428X)n6x9}2gK;zP0(tH-5I zf8=VL*o#0?0)3T?L4ou7KEL zqm%M9QfZsLNa|Hn+}|V}eYAb&4m0Z_F&nF%pVpgqC4oB=*w1zHq;vNYze@hJw!hYX06y#OXTANbf44uU z-kx)RnxkK9b-I3+EE3n{D6uOpxz{_V8k4dmnpVmEexdZu&CQ$5o1$2^D$gC?ta@~M zaOWx|+pi=_SY2DJcgb$^!L>dpiQG5rPLbR?W$S);5Wng~>D9W#*sYU;<@u}Re-Em) zgmixsyTo}eb`Cz*C5Ki2>9t8jfAYrNq%4gc&VE|h@ztBF?v}9C+ncM;(yPq=b9KJF zFQtTLDd?2<)uOoG+Uxa0pmys4gxSF=o`&s{cuG^w+x7?TP`CAF__neN&bzQplb^2f zizCac%S>ki388Vy83T$SB7vlYP)xZrgexd{%3UeVrp7PgAV+{OEtSauf6OFi8A&Nb zS!R^VrOG%&0*I?GM{4|f3^=Uun;WSq%2mphra(}IARttwCbi?)pD~t7G9_N*N8LzX z+1H!ADi^{?dzYnM?^Y@*W)myFul7aqN1Z1LOzKq+lmB@AZQ`WiDyZh~_4yR0Kkaa92!IOpLS(DHj=* z8oAQZT!@rI!!T5sVr3AUgHcIw2%4zyJEXpT56Y1$HYLpU6fYXTO2ZC69Bu4>AG?$w(`Y zKYYkH>-#`)Y>308vEpS+8Ucs6mBw#g|MJJhX1ja2&evrF$nu9;*Up&86ZB`KGO{P3 z0VWmN5RFA{h(tixex^&I0RiQlq0Zse6Y-B)4X zt0${qetrDUPJ}L=z&hBKhvssLLHrXIR4<%Bz+Oky9;ck!7|2=%9&eBcFo<&lCXWFhReq>SP zJHGe$s1sMSUE8KYQ->>-G@tq?NX14-&C0Rc=juJ%zt9#^Sro|GQK~B~l)trO~k@SDF4Hy+z5tQA0!cp3FXin8C|O zHDd76Wn#H)+2T%L*T;+$#nT6V7wS6oG2MJYQ#iGza0i;gU2{z)Tmo(S2rNh{?j7uD zff@MVXIez3f*5OR;A=gu2zxGpyNl>@@O7Pjf0Fv2)YPTUsOEbr@$|sAsvCxennDe+ z?GQhJ+n@t#9f!JWt0r8griZHI`XP1rU1E%D3X^tdTZX4u)Z26nWFptZma9A1B4&V8 z%~ow>`QcY=I`hjPQ1DHf-e3MHORwTIK=DQYc`4Q6=n1bP&Y6CAC=2pg`9zBNc0H502BJFDM4F|$e-oz8tIY4)=rh0H`WVG*c`5|b)L&{2S5HVg zheX9Bw2-H{rfUM}f_z`Kh^l(n4Tx?!cA!y|@w;+gXN_iQo*(J-!Ko^Rjy6hI8Xyai z*RHSv8Wt5L^_3q`kfkp*!>gyOong}t1088$s3K}NnwCyYJ46CRpOA0*)YPYzeq+O0LiufR2eD7Vdk$BKgS9i( zPO^mq-*ZBQkY+m`_FWX3z-Z|fe+sdssTv8X`)-)OqBuzN2U2k{FEVSZr=YXN*PCv? zsv`OnVXD|cZt|65>p+nPDsZ@oQzw%>lA9NN<{NgRL>6STbfwO`)0b^`w+NMq1< z7u3J@bm~8M4p>ZGe{J0!v3MR;j6KaMrR5~#MrS-Lh3sZ|(M+14?1JnE(>OkG+7fznluCI<>>0c5;@Yekd+@jC}qx) z;O|JCy$e17@T_UbUOmR;Em$52RsxhLe2m->$rZmWluvgge`%}~NRp?n3Y{w@D&Qwa zPr35n7XX#9W=fJOVMc0*E6Do4fE51i$C{>$tl;EqlPh7ajh=6EFE*TSS~LY~&=Irl zESk@v`7D}WMKtex-s$~alq+{2A_*0bAtPn0tjtQza>7KZil)J7iToJdrUetVetCZ% z6|Z%wNI$Lpe~_O!l}(&VZEl|_l{2Msrc}<9%9&C*Qz~b>{cN|N?e;UJ@~cuRui`!! zYceQ2)lnVDoUVoByEH>b5EOz!t=nEmu2v%&Ni+96OLg0Gl}%6AZ)0>kK3)1zhA&Zk zyhH2!h&h1y+zHdzmOJrvOg^*Pc2({KQUj!#uA#b4e_*(dha7?&+rzpRnyQL@OS4te z!{Gllxf7P**|u*Ou4lQ#vQ@)Is*O!ML^_4gpilu;bzQp{%bnQJ;V^eX(`?Z&9Y`baw_M zhZ8YWf20&pxCy2~D6L}VIX77qt0(A{vMP*DMUlm<3e$6R(`)x|bc?8dxWKl(i8I&+EyW{j z>`I7R*x1KPzF`e2tFO*ypBLWoT-P$S4g+D^e=P;^BeeyxC@A0rH-b6|mml*qxs0Qv z01aPT3IC+1SL&5H!bxCi9+DVmBw|{RbxD<&u9~8!cX?|G*WzSI^HX~A;Y|)kx2?F3 zF@hrFX7u2sJdGaGe$GZ8R@UF>p>1_JdX2Yx9lhrJI36b>+NZYVGEX30-BNw8>Icg_ zf9Q_5-@;%t>p*{Wm*1`u#~-Q_!#k=bet7_wjk^k39EIUNU{?SIGXVF2>;%cMz!0e1 z!A9nT<2NGrZ^;A5e!7m*Ph9S^f^?5*gcStY`e6*MeNudaKoakcxsJshV$J_9Z$1WY zCniTi&AudvsEJFp2~uvTd7Ttda5#Kdf4}#n1TkHZtaKjy_aF4zrX>tA$`xK+{Y}-^ zr5~fko9Cp3rOp&2DESpkz_LtY}a_KK@7LC5}vy1G$trvxRHcS#XHHCWZr#9?n& z3Ja|rGrmuiEI*P!0XIA6huyO>NoY@Rm?xbk6f+8^X1`Xs6cFa#Y9{18{X8=nf8Z^g z1|@7wf}v92$O=g!pPI72P$@F(qp>_xHhtp&4+xe?ipk^kVDj2HNA=a|4^p6XEXvXn zYAVo&8?$fth_WaxbEP~|G`O}F3N0$E%wewNDJ!)I6%)6oT5K?}PO~q(T?N2sK8Cj? zA7is!-O17LhBjH_#dQ568IMNXe->+oGC$qT$2Qqtyp_1+)oU+fb5%f6)>ehFpPf|$ z%34?z&Q{y1YPQ{~sv7QNPc@_X)w3@!dPuI6uet`Fl2Z)_rLh#y99gC7*!@4h+s>0Z zC*?+TC{5@YEi*DIWj60?R$ycbN=V350~IPMVyvlTscDWyec;YaknHIpe|lhd(~3y7 zZhNsZDttsEJg?(9mZ`Q}QQgzEpQiUWL5AqWXDjKU@wN->u4LUj$JRC9(}@vaYIR`S zb^;tYng(ZA9o4pc({weA)GoWLx|R_Jwq_vR43LR64;i*)*qZ10*wb~*qiT-06V)u& zcQ|tV6{$u@u!~*WRs&=af6WP<5PLrT7l%Ped{sp*9z7Y?E`lDJ%S}~kG-6Vh#bc$O zmH#ObREG|ghcYgrBZAWV5d>MNy_Aun%#!LI`bI`_`YSbt6F3qJ<^FkOCCYiJXecKQ zjm0?O@rBWromxAi+v#9Ohz?#;Hpoy}N&b3=BR|cuGEW6Jc&E8oe|#K9Gu+*V>o}&? zGQl@d5XBMtm(ssRv7f@>E;W2+>zVk$b+h{x`hHeG>B>VdSK|-bbx8p zF|h;Pb$pdE4sEd@-|#)hBzov!f^1AI7h{AxO(Qz7EXTomfI>VG!CBwoaE}v&9AxOG z<7g%cbX_+sWCohAe|sL*0+$ZP)3pE@V|)DTglcM}d{eE4-bUxTUu*|XKDjm~v-T?s zZNi?RV4J4l*qzPOj!w)v=(QKDFUy7`WD=O2(C=I7l8%ZC*VgooLGbXD{30kSs-{9; zlZ|4SDXP2ln>bFh3LglS)l}psV8_zEn4m{$*Rn{?s?|2rf4(m$;C@=3HpE4$W~ff5 zw%JJUAux6IFzHN;jgWaVUH`P2vd(jmm_2ddv>L{|XIgz|lV8dvLf9+qw3hKn8H|)i z$`JPRM>(9VH_A}9`l8j!-0F$4X8ZV|_1@U*g|cpe#jiN~<<&a21u^B$p@=7 zq6$F*wg>av`xAO8fbA4VXU#!8Q4?JPhBa@v$n^p2akPqrtoKBH?emt%_&9L0BkGux z1Xcs_d|gXTev`TkXpQh({l=a!x|MmrhzT)2PJIa!fBLa&2Jw7NZDp<@iVt!l157~Z zL4rRZ2MGubWRXcdV>ZN$J>v?nzNcFpv$^T}xq^(TCtp%yUjnm%s8sJQ`eo6GSQul4 z!Q@y{WRYLWim?`@)$6|8Fk6rlQ$ZT@SZS8ZQIh%I44}8jr9oj`o&s;lO`0C+sl??1 zy?RGlf1)=Lvgfvk%G#D(S{(Hspp<$w8Ap_@ii%Cn1pr(>`-mDyS65fhz)Ao#BhDp8 z%iDtv;=ZLg(sPXfRYOt1tmY7gSl{?2G0Fy_UIWtVyt}Gcii9{IYu?3cQc7f=OM}rx zPSPR@qhO8aFbCd)dH!7{PZ;`j|+WJMH2e{|haFN|x0wYiK_s-T~;Se>$!K&%m- zuPeKD1@AKwvC=Me@4f0AEbE}DJpCrWJN=02yjR_Wt{)8JzbawL$4!4*;e)_e0!@(L zwavb28y)5v1u=d08-1R-zsFOBv%*=rOCQLQaeO=rs=9AObu7E5#DlbiWCiuj#V5$H zf9bzeEB+ef4&@*Cc57WZ`ck`_D_;v<{1n|s-M^2_fT){sm#XdQA%d#T{A=>Q;4b5; zv(D$ZF9pAK-tP^c{eek{%DDKz&0Jt>YWBBFGH#7dYMKc1Z+4=4-!lC1D+6@P(Ctp7 z^J7Y$?&(GnB!)W8#n?+eo{0}LW#!9}c6}m`QTTi@&|g1P zQ^*=ebmE>sGPB1z@|gQv0Sh-FC1-{1>i`v;0#bO#$}RmWf1~{Q`ycIMh`?1{+D4_=J>Cf%PN*< zLeb_nv0rb~`v35hMM187hmuJ77R3>j{Yf-dQlMVH-F+dmmc`lldL;9+Q-5j`AI}Dot1uW$xKd?k$n9E-{ z0GmnvE1g7qmr_GAilZZegc5$5A!x+U6>ROzd2NEGK<7r)$sn-iI;LxOe*)hhQJyBj zQz>dUw#~(|!uWWGc_WtYs$Odm_&zG~^bv~L!zDTE41v=|D7bGn{!v)A`^gUDM2ou(~-xkkL2|l$6qGQ1;V1Eu^gKX^^&RpQhn< z1=KX%M+bFA^3w6)ksaH2fA{{xUuOq85``pp)H@_t`ThY>ZK0bA!)yZCIx*1q!mi|i6Q<9>=PugQygWb45J5R3z3r0NE$ZJAf67@J>8lq2TTSOERuI zoc?`6x&z#oh~|z6e{AtU?f`6K40nqD(Jl?`z2+SZ{h)eNFoqJ`(-WKzD~pGf%-gWpO9g9x&Dg7V(K)0Cf4z-AeN%I~vDXZJMnRfI zK}t7FjE>;gibrFo(Zm)88|>J+&@I<)*TqD))O#%0WzZ|u$UH=Ey$C$c-f^#&dIap~ z*GoNt?F-Ucmph`V4qSe#cg})EPi0!<<-KV7NDn}a=dK1dA(A?wfVQtrxAl;sm5Xl) zYA;AK{eD{ZmW?m&gf(Sj3$TYj}FlGF`_r+f#ct z-=rnZB9vd>mhtg2g1eUeGG4bdrW!A0kJLE{P2u%M)l+Jh1$ zR1>k1QzqHVlA7}#sR+ieux5p53&R;eC8v7+xkM>((`=x4Tv3Bpi=Xm@R|H zhFzl-=<7NI z{>O%BD84`>`^P`F<&xI?YY!tkaHZ}kY9$)-QyVLID<>K&KCD!eyLov+mvLFJSI?FV z&dGZpSEt7zAS~=pWC1%5C$itG4QH)n)6>@v{DqIT%Q#Z$B4CxjszJ|4#^-FDh5^Vg zOTUQYF4=kFmM!cSDKk;_O!JDA_f6}br6MxV44pi;U{lNswqXClQ#rAm8jd@$_$634 zvy3rAxj4lFj4ZcdlXMccf$Qu$wt;Ij4z?HcgOzVh{f-j;nfeVc-W`k6 z_|N=U@WRNz>FMF}QTwq}#*O>iAzI`&_GAzk;srS?jfa$>%=A+epr@sySjf2Ikq z#uls5VVl0zQh!BUrBz`Sb$cO;!8TihQ)3f;XI)hfI=g~5JwX~=4XcfVj8w8JM`YaA zEEgmeqG4t5WVFnk){vwH(+o(Kx6-zySlnoNXSCR^A9FYfY%1is_U0zD5X{PTyZ7Mu z5muMSHbpNF=sW&MwPJ+vxo!Rkxf7IYQX%sD5-j`5eO~i>$}gE*w^W3Rz}`gRA|anS z0qBK2rE{bnH&hx}3to2ycE(+$_J2>Wxl{~lyx>)$PIK@|7$>{a02`zt4)XaC_nRz- zs1&WG!CA14d|WD2T3$e^ev3I@Pf9^gTinS8jVB>AvL*}?KAQMmw)x?SYb-3p)B5rT zk#noc8t?Y2%BO}yj~~K_hSg??y2Df#;}lH@dVFBt$i|ON9f)QXrHrZOmLQbooK3cP zwD0=mi;t7h&lw(!fUNVnLnztlbnDOVV_cOR{S6q_$074n&huAO7|=)IT&K&2CJ0tB z*~U$Z&?Q!tzdT1Jng&B?f1Ra&OFJ!g^dw}tVmo4IJgixiPRb3sr4^P63@HLr&SG$d z|9*LnS)4ym0*(3Ov%hl>xpNn21z+rQwxpkerRs%!Dc!ns3XsdQ!T(O}CkpL8b}EVW z>A>7T)#F}`yM7@f{=&12)J6At&ktqLJ1=COhW`&aM&6ysaytASaNJH^7R4s~hc5q}wBzqe-UIao&%F^zo*AAHX z7a}1P0KKfmw2*)|MeC2Ire?CZPkM42uh}9lAiHatNFF3(UeDTtgM`A>$4&VWgW>z2 zL397rwtyr<@Ya2r|G8~3{U$Q}^k1y%G9Ewo@<~tJFUURqDSTTu7 z9YKiH51Jp7>@7?n8L*{bTk-G(zm zew%r+v5kp**JP4SS?WvhF{v8E&_U*U)VhwgdHF6mOK#}7tJepclc%igt3btK2Oz92I zH(A>90&I4x%(CGd(D=(ym+pf(^DQ${FVSs!tJA!O?d8C+ojtfe*wH3ol{(%^xu=b@ zMwYmlwnp|08qC3_z@pgRh8rXju(tskn$+uJ{Wmz+&IWo=YNz9j45QN}QJThyXF$u* zUpsTBZlQ&}`=OvkZf9X~y+dX`PPhgB2|T$DuQ*kqYo&Qb$i!;<? zuX*Sy=*~pca(k4bo44BOGZ1ERLq%C?VMC=xIclXOQ!T{RIx!J?Zes`>kdota+3ylD zvk}xQ(yo~bCC|WWrQCUbqN<=7QWE2;5CttqmfrjWeAkVIfm56@=Qn)Ihi@i<(yRVfX6a2c@dMkxY_d#sE(Xlc2wYQ(IVgn z)!d%L3aYe~&_Ph$UciYcw;j`oD63u}ykWnQRD zlz=RIyXps0!*=Qx(AKq$lcm!O`Y>M)Y5C&NMjuxpey5+s-!+MYBxXwe$tu%$53j{* zXuX^rj3dA(zjxO|-oC*A5> zzPjn9k)7Y_@JEQ+J%Y$uoA!KbeYvyZck9WhC$0j%?oX!??8I&&@R@h%gc1czCJ{{F zKRQhy`7;45o~=5y*S%Qj!33X8GwRA1^s|u!bxS;Cx�>2lg%|cYES4p(`z}Tk!eF z#og%n^7I?>>B+t=}0`C|9JUN)H1NQ_H(24cEUkraWi4viK@};m700*_0 zm-y$?F@pfjqjwYKnR~%7fQo5@hWHm||4jf|kh8QPwBRrr7d-n{5Jg(oBubCRcy&OF z$9h%A_uFrg-6^nd(daMouceq^cwj}I*-ZcCl9K^6)(qGp5^KR1V|-a z0o9&mQui@c0vcQe4ZSVW}G&icJ_izs5^DsYL43{t8K68JYt&dhQsLV&!o z?ktp&fx>QzLwK}dA_LlWLBol-WiLW=m<#S=ZoUFKThP#CE1^;GO zrm(fVnYw?!#-orhBC>Lw@y;?KNL-gC0y;^2m+b+>+B>&Ut!8sFoGrR}leo1;N!meu zTmW)Jc*rjDvy&Xgt^D6_Mwns65hw5X-`s^tk`~%Z%!7@d*MUj0DlBE8Ev^DMbeazdr^(3LYHYqg zPCDOj6CDVAnO0IPrr{p;q_f_xNO0eKbn6paH?87gt5?7WgTtshOx=omANefur% zUxzJb0#oX!l5GeqYx}-Z8+;E@Qt-|x`cY}h!Ww5pYe{OFMqN=r?~oL_>v?ShO6*z1 zMmf||?JGNsmvK!Sh5^|qa>~X`Ev%rXI;B#&j0T~NtViHb^nv`8|$+sYD^&kHmb7B_7{Y{(_q zqAS{>yuhP1?Z`=i^Q0>xvBSy|7pZY(jfs59Vt_+SZiG~dj#g1-7huWCox3G5g*|43 z=e=_-bGBt;W%+rO!%IxJV`|b+1BwtSIREHn{PeIo1k|mTW#AN&c907~CPn;U@GJ{w zMkY{EQYJ(#YJE}n+sl4#d{R#LAxih>naVjZl$L5p14NSwK`ii1{OZ*pt3kfe;ZPs4 zO8`7n2$fr5ClY&jlW zGP5E~FSBmm4_|*u$c@1T(fM}xJbH937=ONf?j8Nk7`dgNA}}Q!`L`K*KN+ie&%~+B85XuxWP_c?|j8Ramn-!)PHrICB zds~L>gKf$*79#(zXY%y4C3I@a`r+wj!agyXURTlQx;Nx7_)+h`SGWAs%=6B9H^B5c zWnS5W{VIQ52|XC88TWv;>aLU4+n16)FtSqw%uZQy;hxS7&lBFCZ$On}E%24x)Qh(v zhrp4O=_`tSagw!NEx&mX2QJ25fGA#EcW51*B!DCfN2EJ}Q{P&KbZpgSJ%A!G(s542 zpZ#}MITq-VP+zeEWYvBE2)DRlD+fY?=cr-5ywx z3HDk!L03UhERBD;pr-upvf2aCzN8+Z@6!A2hOQgQjwGY^WDL*iO;PiVA@?ve)6Z+N zx7gNNbgO@;WRSn3F8jF*ssL0vQhu=Bk1;3Ulnevz*%a#Y1Y{{{mW}((f7U`MG0(Kd z9iE#uM=cJ6+cQmH*R~;^^K&b71#hD85op_lv&mVxkzF()cUBI5lF5o`>oy8_7BCnVc#DIJ2qU&} z8aapP9q*!uQENRl>Ftvln)&o(L!T%+{HguIN?n^l!vb(;0)CevRfZ-*-+kEK(BdyI zmC2EswaTUDd0dN79P*HzzlrAPEv0_4wcpsRNJzeZW>i{vXv;EjJn`M-pFAzEsW#9% zAD-Mih^i6uVAoES2Y^Igjy)f7T!;8eg)ZWdHmSc7Md|n^#KCcNr0|{Uv zBkro5{GDd*d=E`0!%wnw&KPG%KO4YA6Yb!pVw}D)cs!mqhA#A<{Zh}Li8rfi!!#cb zUq<9q6d@BihE>Dl6wwNC7sW)HXGDo`FSQ-sXReNNl+l!71At=N<2fMXNQQ&aOcoJ^ zK~-q9qTL7>h)1WyU{+)MM%GnWzitw% ze{yTHMZa7;e(bYP>$_d4n`FGxKR}~tDN$Eh{Fav*SztzLo)15gBIA*yg4{ky1|io; z8WvN{YjbxV2RQ7TT2o_d*)PC@>^mb%s?x>yX_9L`K|RCWKlK7eew7)psry5TpduC$ zr3n&IGmmgvmk9+WoK~$s^AzA1WEH4rUg1TgK!Q}zt1kKfBP1SGgK8_3{IgQ;)jAtRs)Gh#nM*jZi6)?6%ady96@KYt?6(>3L9nr3;WZ%?lXU{Mep`xjXESH;%qdb*U9V2P?I43 z7x}W5G~Q3(;C6F_bA-`F$HFekOxMrOaU19uuu31h6IppC4S@I_P5c=gAJSL8mkBDV zV#YGQ3hmUxzwfoqh8?jO(oc0uFMLl7qX-WIaC92^RxeX@{8)ol1KK%368Z8i(Ug$T zCCkBYACd~ndWn=G!?Rm{+PxOihP$${C>oZ7$a-~X4x7>R+AA4PZ!PlbV{O(r)1)(i zwNJ`%V~NtUF%XWi{_WJCXaYv-R;qixnk&RzbXs~-n}Y!*aaQ~=rfT3-`eRUOwB|xF z$9AV#L8k-#Sj^i6Z`7!lJ+K_<1NMa>P>3%wr<&cSWLC^M-MSN8_d#Z}k-lLT*fmjd z!{(zpRk9m_eAIN90;7cy2-s=_mG~rpi1CI} zC}zQ^$%KRWjs!zAFPZ{`*#?HsIWP(mi=;pSRYqh&Iubz%qjfasVgv19W1b$shhBJ` zxt^h$bnBu|Vwo263aj#q{P&zVIx{K|Pg`~{vS6ToiL@ypr~?%Dwmg}>?N5@il^_r_ zX}5IyQZ!$mCm9tSsQ|iVQUDWaAjR)~t%!d6It@I^R*|y4IL19Y=U4Mym#;1r?PUzn z-|Z$2stkrWG!MAg29!`ODjTiCP4k7uyrm=Sl$->I*{|*9)P~DDN7(&)}n3C z$0=hRnLxRWKg@M7(Im|Noi@&Y6Eq=Q>lPNMlTpH%q`t)&Ep7Gj2Eef&RuF|WpLze{ z$VAFL!6TqX8{Hy?FObK{z`DTK2C#~q+T+>i%Y8BZrO{7_e26j~+grUEYun!$pVaQ| zSB>WXQ-E|QQ($xY-U~>&;0JvUwU{8i$@6}T{NX}w!#wM<(~bpkw>EQfe-|l78nLy@ zWj~ZTvG7%s-}$^(8w!R4o%{j>IXND6%(i84b9AuvHPcX($z_Ftz;HsnS{H=(~tqX2~ z?-y+Pswmdt*i!1?^2Ub#Z@tDuHBN6T-2mpwVYcI%n)3tN>IhG)i3&?n-N{9IoGC&fG`|klG&d+{_$T%t)rGbD;X^ge1 z_e2A%D~9D2NX)kohC!n!*a#`*45v*fX^;~SW_~LDO-~&SEuPFA31cvFRfq?>W3|fk z_oNgSH=x3mo*WVWsO7gz{yw{(dWvNg6o#N%T}k->@^G)KMxiqyBprJD!nSq)uIP|@ zYZ0{!XmmjKql)NWJuEn9T5`%eEYI}&*UJG3IPYI}IC$>;96x;{YD{t#Gc5O54@u=X zwMhxu54R3Vqd!NEHyPHAV6K?ggO#mycf+$768p80SMUrQcydFb@%mZalOB_afoHc$ zyj*|fWEo-jEh(t*tOLg-Lb@z?V(>D33wxuSTnSFc`APcEZK`ikI-0*KPa%=*2+^BuSu ztN?&J4_$hBLkf_+(T&pKiWjov`VNp7Ph5TDuY86t&@yyvUa@%xo6J|dX<=E= zGug%X)P-r0k)T*+X42U)%DoZ#fZ>Vh2}@er8=FIi`i3hVPbm!ZwQN@y-Cg)btSeLW zqrQ)xPV8L<%1aXEqw*+VRHC(RDQrASbY(!kj6XV(VqZ-$b`#X)ZT7ocHL^ED&{TpD zPqiR(+AkSI(tlb&vm2DM=V&PZz0e7SY#>5Z;I5P*a+p7evIOaR5=%*maBkG|#$+a6 z-N^)lX^mEu_)g<)y)mBqoBz$5l9p$j+XN>Gp3JoUpXlNGHCDWnH1u5ouXvT7-ztr1L1opZ~8m+Y@nxN2;GK*f@GecQ7lQMh89|XgM z7^O+t^0fSZW73UBkyDV~Qz7k#oe*9AGFHCT1uJD1mndj zFB!cB@@k8B=r1fqXQ6~N??2*kq|#*4H4G*~ z$}1YL!GI|>beq~Kmy7+H-s*12z8&@_btfV(#(R8$NwK~>QHmpKfw2SI@a9_S?EXnE z#m)4GY(fXgqm|Y_;mKL_wMtrC8gu?>YwyvnT*wj&+do85wvh|yS=d-?k+*5&qzV>&o9ep$bo=efTF(UBxJ-K_ z1x`5o&bkad}7%*4Q&iu945IX|b+XfT8i;8I<>{i9e@ zKxRY3Q+L#f*iDy+iCUNGh|HiBXIaHbw;*}$Gf*8_f6;h)yRt$rnz4ueKQOzl~09} z$QOnCK2(fWsx^7b4CbF--t!NSc=r3f(GM~|6t|o9$N4lb0O5dK1gh7*;Dx5dlz0p( zy2)c&`zY%(Rsv0KxKgJu86sE~ymbJ})Wf!G7f_}9+9eQ?qcKyOn}F~8l{^6OekK0( zk3qoq{xh@J<_M}yXhVr{4@pc`v9O9{LA$0aI?D@M7_=vvhl5A5A+I0 z*CUebDjwQjCW%!ibvi}20;PTRKEnMuqV<)?bY z;h#A`AH1=Sd)siN5-In2J{dlvG61XUCQqz4FHlPfNNc# zE`{UdBZ4@9%BKxvDe4ND4LogN-rvE@_@jX85<2@1`QYWiClzH51M;^$+A z!8Q&^lBO4*ZCi*yiVA{MyGJdA_#SsBQMvo)9Qx8})rbLyKclRpPQ?$+UGH|)!u9Z5 z@3r{FZVnH|y}v8-@P(sJXF=fHV|@jlU@*w*Pt-7ZzsF-1C6b&wL{A&f+C%4?HPjLc z3h6$hZ&&Fjh>XWyqLH3v{JwRy5px)w;$}erqivK?A@u%?#Dt7lVpJTD14>4 z&qoNgv(<}e!8iHIduTl3a-+Zf?^ffe0y*^mh=?$ zFwK#rFb4)}0+nc!^LR79L@&g_ptT>w+RfN~j|i7|vJRwgJBgSZKgl7U{JQHl=`_l{ z81t4qC(h$<8(3 z8UraGVm5<^`SUCEG_M=v2?`q(lZl){fhr;Jctg=)iq>) zSz)(P`P-#~h4XXO9=d_iX=SY8@Mj0Ney~fx3%3(CMrhM!|5^YyXSw3MgyERU&UY^N zHU$;Fw%F*MvClx7>S(mg1ddeDl0Rud024bE1o{ZPgoE0Q8B)Q3BUTLSFH>Sog>hX| z(}oA2q=X9rOK!tH3<1jl8;c0Qk!riJ^M2z2o7&|s^gk>2tL`}uZ@ZfUUx$y%1VnHp z6Vhk*LX?S^$1*$y0je%snb!7s;w#5%X9iikmS!$vc~j=}t*{$dme|xB1Z7JInS6bD zPYzH3{oz+gDbPX(6rfOqFGn%vCg1&F=42)_bXS#jpj1H#(?BI`1pj`3OcAY^p>o{_ zq-onfIad2Gz6zZqQlC3=p0hC6c(Q19cUJv*dV5YTjUl2(y*4jw^B@bVDy4moCu)T> zuqi=_^nyK1C-DNVX%n=p-QZjwb+sc=>K)EpVJ-nTuZ@pSRmWWD+?j0?C~vG)C=bVk z#gjPfsFNdg4eE#K+;1Jg)Op%o?7FB69751+F2!aj|CGQ~FHk)ajELt*uYNSehSy%6~zt#1>FS_gVS z;+DP02^MK2eMSRN?7S*NLS=FtRtWUWr-4knpl@;z6%baZ^Rh?MdpPyqjGtm9fmEv^ z{GkftIaW&i+pHAfAjc{t&#wj9GTf&PeM^=9R+iHb(~7D1#Ql1XM4d^ z^>Fu;NPgFcRwp5g>aU&LBQ;EB-7{Zii%|Xj#B5KwuL}s9SA}Tamt`5{pp%pOuFeCk z&2%Q#-ep*=W7n9lC2hTNX1uno%T|^LgElNr(uN(eTg9kDlID1&$uw;5>4+jQ>Y)Ao zrjcD7Yi8YOJjm&5Z`YYcpmlywE6+LRym;kkNmiP5Sddd|x{Oc7^GUGm)Se*~S3<7< z8gm3mQvd*2-TSkUWB_uprwVFdSn#F%{$a|Wc;%dd((_bXyTObl7_ej0)HsjdOh*5m zC@|d8m7Bm5d)!fRPIGDi7H|6$d7?J&Mm}v+=d^ z{k2_(5v2i)UHxhEm<%wYXkBdK197^)i^N&P$Z_hnBpa=l z&19cr&oAR`ao9X=Ggi|bi41|Nlz}&y~8Gh)Z(kC1IsgP9M5|Ay*!vzpO6UvGE>n?dSDd~oen`C{H!JK<{pjh6$iZ z*`v^1&di)SS|vayIDQ_V1bnf+)(D++p=H8>??q>+2oq5jO^ImM%(q2SpNx^tx98{iB5&n+BSE;&{Z~dfl52q)2CYSf1XuLRgQlzT|8Umk_)I@dzxcD=H*;Nh5>3B z7I^n%2fX?2&4KK<(o`QE=9K_>s;tJ)%9n0jM>t*qSp98r4@4&XZ5Jgc8pj``KH2*6 zwbOo>SW>y&1cnQR%UQaZ`3v`&u9%$muGGB%zY+D6$5ttG@Z7H z(G2?1rLw!)K+KM}6OY2%c z7nFY=M-%zcYKB*rjVMi$*!QJ9z={ep_Z`$V=PqcGj;b%mOFi+3!GkhUXaI@a12t#; zZBbM#umOnG#-Qw+Pj$s4{ZCPIMD#p)m>-6D8eB}@A#OURO`~+=H7~*91y$JxH*SwL zi6fAxrQ>vk<;iK1>Q`Rr$i_hgep~CaCsaDmQt!BR&Y^48cr&Dgg>{FQp|Uh$@QN0R zb7`4)$<~!uSPW}byXnX1Pr#dTf^VUK9ehoOP4N0Z0=pRO9FwAH(@nhdaPba4n%8Po zgI+tGEDE1;9MN^3O7s$PzHLGdJUPiGax5x$`w2?g0|`(diYvgT*UHYA4e58p%sUZE z)^>@6B5F6MF}bD=bNk;0iH6j{YjE8N1Z{`Tql-{JU8ApeRmEA~Hh{R2l_^po$dHj+ zWduU~&M@_`NKtc~a8P-ak&;)G&_czMTr1g?sHmk`ss+chxz>kmjV&7S?qOz}|^y@>iY+m|9T%>krW8L-pn|G#A)KeT#gw^~)|N?U>`SY1AMwk|8y!}A09`0F9*MIh(( zLnX$qoSUXAk8cB!ZA9s_UFrKHuVmE@j7(~_OlZO6%+i>Do6u&d+IdNom6HRtqdv5a zX45t1iiYMNpn85|&4`fCI)&;Jz-&}qdz3^rsk#7YecU5QS19{3I7$##oV*<|( zwq?}v8Md~KC4t_}J>F{ImYClAzPR@=+r*P#qj!2cU_uznri!gfUXQjRy`|KWn=g2= zYsxufyuPtgn7zbxD4KSzA9%T%4tA+r)d>0-dVkTwV#FHMv!F39Xa3eqik`C_@`%Mh z=QgUJWgjdbgZHaaKtA%~Pi!4bj*iGjE!jIm3Q+GOhl4{uB_zfwT*@k3Oe8+%18U5? z0k|6*AoE?;Vr$pFcgF7OPkfb+(yk}6vnP{#D3wN1|!O%-`tM5Okhviit1MIcnzZXU#v4<-nF>#JLl8jVm=aFZAH}yR4s?bm# z+g2%8p4~OnsmRskSSflV8kO3sJd2u~2YnNx1C+lt#q_OlY3R~iH?YU+X8nz6C5@~- z`eZZCH7KptnSh-hD)752dcA7Tf7*Z1#C0A1qK!gM{dA)Ga7h^dKbkVJkW)vjnM4Vt ziNV?2AI}|O%~3-QGd^&X&b@%kzU_NiHIBI5)Ge>2^%ZsRU5@i_^`V$Mw;* zZ@xpHp#7G|isx!!EdmP{&HA>9Dr}dmuhi+C-r5?W71Zl*6{;NwhBU#?eiN5)oxr8E z>?I7!#nC6Y(F5xso*5X6D|(2dskAD4O1&YZloe0|l`p)J+wPu_BVd4MsPyQpC&u_* z;Q4v;giZIOb6sbN1ME9i>)XR;Yxa?`Im|G&yN1xAA^j#BdD$M*Ns+~EL4)?AcFD{x zDw^~cgzg;2tIz5H@@ycWEJI`(1OcnqP@3qX*wUr1wbFx3Nm*YygI zqc$61)Eqypko&+!%ze6sfFQGyxZdmaXw*Fr2ex#3dWii&Bs7XVFH}E*JnBv~0==If zKa4ED>t_hHA`qhi9D~ANWi~HhsbOE07}@*zmVZE>eU8kE{01;m)uyzP_fE_Pw-Xp- zMu;_u6_^77M$4-~)FF|kev9mZV|7ISOIL9~zYZ&1lZucF9_f9dNNb_UAJy9ur^ZIU zw4JSbuk3A!=Ly!)-xW}_=N)F^8IemZHQqq59&&4-c=yjHX|(Aug`>vPsm z0n@p)7gAd{_e09KGk%Pb3n3rIX#V;929frNo6`9znhyl_+&646j6X{wg1vRq7x29e z@r~qNS|>NUnVvZinJHgaw^7-A2BPQF`r(~`Cg1hu!2F1a((93Go-_cY;`YlMd++_@ zBbF!BQo*y&P-6?4!P-8f_PSMQw^hLX`<385^*2t@*)YKP_u{*y{GVk;RNDL>61JVG z2g@8d)fenj11_#%>lW=j1-akhvIrMF)V^8wo0)EI+euw)VP+2ZYw@yuVzgNG%xNeUid|buLkFFGY zZSQqX5b^*A^VA$sWV8eHA-}cR2p@o`*EWKlZe}+MyH)DJ9ox8Df)5dstAZ~Ho2!D& zc$cf99~2&|f-7qyR)DW+OPc*g6_-~`o%}jha+T5ZPRO-SmkX)@y6g8>v(EgY4lcrw zE!zAZ0wsoFY#ptEAf)|!zxFYByY!2!>eo(K2X6rP?ZaaC&?*eZzInuN6PX&^=vG4l zZ&HRpm%-V7&|KMu-F1XcW&Qmi>aO9-!*`oJ8e;azgN>yoCLUG6s6`bDNJ3|U4%lfO z&>P-tnW|6@XS}`jjllSkd1$d-hJZ2jmFK)!yeU-%bxUY+Mty1=?ZH_v^GuesznSYK zA20xt<6r!(FiWn0z;LkEr={3GtKya50>c#+($6XqP9I6HTGpxm%SbjWuMkE zG1!bXrpQga{Sc6c8${HU_jC~u5mf>&yHeI>bjH~PR@Y5OpH6+MJ6vnn=LsNiQhE*joSIgH8U=UovjtgJ_bcdmwjY{Phl=VUp7e2-~5r8D#+R zwdnpGo&SDoqHL|VH+qI2O&SQ!CdwYusLoWmDdat)aIp}+7unq*Vaui_8>Mm{!yn3>r((qqrv6A8WCFTk)?)-SGZk*ZaG!!TUhr=<5T=xYzt9$ zIwC)%t*~FA$R(tRb2KopMH56u>z74DL(KEDWhg(8(-Rl*=(3`k2RQYkQ$%BwfCa)D z;J=cHqoW-}Dn$~uqd(ddqNq3}>;HnTX(BV3k{BV6>|H^kJ6JS1ytx4C6WcnY4gR5H z4y6tgXe$H4lP&^8P*u$7t4%NNha4DB;1+1 z{qEY`{BqW{ZWM@jrHxV62!$Suqe>Z>DG6#<5tp&V3x*~wi_snWPvzo6!fIem3HxlW z#K4kjj5ptJfNHQf7*L-?g~DJ2;AUu9wnuiGmBJ2*uSY_|cI8kdx@DY1v7tmrjzT&9 zpN&R=CC3RRLsit`H08nHmesN{14g0+Ys_mp15NHo?~5Mw(_el`D#DgWu&&AwA@oY; zhNftTR4`z-E^)<39#{{{9`$rzxS6Oa6ri%dp<#LRl>nTaUQA$QL_i5kDinW%=hRPz zTV7ba4cm#ON_e9rp*v{|%bk-(L`JBiVGlDd{)%48pA56V54r<4QK$?hNI02c!59^m zhh;fH!KhA!$t1j({7<7$NR%SUiddOyFi8}u>_#wr4Igg`nA4wiWdh^R>*vk?nIZWN zum}$G(}c2-2)oP_Wp|F3u>d`9Gm#4IyVm1ikKvrS@=7h~r z$N3FVe=g55u9!(ByEBsuII05Y@AS^2Bfs9HbfmTy(9azZL0^MJ#^G z!4&C-pZd8S^8=!tMIfo#gB#0qhXDV@M*4ey;4H0TZC*rtEfYV@6BP>2vAF1={UV_= zwO$|_W3zyvj?0gajJO#JLUmjr$t7bRQwDWhkj;LIs^EX)oaARy!Op=Z4@!y9l-{p` z0wc$b-7}iYY+DfPg8llx$g~A@hLz~#Oa1cncacQB2%|(JbH@UZN(BcRnf=d5WREj3 zaYkEESdzW4;h->d0|M*RS`bt*;b)YTF8KFZ*V>oORVWyTdFmecz78Tf`#c=qEJ4LL z1UM(V0_>%=wv4rSTwp(-V7$*d*0o1R=j82}wzJ!F>_mByZlfM!5Mlt|0qqI0H2~ir zG!9}eejyLQPLpiTiIcRUa3>KYi46QSq(mZ=M%IFN_v;!}^!B%O4)Wffe!$uqR+WQ= z4J1Si=0w{%-rXtF9=l|K2MH{ZA|yJoY!X`x2%BZ}P8X;Of>fBZ*xl+=_DqUILgMbV z+{(@pH`jq81rZFxZ%Igl!7#_&3#+Emnu{-$rExic=U*}LtWhOakq&#QT^R@tU0Z(y zk{LsO?ItSorJ6xbJ-QS#cg;6kz=c>O4rAf5c}4S?Gq#j}2{YJ9p{Ci( zX>TRj%YL8tGnR7>$g^53udM1@*ykTw)3(R5)RZ9G0@zv5=lKYE)1J)lRE@q>7=O?e z2Fo4cz7R8Lk?=&~$zuMU)(g8+vZc7cEf_)pR&tT0`y5W|azp{tTTV27S4(`AN&+&v zbNj2JiV1o%SKt0(b8N*vq4aZQ@`KBeiCi4N90Tkr$_VHm9}(RV+!vf}ZgLNMwR;zE zG`|RX+#fG--N@e-6Y;~p-d%RoS_Jsc^~66f$^br6pH@{$zsSC+ZF7wR9(U^%^!Ba* z)Gzlp6r$ujrmjZEzJBDG{2?C4k>|vH2Q!v>!3Zo%Ov89i@6A7Joiej8x!0i2x5K-f zRimCa*suDVUt6)&5e|F`e8l9bwM{K6u+mpgp_O;cw_!+ic3Rd%j_<7<>DC@lhKg)LKFks6z@Op3nAnZ$l zm|Mb&(jlspM+FjL8CQze=P?Ais&_>JIHr(gsqqB^jqH4yDO0QO#F=weXHdy;dIFS4 zNon4Zs8wTv?Fi4(WJWkp}2M(?A`=WJw2i8d0rCaIMm+bHkdnz85IVD z4^N-R)TiE3F1rkU!boP_8*5FWBL2N*yC%E4Hf#R0uX?JfaILSP5+|lSU;e(e+j||v zU((fEY7=VPd1m@vhvImm$A1&0Ape^hz0)Bn_9 za8MP5!%(-n4GpXO*~c7I3^C=jRUqP1*ZlZnSQTnTPbb;$Fs#xO{rl|>Yq#>~PQFsJ z;#easj~ZH~j{hA#48BsxKong5*387JK=RfVi?>WvziK9mu2L`Wdopv_%%qvf+T1^C z0kEl0Ga+8Y?ze4USq^y4w)n+UZOIP$yp9V)1g5@brcl}L0goob+I$+CsimHzAZ*D={gq+71BcJf@yAkmJf4VNAufgWpm2 zy_zkOgT@kb*rI);$Yw|-q`E3Fuiy?su{*#_ms+ZuXUIxe|Z~*BhH@x}Yf6yBJ z_X86?dIlzpzAY4;2?hQ|D<3b<=@LKl-TcdQ7Vuc0!%AInWv;1zKHFkBCNeAc$Il9g zTwM2w;Eh)|xdTM_&)nE{W@oJ26G+r1hZe6YL1^2h#T3m+nn$C>XpR<1mV+i!v2A38 zy@gpO>f%_UkV#g+kmU3x^~9+RGP8#cXLa&$9+CIJvqR3#qFll# zkugQfyx%gQV5;g-dEOari$}--q&<~NkGe_YW*oouq``PktM^1&r1@de?UC3Fs+TG% z%VnMB1^_eKy}9}DOJ{gjzw2@dn3Y Date: Sun, 12 Nov 2023 21:02:11 +0100 Subject: [PATCH 27/27] fix(doc): reword monitoring (#4122) --- docs/operating/monitoring.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operating/monitoring.md b/docs/operating/monitoring.md index afd3d111dcb..335be31b52c 100644 --- a/docs/operating/monitoring.md +++ b/docs/operating/monitoring.md @@ -10,7 +10,7 @@ We provide three Grafana dashboards to help you monitor: - [searchers performance](https://github.com/quickwit-oss/quickwit/blob/main/monitoring/grafana/dashboards/searchers.json) - [metastore queries](https://github.com/quickwit-oss/quickwit/blob/main/monitoring/grafana/dashboards/metastore.json) -Both dashboards relies on a prometheus datasource fed with [Quickwit metrics](../reference/metrics.md). +Dashboards rely on a prometheus datasource fed with [Quickwit metrics](../reference/metrics.md). ## Screenshots