From 22fae74111a58d075140b8cf315bfcf102d62b3c Mon Sep 17 00:00:00 2001 From: Adrien Guillo Date: Fri, 26 Apr 2024 09:36:18 -0400 Subject: [PATCH] Populate `items` field from Elastic bulk error (#4900) * Populate `items` field from Elastic bulk error * Add `make doc` command --- Makefile | 3 + quickwit/Cargo.lock | 363 +++++++++--------- quickwit/Makefile | 4 + quickwit/quickwit-ingest/src/ingest_v2/mod.rs | 31 +- .../src/elasticsearch_api/bulk.rs | 6 +- .../src/elasticsearch_api/bulk_v2.rs | 202 +++++++--- .../src/elasticsearch_api/model/bulk_body.rs | 15 +- .../model/bulk_query_params.rs | 2 + .../elasticsearch_api/model/cat_indices.rs | 4 +- .../src/elasticsearch_api/model/error.rs | 30 +- .../src/elasticsearch_api/model/mod.rs | 2 +- .../src/elasticsearch_api/rest_handler.rs | 5 + quickwit/rest-api-tests/run_tests.py | 39 +- .../bulk/0001-malformed-action.yaml | 9 + .../0002-validation-failed-index-missing.yaml | 9 + ...3-validation-failed-no-requests-added.yaml | 8 + .../bulk/_ctx.elasticsearch.yaml | 1 + .../es_compatibility/bulk/_ctx.quickwit.yaml | 3 + .../scenarii/es_compatibility/bulk/_ctx.yaml | 2 + .../index_not_found/0001-index-not-found.yaml | 28 ++ .../index_not_found/_ctx.elasticsearch.yaml | 1 + .../bulk/index_not_found/_ctx.quickwit.yaml | 3 + .../bulk/index_not_found/_ctx.yaml | 2 + .../index_not_found/_setup.elasticsearch.yaml | 6 + .../_teardown.elasticsearch.yaml | 6 + 25 files changed, 505 insertions(+), 279 deletions(-) create mode 100644 quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0001-malformed-action.yaml create mode 100644 quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0002-validation-failed-index-missing.yaml create mode 100644 quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0003-validation-failed-no-requests-added.yaml create mode 100644 quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_ctx.elasticsearch.yaml create mode 100644 quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_ctx.quickwit.yaml create mode 100644 quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_ctx.yaml create mode 100644 quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/0001-index-not-found.yaml create mode 100644 quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.elasticsearch.yaml create mode 100644 quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.quickwit.yaml create mode 100644 quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.yaml create mode 100644 quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_setup.elasticsearch.yaml create mode 100644 quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_teardown.elasticsearch.yaml diff --git a/Makefile b/Makefile index d0f62bca4f0..f86b2995679 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,9 @@ docker-compose-logs: docker-compose-monitoring: COMPOSE_PROFILES=monitoring docker compose -f docker-compose.yml up -d --remove-orphans +doc: + @$(MAKE) -C $(QUICKWIT_SRC) doc + fmt: @$(MAKE) -C $(QUICKWIT_SRC) fmt diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index f596d10cbbe..65b72f0ff8c 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -60,7 +60,7 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ - "getrandom 0.2.12", + "getrandom 0.2.14", "once_cell", "version_check", ] @@ -72,7 +72,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", - "getrandom 0.2.12", + "getrandom 0.2.14", "once_cell", "version_check", "zerocopy", @@ -95,9 +95,9 @@ checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" [[package]] name = "allocator-api2" -version = "0.2.16" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "android-tzdata" @@ -189,9 +189,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.81" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" +checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" [[package]] name = "anymap" @@ -260,22 +260,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a" -dependencies = [ - "flate2", - "futures-core", - "memchr", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "async-compression" -version = "0.4.6" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a116f46a969224200a0a97f29cfd4c50e7534e4b4826bd23ea2c3c533039c82c" +checksum = "07dbbf24db18d609b1462965249abdf49129ccad073ec257da372adc83259c60" dependencies = [ "flate2", "futures-core", @@ -315,18 +302,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] name = "async-trait" -version = "0.1.79" +version = "0.1.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] @@ -776,7 +763,7 @@ dependencies = [ "flate2", "http 1.1.0", "http-body 1.0.0", - "http-serde 2.0.0", + "http-serde 2.1.0", "query_map", "serde", "serde_dynamo", @@ -840,7 +827,7 @@ dependencies = [ "bytes", "dyn-clone", "futures", - "getrandom 0.2.12", + "getrandom 0.2.14", "http-types", "log", "paste", @@ -901,9 +888,9 @@ dependencies = [ [[package]] name = "backon" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c491fa80d69c03084223a4e73c378dd9f9a1e612eb54051213f88b2d5249b458" +checksum = "d67782c3f868daa71d3533538e98a8e13713231969def7536e8039606fc46bf0" dependencies = [ "fastrand 2.0.2", "futures-core", @@ -1070,15 +1057,15 @@ dependencies = [ "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", "syn_derive", ] [[package]] name = "bumpalo" -version = "3.15.4" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytecheck" @@ -1189,9 +1176,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.90" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" +checksum = "17f6e324229dc011159fcc089755d1e2e216a90d43a7dea6853ca740b84f35e7" dependencies = [ "jobserver", "libc", @@ -1277,9 +1264,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.37" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1287,7 +1274,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -1381,7 +1368,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim 0.11.0", + "strsim 0.11.1", ] [[package]] @@ -1559,9 +1546,9 @@ dependencies = [ [[package]] name = "crc" -version = "3.0.1" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" dependencies = [ "crc-catalog", ] @@ -1789,7 +1776,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.10.0", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] @@ -1811,7 +1798,7 @@ checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ "darling_core 0.20.8", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] @@ -1853,9 +1840,9 @@ checksum = "f578e8e2c440e7297e008bb5486a3a8a194775224bbc23729b0dbdfaeebf162e" [[package]] name = "der" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" dependencies = [ "const-oid", "pem-rfc7468", @@ -1980,9 +1967,9 @@ checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" [[package]] name = "downcast-rs" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" [[package]] name = "dtoa" @@ -2010,18 +1997,18 @@ checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" [[package]] name = "either" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" dependencies = [ "serde", ] [[package]] name = "elasticsearch-dsl" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e94d61648a38799f94a0636b34fa435581fc9a2587db354efc3e7a76cb31f14" +checksum = "83cc59a4831e22b0001ef8bff1898d3cee071ab16e0ee192c321ebdebbd7782e" dependencies = [ "chrono", "num-traits", @@ -2149,7 +2136,7 @@ checksum = "03cdc46ec28bd728e67540c528013c6a10eb69a02eb31078a1bda695438cbfb8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] @@ -2232,9 +2219,9 @@ dependencies = [ [[package]] name = "fastdivide" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25c7df09945d65ea8d70b3321547ed414bbc540aad5bac6883d021b970f35b04" +checksum = "59668941c55e5c186b8b58c391629af56774ec768f73c08bbcd56f09348eb00b" [[package]] name = "fastrand" @@ -2350,9 +2337,9 @@ checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" [[package]] name = "fs4" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b1e34e369d7f0151309821497440bd0266b86c77ccd69717c3b67e5eaeffe4" +checksum = "21dabded2e32cd57ded879041205c60a4a4c4bab47bd0fd2fa8b01f30849f02b" dependencies = [ "rustix 0.38.32", "windows-sys 0.52.0", @@ -2446,7 +2433,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] @@ -2522,9 +2509,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" dependencies = [ "cfg-if", "js-sys", @@ -2664,9 +2651,9 @@ dependencies = [ [[package]] name = "half" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eceaaeec696539ddaf7b333340f1af35a5aa87ae3e4f3ead0532f72affab2e" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ "cfg-if", "crunchy", @@ -2876,9 +2863,9 @@ dependencies = [ [[package]] name = "http-serde" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fb7239a6d49eda628c2dfdd7e982c59b0c3f0fb99ce45c4237f02a520030688" +checksum = "1133cafcce27ea69d35e56b3a8772e265633e04de73c5f4e1afdffc1d19b5419" dependencies = [ "http 1.1.0", "serde", @@ -2949,9 +2936,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186548d73ac615b32a73aafe38fb4f56c0d340e110e5a200bcadbaf2e199263a" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" dependencies = [ "bytes", "futures-channel", @@ -3033,7 +3020,7 @@ dependencies = [ "futures-util", "http 1.1.0", "http-body 1.0.0", - "hyper 1.2.0", + "hyper 1.3.1", "pin-project-lite", "socket2", "tokio", @@ -3136,7 +3123,7 @@ checksum = "0122b7114117e64a63ac49f752a5ca4624d534c7b1c7de796ac196381cd2d947" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] @@ -3239,9 +3226,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.28" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" +checksum = "685a7d121ee3f65ae4fddd72b25a04bb36b6af81bc0828f7d5434c0fe60fa3a2" dependencies = [ "libc", ] @@ -3283,7 +3270,7 @@ checksum = "b9ae10193d25051e74945f1ea2d0b42e03cc3b890f7e4cc5faa44997d808193f" dependencies = [ "base64 0.21.7", "js-sys", - "pem 3.0.3", + "pem 3.0.4", "ring 0.17.8", "serde", "serde_json", @@ -3399,8 +3386,8 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "http-serde 2.0.0", - "hyper 1.2.0", + "http-serde 2.1.0", + "hyper 1.3.1", "hyper-util", "lambda_runtime_api_client 0.11.0", "pin-project", @@ -3438,7 +3425,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.2.0", + "hyper 1.3.1", "hyper-util", "tokio", "tower", @@ -3476,13 +3463,12 @@ checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "libredox" -version = "0.0.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.5.0", "libc", - "redox_syscall", ] [[package]] @@ -3814,9 +3800,9 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "912b45c753ff5f7f5208307e8ace7d2a2e30d024e26d3509f3dce546c044ce15" +checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" [[package]] name = "match_cfg" @@ -4004,7 +3990,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" dependencies = [ - "getrandom 0.2.12", + "getrandom 0.2.14", ] [[package]] @@ -4186,7 +4172,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] @@ -4222,7 +4208,7 @@ checksum = "c38841cdd844847e3e7c8d29cef9dcfed8877f8f56f9071f77843ecf3baf937f" dependencies = [ "base64 0.13.1", "chrono", - "getrandom 0.2.12", + "getrandom 0.2.14", "http 0.2.12", "rand 0.8.5", "reqwest", @@ -4315,7 +4301,7 @@ dependencies = [ "chrono", "flagset", "futures", - "getrandom 0.2.12", + "getrandom 0.2.14", "http 0.2.12", "log", "md-5", @@ -4380,7 +4366,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] @@ -4574,7 +4560,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] @@ -4704,11 +4690,11 @@ dependencies = [ [[package]] name = "pem" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310" +checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" dependencies = [ - "base64 0.21.7", + "base64 0.22.0", "serde", ] @@ -4729,9 +4715,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.8" +version = "2.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f8023d0fb78c8e03784ea1c7f3fa36e68a723138990b8d5a47d916b651e7a8" +checksum = "311fb059dee1a7b802f036316d790138c613a4e8b180c822e3925a662e9f0c95" dependencies = [ "memchr", "thiserror", @@ -4740,9 +4726,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.8" +version = "2.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0d24f72393fd16ab6ac5738bc33cdb6a9aa73f8b902e8fe29cf4e67d7dd1026" +checksum = "f73541b156d32197eecda1a4014d7f868fd2bcb3c550d5386087cfba442bf69c" dependencies = [ "pest", "pest_generator", @@ -4750,22 +4736,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.8" +version = "2.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc17e2a6c7d0a492f0158d7a4bd66cc17280308bbaff78d5bef566dca35ab80" +checksum = "c35eeed0a3fab112f75165fdc026b3913f4183133f19b49be773ac9ea966e8bd" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] name = "pest_meta" -version = "2.7.8" +version = "2.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "934cd7631c050f4674352a6e835d5f6711ffbfb9345c2fc0107155ac495ae293" +checksum = "2adbf29bb9776f28caece835398781ab24435585fe0d4dc1374a61db5accedca" dependencies = [ "once_cell", "pest", @@ -4846,14 +4832,14 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -5136,12 +5122,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.17" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3928fb5db768cb86f891ff014f0144589297e3c6a1aba6ed7cecfdace270c7" +checksum = "5ac2cf0f2e4f42b49f5ffd07dae8d746508ef7526c13940e5f524012ae6c6550" dependencies = [ "proc-macro2", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] @@ -5189,9 +5175,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.79" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" +checksum = "a56dea16b0a29e94408b9aa5e2940a4eedbd128a1ba20e8f7ae60fd3d465af0e" dependencies = [ "unicode-ident", ] @@ -5204,7 +5190,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", "version_check", "yansi 1.0.1", ] @@ -5536,12 +5522,12 @@ dependencies = [ "anyhow", "futures", "heck", - "prettyplease 0.2.17", + "prettyplease 0.2.19", "proc-macro2", "prost-build", "quote", "serde", - "syn 2.0.55", + "syn 2.0.59", "tonic-build", ] @@ -5762,7 +5748,7 @@ version = "0.8.0" dependencies = [ "anyhow", "arc-swap", - "async-compression 0.4.6", + "async-compression", "async-trait", "aws-sdk-kinesis", "bytes", @@ -5989,7 +5975,7 @@ version = "0.8.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] @@ -6318,9 +6304,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -6396,7 +6382,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.12", + "getrandom 0.2.14", ] [[package]] @@ -6492,11 +6478,11 @@ dependencies = [ [[package]] name = "redox_users" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" +checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" dependencies = [ - "getrandom 0.2.12", + "getrandom 0.2.14", "libredox", "thiserror", ] @@ -6565,7 +6551,7 @@ dependencies = [ "base64 0.21.7", "chrono", "form_urlencoded", - "getrandom 0.2.12", + "getrandom 0.2.14", "hex", "hmac", "home", @@ -6658,7 +6644,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.12", + "getrandom 0.2.14", "libc", "spin 0.9.8", "untrusted 0.9.0", @@ -6753,7 +6739,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.55", + "syn 2.0.59", "walkdir", ] @@ -6908,9 +6894,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" [[package]] name = "rusty-fork" @@ -7046,7 +7032,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", "thiserror", ] @@ -7058,9 +7044,9 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "security-framework" -version = "2.9.2" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" dependencies = [ "bitflags 1.3.2", "core-foundation", @@ -7071,9 +7057,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.1" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" dependencies = [ "core-foundation-sys", "libc", @@ -7112,14 +7098,14 @@ checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] name = "serde_dynamo" -version = "4.2.13" +version = "4.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b652e4dd5549c24a4ec779981278cccae2f85b4d5649441c745d58866e20283" +checksum = "e36c1b1792cfd9de29eb373ee6a4b74650369c096f55db7198ceb7b8921d1f7f" dependencies = [ "base64 0.21.7", "serde", @@ -7127,9 +7113,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.115" +version = "1.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd" +checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" dependencies = [ "itoa", "ryu", @@ -7250,7 +7236,7 @@ dependencies = [ "darling 0.20.8", "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] @@ -7288,7 +7274,7 @@ checksum = "a9bb72430492e9549b0c4596725c0f82729bff861c45aa8099c0a8e67fc3b721" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] @@ -7754,9 +7740,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "strsim" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "subtle" @@ -7777,9 +7763,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.55" +version = "2.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "002a1b3dbf967edfafc32655d0f377ab0bb7b994aa1d32c8cc7e9b8bf3ebb8f0" +checksum = "4a6531ffc7b071655e4ce2e04bd464c4830bb585a61cabb96cf808f05172615a" dependencies = [ "proc-macro2", "quote", @@ -7795,7 +7781,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] @@ -8069,7 +8055,7 @@ checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] @@ -8121,9 +8107,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.34" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", @@ -8154,9 +8140,9 @@ dependencies = [ [[package]] name = "time-macros" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ "num-conv", "time-core", @@ -8234,7 +8220,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] @@ -8293,9 +8279,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.20.1" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" +checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38" dependencies = [ "futures-util", "log", @@ -8438,7 +8424,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ - "async-compression 0.4.6", + "async-compression", "bitflags 2.5.0", "bytes", "futures-core", @@ -8485,7 +8471,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] @@ -8583,14 +8569,14 @@ dependencies = [ [[package]] name = "tungstenite" -version = "0.20.1" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" +checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" dependencies = [ "byteorder", "bytes", "data-encoding", - "http 0.2.12", + "http 1.1.0", "httparse", "log", "rand 0.8.5", @@ -8627,7 +8613,7 @@ checksum = "ac73887f47b9312552aa90ef477927ff014d63d1920ca8037c6c1951eab64bb1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] @@ -8656,7 +8642,7 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34778c17965aa2a08913b57e1f34db9b4a63f5de31768b55bf20d2795f921259" dependencies = [ - "getrandom 0.2.12", + "getrandom 0.2.14", "rand 0.8.5", "serde", "web-time", @@ -8823,7 +8809,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] @@ -8832,7 +8818,7 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" dependencies = [ - "getrandom 0.2.12", + "getrandom 0.2.14", "rand 0.8.5", "serde", "wasm-bindgen", @@ -9002,11 +8988,11 @@ dependencies = [ [[package]] name = "warp" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e92e22e03ff1230c03a1a8ee37d2f89cd489e2e541b7550d6afad96faed169" +checksum = "4378d202ff965b011c64817db11d5829506d3404edeadb61f190d111da3f231c" dependencies = [ - "async-compression 0.3.15", + "async-compression", "bytes", "futures-channel", "futures-util", @@ -9019,13 +9005,11 @@ dependencies = [ "multer", "percent-encoding", "pin-project", - "rustls-pemfile", "scoped-tls", "serde", "serde_json", "serde_urlencoded", "tokio", - "tokio-stream", "tokio-tungstenite", "tokio-util", "tower-service", @@ -9080,7 +9064,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", "wasm-bindgen-shared", ] @@ -9114,7 +9098,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -9268,7 +9252,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -9295,7 +9279,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -9330,17 +9314,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.52.4", - "windows_aarch64_msvc 0.52.4", - "windows_i686_gnu 0.52.4", - "windows_i686_msvc 0.52.4", - "windows_x86_64_gnu 0.52.4", - "windows_x86_64_gnullvm 0.52.4", - "windows_x86_64_msvc 0.52.4", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] [[package]] @@ -9357,9 +9342,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" [[package]] name = "windows_aarch64_msvc" @@ -9375,9 +9360,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" [[package]] name = "windows_i686_gnu" @@ -9393,9 +9378,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.4" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" [[package]] name = "windows_i686_msvc" @@ -9411,9 +9402,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" [[package]] name = "windows_x86_64_gnu" @@ -9429,9 +9420,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" [[package]] name = "windows_x86_64_gnullvm" @@ -9447,9 +9438,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" [[package]] name = "windows_x86_64_msvc" @@ -9465,9 +9456,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] name = "winnow" @@ -9581,7 +9572,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.55", + "syn 2.0.59", ] [[package]] diff --git a/quickwit/Makefile b/quickwit/Makefile index 6ec296839f2..3d90351fc53 100644 --- a/quickwit/Makefile +++ b/quickwit/Makefile @@ -1,6 +1,10 @@ help: @grep '^[^\.#[:space:]].*:' Makefile +doc: + @echo "Running cargo doc" + @RUSTDOCFLAGS='-Dwarnings -Arustdoc::private_intra_doc_links' cargo doc --all-features + fmt: @echo "Formatting Rust files" @(rustup toolchain list | ( ! grep -q nightly && echo "Toolchain 'nightly' is not installed. Please install using 'rustup toolchain install nightly'.") ) || cargo +nightly fmt diff --git a/quickwit/quickwit-ingest/src/ingest_v2/mod.rs b/quickwit/quickwit-ingest/src/ingest_v2/mod.rs index 192b07c670f..55c6a0aee9f 100644 --- a/quickwit/quickwit-ingest/src/ingest_v2/mod.rs +++ b/quickwit/quickwit-ingest/src/ingest_v2/mod.rs @@ -33,6 +33,7 @@ mod routing_table; mod state; mod workbench; +use std::collections::hash_map::Entry; use std::collections::HashMap; use std::ops::{Add, AddAssign}; use std::time::Duration; @@ -138,17 +139,28 @@ impl DocBatchV2Builder { /// Helper struct to build an [`IngestRequestV2`]. #[derive(Debug, Default)] pub struct IngestRequestV2Builder { - per_index_id_doc_batch_builders: HashMap, + per_index_id_doc_batch_builders: HashMap, + subrequest_id_sequence: u32, } impl IngestRequestV2Builder { /// Adds a document to the request. - pub fn add_doc(&mut self, index_id: IndexId, doc: &[u8]) { - let doc_batch_builder = self - .per_index_id_doc_batch_builders - .entry(index_id) - .or_default(); - doc_batch_builder.add_doc(doc); + pub fn add_doc(&mut self, index_id: IndexId, doc: &[u8]) -> u32 { + match self.per_index_id_doc_batch_builders.entry(index_id) { + Entry::Occupied(mut entry) => { + let (subrequest_id, doc_batch_builder) = entry.get_mut(); + doc_batch_builder.add_doc(doc); + *subrequest_id + } + Entry::Vacant(entry) => { + let subrequest_id = self.subrequest_id_sequence; + self.subrequest_id_sequence += 1; + let mut doc_batch_builder = DocBatchV2Builder::default(); + doc_batch_builder.add_doc(doc); + entry.insert((subrequest_id, doc_batch_builder)); + subrequest_id + } + } } /// Builds the [`IngestRequestV2`], returning `None` if the request is empty. @@ -156,11 +168,10 @@ impl IngestRequestV2Builder { let subrequests: Vec = self .per_index_id_doc_batch_builders .into_iter() - .enumerate() - .flat_map(|(subrequest_id, (index_id, doc_batch_builder))| { + .flat_map(|(index_id, (subrequest_id, doc_batch_builder))| { let doc_batch = doc_batch_builder.build()?; let ingest_subrequest = IngestSubrequest { - subrequest_id: subrequest_id as u32, + subrequest_id, index_id, source_id: source_id.to_string(), doc_batch: Some(doc_batch), diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/bulk.rs b/quickwit/quickwit-serve/src/elasticsearch_api/bulk.rs index ad9808d5a06..d0e1d24c51e 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/bulk.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/bulk.rs @@ -82,7 +82,7 @@ async fn elastic_ingest_bulk( mut ingest_service: IngestServiceClient, ingest_router: IngestRouterServiceClient, ) -> Result { - if enable_ingest_v2() { + if enable_ingest_v2() || bulk_options.enable_ingest_v2 { return elastic_bulk_ingest_v2(default_index_id, body, bulk_options, ingest_router).await; } let now = Instant::now(); @@ -94,12 +94,14 @@ async fn elastic_ingest_bulk( ElasticsearchError::new( StatusCode::BAD_REQUEST, format!("Malformed action/metadata line [#{line_number}]. Details: `{error}`"), + None, ) })?; let (_, source) = lines.next().ok_or_else(|| { ElasticsearchError::new( StatusCode::BAD_REQUEST, "expected source for the action".to_string(), + None, ) })?; // when ingesting on /my-index/_bulk, if _index: is set to something else than my-index, @@ -112,6 +114,7 @@ async fn elastic_ingest_bulk( ElasticsearchError::new( StatusCode::BAD_REQUEST, format!("missing required field: `_index` in the line [#{line_number}]."), + None, ) })?; let doc_batch_builder = doc_batch_builders @@ -136,6 +139,7 @@ async fn elastic_ingest_bulk( let bulk_response = ElasticBulkResponse { took_millis, errors, + items: Vec::new(), }; Ok(bulk_response) } diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/bulk_v2.rs b/quickwit/quickwit-serve/src/elasticsearch_api/bulk_v2.rs index 2e725999492..e7e48617059 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/bulk_v2.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/bulk_v2.rs @@ -17,6 +17,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +use std::collections::HashMap; use std::time::Instant; use hyper::StatusCode; @@ -30,6 +31,7 @@ use quickwit_proto::types::IndexId; use serde::{Deserialize, Serialize}; use tracing::warn; +use super::model::ErrorCauseException; use crate::elasticsearch_api::model::{BulkAction, ElasticBulkOptions, ElasticsearchError}; use crate::ingest_api::lines; use crate::Body; @@ -39,6 +41,35 @@ pub(crate) struct ElasticBulkResponse { #[serde(rename = "took")] pub took_millis: u64, pub errors: bool, + pub items: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) enum ElasticBulkItemAction { + #[serde(rename = "create")] + Create(ElasticBulkItem), + #[serde(rename = "index")] + Index(ElasticBulkItem), +} + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct ElasticBulkItem { + #[serde(rename = "_index")] + pub index_id: IndexId, + #[serde(rename = "_id")] + pub es_doc_id: Option, + #[serde(with = "http_serde::status_code")] + pub status: StatusCode, + pub error: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct ElasticBulkError { + #[serde(rename = "index")] + pub index_id: Option, + #[serde(rename = "type")] + pub exception: ErrorCauseException, + pub reason: String, } pub(crate) async fn elastic_bulk_ingest_v2( @@ -50,34 +81,44 @@ pub(crate) async fn elastic_bulk_ingest_v2( let now = Instant::now(); let mut ingest_request_builder = IngestRequestV2Builder::default(); let mut lines = lines(&body.content).enumerate(); + let mut per_subrequest_id_es_doc_ids: HashMap>> = HashMap::new(); while let Some((line_no, line)) = lines.next() { let action = serde_json::from_slice::(line).map_err(|error| { ElasticsearchError::new( StatusCode::BAD_REQUEST, - format!("unsupported or malformed action on line #{line_no}: `{error}`"), + format!("Malformed action/metadata line [{}]: {error}", line_no + 1), + Some(ErrorCauseException::IllegalArgument), ) })?; let (_, source) = lines.next().ok_or_else(|| { ElasticsearchError::new( StatusCode::BAD_REQUEST, - format!("associated source data with action on line #{line_no} is missing"), + "Validation Failed: 1: no requests added;".to_string(), + Some(ErrorCauseException::ActionRequestValidation), ) })?; + let meta = action.into_meta(); // When ingesting into `/my-index/_bulk`, if `_index` is set to something other than // `my-index`, ES honors it and creates the doc for the requested index. That is, // `my-index` is a default value in case `_index`` is missing, but not a constraint on // each sub-action. - let index_id = action - .into_index_id() + let index_id = meta + .index_id .or_else(|| default_index_id.clone()) .ok_or_else(|| { ElasticsearchError::new( StatusCode::BAD_REQUEST, - format!("`_index` field of action on line #{line_no} is missing"), + "Validation Failed: 1: index is missing;".to_string(), + Some(ErrorCauseException::ActionRequestValidation), ) })?; - ingest_request_builder.add_doc(index_id, source); + let subrequest_id = ingest_request_builder.add_doc(index_id, source); + + per_subrequest_id_es_doc_ids + .entry(subrequest_id) + .or_default() + .push(meta.es_doc_id); } let commit_type: CommitTypeV2 = bulk_options.refresh.into(); @@ -86,27 +127,56 @@ pub(crate) async fn elastic_bulk_ingest_v2( } let ingest_request_opt = ingest_request_builder.build(INGEST_V2_SOURCE_ID, commit_type); - if let Some(ingest_request) = ingest_request_opt { - let ingest_response_v2 = ingest_router.ingest(ingest_request).await?; - let took_millis = now.elapsed().as_millis() as u64; - let errors = !ingest_response_v2.failures.is_empty(); - - for failure in ingest_response_v2.failures { - // This custom logic for Airmail is temporary. - if failure.reason() == IngestFailureReason::IndexNotFound { - let reason = format!("index `{}` not found", failure.index_id); - let elasticsearch_error = ElasticsearchError::new(StatusCode::NOT_FOUND, reason); - return Err(elasticsearch_error); + let Some(ingest_request) = ingest_request_opt else { + return Ok(ElasticBulkResponse::default()); + }; + let ingest_response_v2 = ingest_router.ingest(ingest_request).await?; + let errors = !ingest_response_v2.failures.is_empty(); + let mut items = Vec::new(); + + for failure in ingest_response_v2.failures { + let es_doc_ids = per_subrequest_id_es_doc_ids + .remove(&failure.subrequest_id) + .ok_or_else(|| { + ElasticsearchError::new( + StatusCode::INTERNAL_SERVER_ERROR, + format!( + "could not find subrequest `{}` in bulk request", + failure.subrequest_id + ), + None, + ) + })?; + match failure.reason() { + IngestFailureReason::IndexNotFound => { + for es_doc_id in es_doc_ids { + let error = ElasticBulkError { + index_id: Some(failure.index_id.clone()), + exception: ErrorCauseException::IndexNotFound, + reason: format!("no such index [{}]", failure.index_id), + }; + let item = ElasticBulkItem { + index_id: failure.index_id.clone(), + es_doc_id, + status: StatusCode::NOT_FOUND, + error: Some(error), + }; + items.push(ElasticBulkItemAction::Index(item)); + } + } + _ => { + // TODO } } - let bulk_response = ElasticBulkResponse { - took_millis, - errors, - }; - Ok(bulk_response) - } else { - Ok(ElasticBulkResponse::default()) } + let took_millis = now.elapsed().as_millis() as u64; + + let bulk_response = ElasticBulkResponse { + took_millis, + errors, + items, + }; + Ok(bulk_response) } #[cfg(test)] @@ -149,16 +219,15 @@ mod tests { assert_eq!(ingest_request.commit_type(), CommitTypeV2::Auto); let mut subrequests = ingest_request.subrequests; - assert_eq!(subrequests[0].subrequest_id, 0); - assert_eq!(subrequests[1].subrequest_id, 1); - subrequests.sort_by(|left, right| left.index_id.cmp(&right.index_id)); + assert_eq!(subrequests[0].subrequest_id, 0); assert_eq!(subrequests[0].index_id, "my-index-1"); assert_eq!(subrequests[0].source_id, INGEST_V2_SOURCE_ID); assert_eq!(subrequests[0].doc_batch.as_ref().unwrap().num_docs(), 2); assert_eq!(subrequests[0].doc_batch.as_ref().unwrap().num_bytes(), 104); + assert_eq!(subrequests[1].subrequest_id, 1); assert_eq!(subrequests[1].index_id, "my-index-2"); assert_eq!(subrequests[1].source_id, INGEST_V2_SOURCE_ID); assert_eq!(subrequests[1].doc_batch.as_ref().unwrap().num_docs(), 1); @@ -296,7 +365,7 @@ mod tests { let reason = es_error.error.reason.unwrap(); assert_eq!( reason, - "unsupported or malformed action on line #0: `expected value at line 1 column 60`" + "Malformed action/metadata line [1]: expected value at line 1 column 60" ); let payload = r#" @@ -314,10 +383,7 @@ mod tests { assert_eq!(es_error.status, StatusCode::BAD_REQUEST); let reason = es_error.error.reason.unwrap(); - assert_eq!( - reason, - "associated source data with action on line #0 is missing" - ); + assert_eq!(reason, "Validation Failed: 1: no requests added;"); let payload = r#" {"create": {"_id" : "1"}} @@ -335,30 +401,60 @@ mod tests { assert_eq!(es_error.status, StatusCode::BAD_REQUEST); let reason = es_error.error.reason.unwrap(); - assert_eq!(reason, "`_index` field of action on line #0 is missing"); + assert_eq!(reason, "Validation Failed: 1: index is missing;"); } - // Airmail-specific test. It should go away when we straighten out the API response. #[tokio::test] - async fn test_bulk_api_returns_404_on_index_not_found() { + async fn test_bulk_api_index_not_found() { let mut mock_ingest_router = MockIngestRouterService::new(); - mock_ingest_router.expect_ingest().once().returning(|_| { - Ok(IngestResponseV2 { - successes: Vec::new(), - failures: vec![IngestFailure { - subrequest_id: 2, - index_id: "my-index".to_string(), - source_id: INGEST_V2_SOURCE_ID.to_string(), - reason: IngestFailureReason::IndexNotFound as i32, - }], - }) - }); + mock_ingest_router + .expect_ingest() + .once() + .returning(|ingest_request| { + assert_eq!(ingest_request.subrequests.len(), 2); + assert_eq!(ingest_request.commit_type(), CommitTypeV2::Auto); + + let mut subrequests = ingest_request.subrequests; + subrequests.sort_by(|left, right| left.index_id.cmp(&right.index_id)); + + assert_eq!(subrequests[0].subrequest_id, 0); + assert_eq!(subrequests[0].index_id, "my-index-1"); + assert_eq!(subrequests[0].source_id, INGEST_V2_SOURCE_ID); + assert_eq!(subrequests[0].doc_batch.as_ref().unwrap().num_docs(), 2); + + assert_eq!(subrequests[1].subrequest_id, 1); + assert_eq!(subrequests[1].index_id, "my-index-2"); + assert_eq!(subrequests[1].source_id, INGEST_V2_SOURCE_ID); + assert_eq!(subrequests[1].doc_batch.as_ref().unwrap().num_docs(), 1); + + Ok(IngestResponseV2 { + successes: Vec::new(), + failures: vec![ + IngestFailure { + subrequest_id: 0, + index_id: "my-index-1".to_string(), + source_id: INGEST_V2_SOURCE_ID.to_string(), + reason: IngestFailureReason::IndexNotFound as i32, + }, + IngestFailure { + subrequest_id: 1, + index_id: "my-index-2".to_string(), + source_id: INGEST_V2_SOURCE_ID.to_string(), + reason: IngestFailureReason::IndexNotFound as i32, + }, + ], + }) + }); let ingest_router = IngestRouterServiceClient::from_mock(mock_ingest_router); let handler = es_compat_bulk_handler_v2(ingest_router); let payload = r#" - {"create": {"_index": "my-index", "_id" : "1"}} - {"ts": 1, "message": "my-message"} + {"index": {"_index": "my-index-1", "_id" : "1"}} + {"ts": 1, "message": "my-message-1"} + {"index": {"_index": "my-index-1"}} + {"ts": 2, "message": "my-message-1"} + {"index": {"_index": "my-index-2", "_id" : "1"}} + {"ts": 3, "message": "my-message-2"} "#; let response = warp::test::request() .path("/_elastic/_bulk") @@ -366,12 +462,10 @@ mod tests { .body(payload) .reply(&handler) .await; - assert_eq!(response.status(), 404); - - let es_error: ElasticsearchError = serde_json::from_slice(response.body()).unwrap(); - assert_eq!(es_error.status, StatusCode::NOT_FOUND); + assert_eq!(response.status(), 200); - let reason = es_error.error.reason.unwrap(); - assert_eq!(reason, "index `my-index` not found"); + let bulk_response: ElasticBulkResponse = serde_json::from_slice(response.body()).unwrap(); + assert!(bulk_response.errors); + assert_eq!(bulk_response.items.len(), 3); } } diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/bulk_body.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/bulk_body.rs index 2c394aafbbe..cf033bfd03a 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/model/bulk_body.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/bulk_body.rs @@ -34,6 +34,13 @@ impl BulkAction { BulkAction::Create(meta) => meta.index_id, } } + + pub fn into_meta(self) -> BulkActionMeta { + match self { + BulkAction::Create(meta) => meta, + BulkAction::Index(meta) => meta, + } + } } #[derive(Clone, Debug, Deserialize, PartialEq)] @@ -43,7 +50,7 @@ pub struct BulkActionMeta { pub index_id: Option, #[serde(alias = "_id")] #[serde(default)] - pub doc_id: Option, + pub es_doc_id: Option, } #[cfg(test)] @@ -65,7 +72,7 @@ mod tests { bulk_action, BulkAction::Create(BulkActionMeta { index_id: Some("test".to_string()), - doc_id: Some("2".to_string()), + es_doc_id: Some("2".to_string()), }) ); } @@ -80,7 +87,7 @@ mod tests { bulk_action, BulkAction::Create(BulkActionMeta { index_id: Some("test".to_string()), - doc_id: None, + es_doc_id: None, }) ); } @@ -95,7 +102,7 @@ mod tests { bulk_action, BulkAction::Create(BulkActionMeta { index_id: None, - doc_id: Some("3".to_string()), + es_doc_id: Some("3".to_string()), }) ); } diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/bulk_query_params.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/bulk_query_params.rs index 8c343e20783..a1d982b4e25 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/model/bulk_query_params.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/bulk_query_params.rs @@ -25,6 +25,8 @@ use serde::Deserialize; pub struct ElasticBulkOptions { #[serde(default)] pub refresh: ElasticRefresh, + #[serde(default)] + pub enable_ingest_v2: bool, } /// ?refresh parameter for elasticsearch bulk request diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/cat_indices.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/cat_indices.rs index 9faf21921a7..7c73de44653 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/model/cat_indices.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/cat_indices.rs @@ -67,18 +67,21 @@ impl CatIndexQueryParams { "Format {:?} is not supported. Only format=json is supported.", format ), + None, )); } } else { return Err(ElasticsearchError::new( StatusCode::BAD_REQUEST, "Only format=json is supported.".to_string(), + None, )); } let unsupported_parameter_error = |field: &str| { ElasticsearchError::new( StatusCode::BAD_REQUEST, format!("Parameter {:?} is not supported.", field), + None, ) }; if self.bytes.is_some() { @@ -90,7 +93,6 @@ impl CatIndexQueryParams { if self.s.is_some() { return Err(unsupported_parameter_error("s")); } - Ok(()) } } diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/error.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/error.rs index 3b19b0c2dae..c648eefc89f 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/model/error.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/error.rs @@ -34,16 +34,20 @@ pub struct ElasticsearchError { } impl ElasticsearchError { - pub fn new(status: StatusCode, reason_string: String) -> Self { + pub fn new( + status: StatusCode, + reason: String, + exception_opt: Option, + ) -> Self { ElasticsearchError { status, error: ErrorCause { - reason: Some(reason_string), + reason: Some(reason), caused_by: None, root_cause: Vec::new(), stack_trace: None, suppressed: Vec::new(), - ty: None, + ty: exception_opt.map(|exception| exception.as_str().to_string()), additional_details: Default::default(), }, } @@ -129,3 +133,23 @@ impl From for ElasticsearchError { } } } + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub enum ErrorCauseException { + #[serde(rename = "action_request_validation_exception")] + ActionRequestValidation, + #[serde(rename = "illegal_argument_exception")] + IllegalArgument, + #[serde(rename = "index_not_found_exception")] + IndexNotFound, +} + +impl ErrorCauseException { + pub fn as_str(&self) -> &'static str { + match self { + Self::ActionRequestValidation => "action_request_validation_exception", + Self::IllegalArgument => "illegal_argument_exception", + Self::IndexNotFound => "index_not_found_exception", + } + } +} diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/model/mod.rs b/quickwit/quickwit-serve/src/elasticsearch_api/model/mod.rs index f4b9c1ff510..1ce635829e1 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/model/mod.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/model/mod.rs @@ -31,7 +31,7 @@ mod stats; pub use bulk_body::BulkAction; pub use bulk_query_params::ElasticBulkOptions; pub use cat_indices::{CatIndexQueryParams, ElasticsearchCatIndexResponse}; -pub use error::ElasticsearchError; +pub use error::{ElasticsearchError, ErrorCauseException}; pub use field_capability::{ build_list_field_request_for_es_api, convert_to_es_field_capabilities_response, FieldCapabilityQueryParams, FieldCapabilityRequestBody, FieldCapabilityResponse, diff --git a/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs b/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs index e26ce41aaa2..ed171896728 100644 --- a/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs +++ b/quickwit/quickwit-serve/src/elasticsearch_api/rest_handler.rs @@ -341,6 +341,7 @@ fn partial_hit_from_search_after_param( return Err(ElasticsearchError::new( StatusCode::BAD_REQUEST, "sort and search_after are of different length".to_string(), + None, )); } let mut parsed_search_after = PartialHit::default(); @@ -354,6 +355,7 @@ fn partial_hit_from_search_after_param( "invalid search_after doc id, must be of form \ `{split_id}:{segment_id: u32}:{doc_id: u32}`" .to_string(), + None, ) })?; parsed_search_after.split_id = address.split; @@ -364,6 +366,7 @@ fn partial_hit_from_search_after_param( return Err(ElasticsearchError::new( StatusCode::BAD_REQUEST, "search_after doc id must be of string type".to_string(), + None, )); } } else { @@ -371,6 +374,7 @@ fn partial_hit_from_search_after_param( ElasticsearchError::new( StatusCode::BAD_REQUEST, "invalid search_after field value, expect bool, number or string".to_string(), + None, ) })?; // TODO make cleaner once we support Vec @@ -531,6 +535,7 @@ async fn es_compat_index_cat_indices( ElasticsearchError::new( StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to serialize cat indices response: {}", serde_errror), + None, ) })?; diff --git a/quickwit/rest-api-tests/run_tests.py b/quickwit/rest-api-tests/run_tests.py index 5a8b7cbb95f..b2cd21aba1b 100755 --- a/quickwit/rest-api-tests/run_tests.py +++ b/quickwit/rest-api-tests/run_tests.py @@ -1,18 +1,20 @@ #!/usr/bin/env python3 -import subprocess -import requests import glob -import yaml -import sys -from os import path as osp -from os import mkdir import gzip import http import json -import tempfile +import os +import requests import shutil +import subprocess +import sys +import tempfile import time +import yaml + +from os import mkdir +from os import path as osp def debug_http(): old_send = http.client.HTTPConnection.send @@ -27,8 +29,6 @@ def new_send(self, data): return old_send(self, data) http.client.HTTPConnection.send = new_send -# debug_http() - def open_scenario(scenario_filepath): data = open(scenario_filepath).read() steps_data = data.split("\n---") @@ -85,7 +85,7 @@ def run_request_step(method, step, previous_result): step["headers"] = {'user-agent': 'my-app/0.0.1'} method_req = getattr(requests, method.lower()) endpoint = step.get("endpoint", "") - url = step["api_root"] + endpoint + url = "{}/{}".format(step["api_root"].rstrip('/'), endpoint.lstrip('/')) kvargs = { k: v for k, v in step.items() @@ -107,14 +107,14 @@ def run_request_step(method, step, previous_result): run_req = lambda : method_req(url, **kvargs) r = run_request_with_retry(run_req, expected_status_code, num_retries) expected_resp = step.get("expected", None) - json_res = r.json() + json_resp = r.json() if expected_resp is not None: try: - check_result(json_res, expected_resp, context_path="") + check_result(json_resp, expected_resp, context_path="") except Exception as e: - print(json.dumps(json_res, indent=2)) + print(json.dumps(json_resp, indent=2)) raise e - return json_res + return json_resp def check_result(result, expected, context_path = ""): if type(expected) == dict and "$expect" in expected: @@ -153,11 +153,12 @@ def check_result_list(result, expected, context_path=""): check_result(left, right, context_path + "[%s]" % i) def check_result_dict(result, expected, context_path=""): - for (k, v) in expected.items(): - child = result.get(k, None) - if child is None: - raise Exception("Missing key %s at context %s" % (k, context_path)) - check_result(child, v, context_path + "." + k) + for key, value in expected.items(): + try: + child = result[key] + except KeyError: + raise Exception("Missing key `%s` at context %s" % (key, context_path)) + check_result(child, value, context_path + "." + key) class PathTree: def __init__(self): diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0001-malformed-action.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0001-malformed-action.yaml new file mode 100644 index 00000000000..7dbbbefa257 --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0001-malformed-action.yaml @@ -0,0 +1,9 @@ +ndjson: + - del: { "_index": "test-index", "_id": "1" } +status_code: 400 +expected: + status: 400 + error: + type: illegal_argument_exception + reason: + $expect: val.startswith('Malformed action/metadata line [1]') diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0002-validation-failed-index-missing.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0002-validation-failed-index-missing.yaml new file mode 100644 index 00000000000..d4ebdfa7f8e --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0002-validation-failed-index-missing.yaml @@ -0,0 +1,9 @@ +ndjson: + - index: { "_id": "1" } + - message: Hello, World! +status_code: 400 +expected: + status: 400 + error: + type: action_request_validation_exception + reason: "Validation Failed: 1: index is missing;" diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0003-validation-failed-no-requests-added.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0003-validation-failed-no-requests-added.yaml new file mode 100644 index 00000000000..c233ff577af --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0003-validation-failed-no-requests-added.yaml @@ -0,0 +1,8 @@ +ndjson: + - index: { "_index": "test-index", "_id": "2" } +status_code: 400 +expected: + status: 400 + error: + type: action_request_validation_exception + reason: "Validation Failed: 1: no requests added;" diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_ctx.elasticsearch.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_ctx.elasticsearch.yaml new file mode 100644 index 00000000000..567175725b1 --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_ctx.elasticsearch.yaml @@ -0,0 +1 @@ +api_root: http://localhost:9200 diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_ctx.quickwit.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_ctx.quickwit.yaml new file mode 100644 index 00000000000..5129310b10f --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_ctx.quickwit.yaml @@ -0,0 +1,3 @@ +api_root: http://localhost:7280/api/v1/_elastic +params: + enable_ingest_v2: 'true' diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_ctx.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_ctx.yaml new file mode 100644 index 00000000000..d132261bb6e --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_ctx.yaml @@ -0,0 +1,2 @@ +method: [POST] +endpoint: "_bulk" diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/0001-index-not-found.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/0001-index-not-found.yaml new file mode 100644 index 00000000000..ea450d5de16 --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/0001-index-not-found.yaml @@ -0,0 +1,28 @@ +ndjson: + - index: { "_index": "test-index-not-found", "_id": "1" } + - message: Hello, World!" + - index: { "_index": "test-index-not-found" } + - message: Hola, Mundo! +status_code: 200 +expected: + errors: true + items: + - index: + _index: test-index-not-found + _id: "1" + status: 404 + error: + index: test-index-not-found + type: index_not_found_exception + reason: + $expect: val.startswith('no such index [test-index-not-found]') + - index: + _index: test-index-not-found + _id: null + status: 404 + error: + index: test-index-not-found + type: index_not_found_exception + reason: + $expect: val.startswith('no such index [test-index-not-found]') + diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.elasticsearch.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.elasticsearch.yaml new file mode 100644 index 00000000000..567175725b1 --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.elasticsearch.yaml @@ -0,0 +1 @@ +api_root: http://localhost:9200 diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.quickwit.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.quickwit.yaml new file mode 100644 index 00000000000..5129310b10f --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.quickwit.yaml @@ -0,0 +1,3 @@ +api_root: http://localhost:7280/api/v1/_elastic +params: + enable_ingest_v2: 'true' diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.yaml new file mode 100644 index 00000000000..d132261bb6e --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.yaml @@ -0,0 +1,2 @@ +method: [POST] +endpoint: "_bulk" diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_setup.elasticsearch.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_setup.elasticsearch.yaml new file mode 100644 index 00000000000..e0d0f4d0ca7 --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_setup.elasticsearch.yaml @@ -0,0 +1,6 @@ +# Disable automatic creation of indexes. +method: PUT +endpoint: _cluster/settings +json: + transient: + action.auto_create_index: "false" diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_teardown.elasticsearch.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_teardown.elasticsearch.yaml new file mode 100644 index 00000000000..3fb2a31099b --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_teardown.elasticsearch.yaml @@ -0,0 +1,6 @@ +# Enable automatic creation of indexes. +method: PUT +endpoint: _cluster/settings +json: + transient: + action.auto_create_index: "true"