diff --git a/quickwit/rest-api-tests/README.md b/quickwit/rest-api-tests/README.md
index dbd41d4d286..8d493df4d96 100644
--- a/quickwit/rest-api-tests/README.md
+++ b/quickwit/rest-api-tests/README.md
@@ -34,7 +34,7 @@ running in DEBUG mode is fine.
 ```
 
 When targeting elasticsearch, the script expects elastic to be running on
-`http://localhost:9200`.
+`http://localhost:9200` (see [compose script](./docker-compose.yaml)).
 
 In both cases, the test will take care of setting up, ingesting and tearing down the
 indexes involved.
diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/0001-index-not-found.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0006-partial-index-not-found.yaml
similarity index 55%
rename from quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/0001-index-not-found.yaml
rename to quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0006-partial-index-not-found.yaml
index ea450d5de16..b4ecf6f678b 100644
--- a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/0001-index-not-found.yaml
+++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/0006-partial-index-not-found.yaml
@@ -1,7 +1,9 @@
 ndjson:
-  - index: { "_index": "test-index-not-found", "_id": "1" }
-  - message: Hello, World!"
   - index: { "_index": "test-index-not-found" }
+  - message: Hello, World!"
+  - index: { "_index": "test-index" }
+  - message: Hola, Mundo!
+  - index: { "_index": "test-index-pattern-777" }
   - message: Hola, Mundo!
 status_code: 200
 expected:
@@ -9,7 +11,6 @@ expected:
   items:
     - index:
         _index: test-index-not-found
-        _id: "1"
         status: 404
         error:
           index: test-index-not-found
@@ -17,12 +18,8 @@ expected:
           reason:
             $expect: val.startswith('no such index [test-index-not-found]')
     - index:
-        _index: test-index-not-found
-        _id: null
-        status: 404
-        error:
-          index: test-index-not-found
-          type: index_not_found_exception
-          reason:
-            $expect: val.startswith('no such index [test-index-not-found]')
-
+        _index: test-index
+        status: 201
+    - index:
+        _index: test-index-pattern-777
+        status: 201
diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.elasticsearch.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.elasticsearch.yaml
index 1cc65fa967d..3911140cb97 100644
--- a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.elasticsearch.yaml
+++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.elasticsearch.yaml
@@ -3,6 +3,10 @@ method: DELETE
 endpoint: test-index
 status_code: null
 ---
+method: DELETE
+endpoint: test-index-pattern-777
+status_code: null
+---
 method: PUT
 endpoint: test-index
 json: {
@@ -19,3 +23,10 @@ json: {
     }
   }
 }
+---
+# Only create indexes automatically for specific pattern
+method: PUT
+endpoint: _cluster/settings
+json:
+  transient:
+    action.auto_create_index: "test-index-pattern-*"
diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.quickwit.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.quickwit.yaml
index b1a48066ca5..82febfc2517 100644
--- a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.quickwit.yaml
+++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_setup.quickwit.yaml
@@ -6,6 +6,11 @@ status_code: null
 ---
 method: DELETE
 api_root: http://localhost:7280/api/v1/
+endpoint: indexes/test-index-pattern-777
+status_code: null
+---
+method: DELETE
+api_root: http://localhost:7280/api/v1/
 endpoint: templates/test-index-template
 status_code: null
 ---
@@ -31,7 +36,7 @@ json:
   version: "0.8"
   template_id: test-index-template
   index_id_patterns:
-    - test-index-happy-path*
+    - test-index-pattern-*
   doc_mapping:
     mode: dynamic
   indexing_settings:
diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_teardown.elasticsearch.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_teardown.elasticsearch.yaml
index e69de29bb2d..4249b11275e 100644
--- a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_teardown.elasticsearch.yaml
+++ b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/_teardown.elasticsearch.yaml
@@ -0,0 +1,10 @@
+# Reconfigure with the default settings
+method: PUT
+endpoint: _cluster/settings
+json:
+  transient:
+    action.auto_create_index: "true"
+---
+method: DELETE
+endpoint: test-index*
+status_code: null
diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.elasticsearch.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.elasticsearch.yaml
deleted file mode 100644
index 567175725b1..00000000000
--- a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.elasticsearch.yaml
+++ /dev/null
@@ -1 +0,0 @@
-api_root: http://localhost:9200
diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.quickwit.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.quickwit.yaml
deleted file mode 100644
index 5129310b10f..00000000000
--- a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.quickwit.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
-api_root: http://localhost:7280/api/v1/_elastic
-params:
-  enable_ingest_v2: 'true'
diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.yaml
deleted file mode 100644
index d132261bb6e..00000000000
--- a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_ctx.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
-method: [POST]
-endpoint: "_bulk"
diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_setup.elasticsearch.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_setup.elasticsearch.yaml
deleted file mode 100644
index e0d0f4d0ca7..00000000000
--- a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_setup.elasticsearch.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-# Disable automatic creation of indexes.
-method: PUT
-endpoint: _cluster/settings
-json:
-  transient:
-    action.auto_create_index: "false"
diff --git a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_teardown.elasticsearch.yaml b/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_teardown.elasticsearch.yaml
deleted file mode 100644
index 3fb2a31099b..00000000000
--- a/quickwit/rest-api-tests/scenarii/es_compatibility/bulk/index_not_found/_teardown.elasticsearch.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-# Enable automatic creation of indexes.
-method: PUT
-endpoint: _cluster/settings
-json:
-  transient:
-    action.auto_create_index: "true"