Address code review comments

Signed-off-by: Andriy Redko <[email protected]>
opensearch-project · Sep 27, 2024 · 60ec1ce · 60ec1ce
1 parent ae4dae1
commit 60ec1ce
Show file tree

Hide file tree

Showing 6 changed files with 190 additions and 65 deletions.
diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml
@@ -37,8 +37,9 @@ jobs:
           - version: 2.16.0
             tests: snapshot
           - version: 2.17.0
+          - version: 2.18.0
             hub: opensearchstaging
-            ref: '@sha256:1273489ebbedcb470ea13563dae4c6dc6b2ed431e87e686ed931ae0733034b25'
+            ref: '@sha256:2f215e1bed35f49d5e0bb9408824767e7e18dd50a9e85b4f414bf39fa573a37a'
           - version: 3.0.0
             hub: opensearchstaging
             ref: '@sha256:06af2ba4037f8423dc1a4ed3cd29108a1912774e7c659e73f0fac09e1bb2b63d'

diff --git a/.github/workflows/test-tools-integ.yml b/.github/workflows/test-tools-integ.yml
@@ -24,7 +24,7 @@ jobs:
   test:
     runs-on: ubuntu-latest
     env:
-      OPENSEARCH_VERSION: 2.16.0
+      OPENSEARCH_VERSION: 2.17.0
       OPENSEARCH_PASSWORD: myStrongPassword123!
       OPENSEARCH_URL: https://localhost:9200
     steps:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -95,6 +95,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added `/_plugins/_flow_framework/`, `_search`, `state/_search`, `_provision`, `_deprovision`, `_steps`, and `_status` ([#508](https://github.com/opensearch-project/opensearch-api-specification/issues/508)) ([#833](https://github.com/opensearch-project/flow-framework/issues/833))
 - Added `/_plugins/_ism/policies`, `add`, `remove`, `change_policy`, `explain`, and `retry` ([#568](https://github.com/opensearch-project/opensearch-api-specification/pull/568)) ([#578](https://github.com/opensearch-project/opensearch-api-specification/pull/578))
 - Added `/_plugins/refresh_search_analyzers` ([#578](https://github.com/opensearch-project/opensearch-api-specification/pull/578))
+- Added `/_bulk/stream` ([#584](https://github.com/opensearch-project/opensearch-api-specification/pull/584))
 
 ### Changed
 

diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml
@@ -85,23 +85,23 @@ paths:
       externalDocs:
         url: https://opensearch.org/docs/latest/api-reference/document-apis/bulk-streaming/
       parameters:
-        - $ref: '#/components/parameters/bulk::query._source'
-        - $ref: '#/components/parameters/bulk::query._source_excludes'
-        - $ref: '#/components/parameters/bulk::query._source_includes'
-        - $ref: '#/components/parameters/bulk::query.pipeline'
-        - $ref: '#/components/parameters/bulk::query.refresh'
-        - $ref: '#/components/parameters/bulk::query.require_alias'
-        - $ref: '#/components/parameters/bulk::query.routing'
-        - $ref: '#/components/parameters/bulk::query.timeout'
-        - $ref: '#/components/parameters/bulk::query.type'
-        - $ref: '#/components/parameters/bulk::query.wait_for_active_shards'
-        - $ref: '#/components/parameters/bulk::query.batch_size'
-        - $ref: '#/components/parameters/bulk::query.batch_interval'
+        - $ref: '#/components/parameters/bulk_stream::query._source'
+        - $ref: '#/components/parameters/bulk_stream::query._source_excludes'
+        - $ref: '#/components/parameters/bulk_stream::query._source_includes'
+        - $ref: '#/components/parameters/bulk_stream::query.batch_interval'
+        - $ref: '#/components/parameters/bulk_stream::query.batch_size'
+        - $ref: '#/components/parameters/bulk_stream::query.pipeline'
+        - $ref: '#/components/parameters/bulk_stream::query.refresh'
+        - $ref: '#/components/parameters/bulk_stream::query.require_alias'
+        - $ref: '#/components/parameters/bulk_stream::query.routing'
+        - $ref: '#/components/parameters/bulk_stream::query.timeout'
+        - $ref: '#/components/parameters/bulk_stream::query.type'
+        - $ref: '#/components/parameters/bulk_stream::query.wait_for_active_shards'
       requestBody:
-        $ref: '#/components/requestBodies/bulk'
+        $ref: '#/components/requestBodies/bulk_stream'
       responses:
         '200':
-          $ref: '#/components/responses/bulk@200'
+          $ref: '#/components/responses/bulk_stream@200'
     put:
       operationId: bulk_stream.1
       x-operation-group: bulk_stream
@@ -110,23 +110,23 @@ paths:
       externalDocs:
         url: https://opensearch.org/docs/latest/api-reference/document-apis/bulk-streaming/
       parameters:
-        - $ref: '#/components/parameters/bulk::query._source'
-        - $ref: '#/components/parameters/bulk::query._source_excludes'
-        - $ref: '#/components/parameters/bulk::query._source_includes'
-        - $ref: '#/components/parameters/bulk::query.pipeline'
-        - $ref: '#/components/parameters/bulk::query.refresh'
-        - $ref: '#/components/parameters/bulk::query.require_alias'
-        - $ref: '#/components/parameters/bulk::query.routing'
-        - $ref: '#/components/parameters/bulk::query.timeout'
-        - $ref: '#/components/parameters/bulk::query.type'
-        - $ref: '#/components/parameters/bulk::query.wait_for_active_shards'
-        - $ref: '#/components/parameters/bulk::query.batch_size'
-        - $ref: '#/components/parameters/bulk::query.batch_interval'
+        - $ref: '#/components/parameters/bulk_stream::query._source'
+        - $ref: '#/components/parameters/bulk_stream::query._source_excludes'
+        - $ref: '#/components/parameters/bulk_stream::query._source_includes'
+        - $ref: '#/components/parameters/bulk_stream::query.batch_interval'
+        - $ref: '#/components/parameters/bulk_stream::query.batch_size'
+        - $ref: '#/components/parameters/bulk_stream::query.pipeline'
+        - $ref: '#/components/parameters/bulk_stream::query.refresh'
+        - $ref: '#/components/parameters/bulk_stream::query.require_alias'
+        - $ref: '#/components/parameters/bulk_stream::query.routing'
+        - $ref: '#/components/parameters/bulk_stream::query.timeout'
+        - $ref: '#/components/parameters/bulk_stream::query.type'
+        - $ref: '#/components/parameters/bulk_stream::query.wait_for_active_shards'
       requestBody:
-        $ref: '#/components/requestBodies/bulk'
+        $ref: '#/components/requestBodies/bulk_stream'
       responses:
         '200':
-          $ref: '#/components/responses/bulk@200'
+          $ref: '#/components/responses/bulk_stream@200'
   /_count:
     get:
       operationId: count.0
@@ -1110,52 +1110,52 @@ paths:
       x-version-added: '2.17'
       description: Allows to perform multiple index/update/delete operations using request response streaming.
       externalDocs:
-        url: https://opensearch.org/docs/latest/api-reference/document-apis/bulk/
+        url: https://opensearch.org/docs/latest/api-reference/document-apis/bulk-streaming/
       parameters:
-        - $ref: '#/components/parameters/bulk::path.index'
-        - $ref: '#/components/parameters/bulk::query._source'
-        - $ref: '#/components/parameters/bulk::query._source_excludes'
-        - $ref: '#/components/parameters/bulk::query._source_includes'
-        - $ref: '#/components/parameters/bulk::query.pipeline'
-        - $ref: '#/components/parameters/bulk::query.refresh'
-        - $ref: '#/components/parameters/bulk::query.require_alias'
-        - $ref: '#/components/parameters/bulk::query.routing'
-        - $ref: '#/components/parameters/bulk::query.timeout'
-        - $ref: '#/components/parameters/bulk::query.type'
-        - $ref: '#/components/parameters/bulk::query.wait_for_active_shards'
-        - $ref: '#/components/parameters/bulk::query.batch_size'
-        - $ref: '#/components/parameters/bulk::query.batch_interval'
+        - $ref: '#/components/parameters/bulk_stream::path.index'
+        - $ref: '#/components/parameters/bulk_stream::query._source'
+        - $ref: '#/components/parameters/bulk_stream::query._source_excludes'
+        - $ref: '#/components/parameters/bulk_stream::query._source_includes'
+        - $ref: '#/components/parameters/bulk_stream::query.batch_interval'
+        - $ref: '#/components/parameters/bulk_stream::query.batch_size'
+        - $ref: '#/components/parameters/bulk_stream::query.pipeline'
+        - $ref: '#/components/parameters/bulk_stream::query.refresh'
+        - $ref: '#/components/parameters/bulk_stream::query.require_alias'
+        - $ref: '#/components/parameters/bulk_stream::query.routing'
+        - $ref: '#/components/parameters/bulk_stream::query.timeout'
+        - $ref: '#/components/parameters/bulk_stream::query.type'
+        - $ref: '#/components/parameters/bulk_stream::query.wait_for_active_shards'
       requestBody:
-        $ref: '#/components/requestBodies/bulk'
+        $ref: '#/components/requestBodies/bulk_stream'
       responses:
         '200':
-          $ref: '#/components/responses/bulk@200'
+          $ref: '#/components/responses/bulk_stream@200'
     put:
       operationId: bulk_stream.3
       x-operation-group: bulk_stream
       x-version-added: '2.17'
       description: Allows to perform multiple index/update/delete operations using request response streaming.
       externalDocs:
-        url: https://opensearch.org/docs/latest/api-reference/document-apis/bulk/
+        url: https://opensearch.org/docs/latest/api-reference/document-apis/bulk-streaming/
       parameters:
-        - $ref: '#/components/parameters/bulk::path.index'
-        - $ref: '#/components/parameters/bulk::query._source'
-        - $ref: '#/components/parameters/bulk::query._source_excludes'
-        - $ref: '#/components/parameters/bulk::query._source_includes'
-        - $ref: '#/components/parameters/bulk::query.pipeline'
-        - $ref: '#/components/parameters/bulk::query.refresh'
-        - $ref: '#/components/parameters/bulk::query.require_alias'
-        - $ref: '#/components/parameters/bulk::query.routing'
-        - $ref: '#/components/parameters/bulk::query.timeout'
-        - $ref: '#/components/parameters/bulk::query.type'
-        - $ref: '#/components/parameters/bulk::query.wait_for_active_shards'
-        - $ref: '#/components/parameters/bulk::query.batch_size'
-        - $ref: '#/components/parameters/bulk::query.batch_interval'
+        - $ref: '#/components/parameters/bulk_stream::path.index'
+        - $ref: '#/components/parameters/bulk_stream::query._source'
+        - $ref: '#/components/parameters/bulk_stream::query._source_excludes'
+        - $ref: '#/components/parameters/bulk_stream::query._source_includes'
+        - $ref: '#/components/parameters/bulk_stream::query.batch_interval'
+        - $ref: '#/components/parameters/bulk_stream::query.batch_size'
+        - $ref: '#/components/parameters/bulk_stream::query.pipeline'
+        - $ref: '#/components/parameters/bulk_stream::query.refresh'
+        - $ref: '#/components/parameters/bulk_stream::query.require_alias'
+        - $ref: '#/components/parameters/bulk_stream::query.routing'
+        - $ref: '#/components/parameters/bulk_stream::query.timeout'
+        - $ref: '#/components/parameters/bulk_stream::query.type'
+        - $ref: '#/components/parameters/bulk_stream::query.wait_for_active_shards'
       requestBody:
-        $ref: '#/components/requestBodies/bulk'
+        $ref: '#/components/requestBodies/bulk_stream'
       responses:
         '200':
-          $ref: '#/components/responses/bulk@200'
+          $ref: '#/components/responses/bulk_stream@200'
   /{index}/_count:
     get:
       operationId: count.2
@@ -2256,6 +2256,18 @@ components:
                 - type: object
             description: The operation definition and data (action-data pairs), separated by newlines
       required: true
+    bulk_stream:
+      content:
+        application/x-ndjson:
+          schema:
+            type: array
+            items:
+              anyOf:
+                - $ref: '../schemas/_core.bulk.yaml#/components/schemas/OperationContainer'
+                - $ref: '../schemas/_core.bulk.yaml#/components/schemas/UpdateAction'
+                - type: object
+            description: The operation definition and data (action-data pairs), separated by newlines
+      required: true
     clear_scroll:
       content:
         application/json:
@@ -2751,6 +2763,30 @@ components:
               - errors
               - items
               - took
+    bulk_stream@200:
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              errors:
+                type: boolean
+              items:
+                type: array
+                items:
+                  type: object
+                  additionalProperties:
+                    $ref: '../schemas/_core.bulk.yaml#/components/schemas/ResponseItem'
+                  minProperties: 1
+                  maxProperties: 1
+              took:
+                type: number
+              ingest_took:
+                type: number
+            required:
+              - errors
+              - items
+              - took
     clear_scroll@200:
       content:
         application/json:
@@ -3321,7 +3357,94 @@ components:
         $ref: '../schemas/_common.yaml#/components/schemas/WaitForActiveShards'
       style: form
       x-default: '1'
-    bulk::query.batch_size:
+    bulk_stream::path.index:
+      in: path
+      name: index
+      description: Name of the data stream, index, or index alias to perform bulk actions on.
+      required: true
+      schema:
+        $ref: '../schemas/_common.yaml#/components/schemas/IndexName'
+      style: simple
+    bulk_stream::query._source:
+      in: query
+      name: _source
+      description: '`true` or `false` to return the `_source` field or not, or a list of fields to return.'
+      schema:
+        $ref: '../schemas/_core.search.yaml#/components/schemas/SourceConfigParam'
+      style: form
+    bulk_stream::query._source_excludes:
+      in: query
+      name: _source_excludes
+      description: A comma-separated list of source fields to exclude from the response.
+      schema:
+        $ref: '../schemas/_common.yaml#/components/schemas/Fields'
+      style: form
+    bulk_stream::query._source_includes:
+      in: query
+      name: _source_includes
+      description: A comma-separated list of source fields to include in the response.
+      schema:
+        $ref: '../schemas/_common.yaml#/components/schemas/Fields'
+      style: form
+    bulk_stream::query.pipeline:
+      in: query
+      name: pipeline
+      description: |-
+        ID of the pipeline to use to preprocess incoming documents.
+        If the index has a default ingest pipeline specified, then setting the value to `_none` disables the default ingest pipeline for this request.
+        If a final pipeline is configured it will always run, regardless of the value of this parameter.
+      schema:
+        type: string
+      style: form
+    bulk_stream::query.refresh:
+      in: query
+      name: refresh
+      description: |-
+        If `true`, OpenSearch refreshes the affected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` do nothing with refreshes.
+        Valid values: `true`, `false`, `wait_for`.
+      schema:
+        $ref: '../schemas/_common.yaml#/components/schemas/Refresh'
+      style: form
+    bulk_stream::query.require_alias:
+      in: query
+      name: require_alias
+      description: If `true`, the request's actions must target an index alias.
+      schema:
+        type: boolean
+        default: false
+      style: form
+    bulk_stream::query.routing:
+      in: query
+      name: routing
+      description: Custom value used to route operations to a specific shard.
+      schema:
+        $ref: '../schemas/_common.yaml#/components/schemas/Routing'
+      style: form
+    bulk_stream::query.timeout:
+      in: query
+      name: timeout
+      description: 'Period each action waits for the following operations: automatic index creation, dynamic mapping updates, waiting for active shards.'
+      schema:
+        $ref: '../schemas/_common.yaml#/components/schemas/Duration'
+      style: form
+    bulk_stream::query.type:
+      name: type
+      in: query
+      description: Default document type for items which don't provide one.
+      schema:
+        type: string
+        description: Default document type for items which don't provide one.
+    bulk_stream::query.wait_for_active_shards:
+      in: query
+      name: wait_for_active_shards
+      description: |-
+        The number of shard copies that must be active before proceeding with the operation.
+        Set to all or any positive integer up to the total number of shards in the index (`number_of_replicas+1`).
+      schema:
+        $ref: '../schemas/_common.yaml#/components/schemas/WaitForActiveShards'
+      style: form
+      x-default: '1'
+    bulk_stream::query.batch_size:
       in: query
       name: batch_size
       description: |-
@@ -3330,7 +3453,7 @@ components:
         $ref: '../schemas/_common.yaml#/components/schemas/BatchSize'
       style: form
       x-default: 1
-    bulk::query.batch_interval:
+    bulk_stream::query.batch_interval:
       in: query
       name: batch_interval
       description: |-

diff --git a/tests/default/_core/bulk_stream.yaml → tests/default/_core/bulk/stream.yaml b/tests/default/_core/bulk_stream.yaml → tests/default/_core/bulk/stream.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test bulk streaming endpoint.
 epilogues:

diff --git a/tests/default/indices/bulk_stream.yaml → tests/default/indices/bulk/stream.yaml b/tests/default/indices/bulk_stream.yaml → tests/default/indices/bulk/stream.yaml