From 60ec1ce331575f67f9c1991e587fd155a8dd56f4 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 27 Sep 2024 11:15:00 -0400 Subject: [PATCH] Address code review comments Signed-off-by: Andriy Redko --- .github/workflows/test-spec.yml | 3 +- .github/workflows/test-tools-integ.yml | 2 +- CHANGELOG.md | 1 + spec/namespaces/_core.yaml | 247 +++++++++++++----- .../{bulk_stream.yaml => bulk/stream.yaml} | 2 +- .../{bulk_stream.yaml => bulk/stream.yaml} | 0 6 files changed, 190 insertions(+), 65 deletions(-) rename tests/default/_core/{bulk_stream.yaml => bulk/stream.yaml} (96%) rename tests/default/indices/{bulk_stream.yaml => bulk/stream.yaml} (100%) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index a202207d..72fa6ae0 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -37,8 +37,9 @@ jobs: - version: 2.16.0 tests: snapshot - version: 2.17.0 + - version: 2.18.0 hub: opensearchstaging - ref: '@sha256:1273489ebbedcb470ea13563dae4c6dc6b2ed431e87e686ed931ae0733034b25' + ref: '@sha256:2f215e1bed35f49d5e0bb9408824767e7e18dd50a9e85b4f414bf39fa573a37a' - version: 3.0.0 hub: opensearchstaging ref: '@sha256:06af2ba4037f8423dc1a4ed3cd29108a1912774e7c659e73f0fac09e1bb2b63d' diff --git a/.github/workflows/test-tools-integ.yml b/.github/workflows/test-tools-integ.yml index 5bb9ac5d..d4384f4a 100644 --- a/.github/workflows/test-tools-integ.yml +++ b/.github/workflows/test-tools-integ.yml @@ -24,7 +24,7 @@ jobs: test: runs-on: ubuntu-latest env: - OPENSEARCH_VERSION: 2.16.0 + OPENSEARCH_VERSION: 2.17.0 OPENSEARCH_PASSWORD: myStrongPassword123! OPENSEARCH_URL: https://localhost:9200 steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index 0129e440..9c1317eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,6 +95,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `/_plugins/_flow_framework/`, `_search`, `state/_search`, `_provision`, `_deprovision`, `_steps`, and `_status` ([#508](https://github.com/opensearch-project/opensearch-api-specification/issues/508)) ([#833](https://github.com/opensearch-project/flow-framework/issues/833)) - Added `/_plugins/_ism/policies`, `add`, `remove`, `change_policy`, `explain`, and `retry` ([#568](https://github.com/opensearch-project/opensearch-api-specification/pull/568)) ([#578](https://github.com/opensearch-project/opensearch-api-specification/pull/578)) - Added `/_plugins/refresh_search_analyzers` ([#578](https://github.com/opensearch-project/opensearch-api-specification/pull/578)) +- Added `/_bulk/stream` ([#584](https://github.com/opensearch-project/opensearch-api-specification/pull/584)) ### Changed diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index 36c7535a..d358a08f 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -85,23 +85,23 @@ paths: externalDocs: url: https://opensearch.org/docs/latest/api-reference/document-apis/bulk-streaming/ parameters: - - $ref: '#/components/parameters/bulk::query._source' - - $ref: '#/components/parameters/bulk::query._source_excludes' - - $ref: '#/components/parameters/bulk::query._source_includes' - - $ref: '#/components/parameters/bulk::query.pipeline' - - $ref: '#/components/parameters/bulk::query.refresh' - - $ref: '#/components/parameters/bulk::query.require_alias' - - $ref: '#/components/parameters/bulk::query.routing' - - $ref: '#/components/parameters/bulk::query.timeout' - - $ref: '#/components/parameters/bulk::query.type' - - $ref: '#/components/parameters/bulk::query.wait_for_active_shards' - - $ref: '#/components/parameters/bulk::query.batch_size' - - $ref: '#/components/parameters/bulk::query.batch_interval' + - $ref: '#/components/parameters/bulk_stream::query._source' + - $ref: '#/components/parameters/bulk_stream::query._source_excludes' + - $ref: '#/components/parameters/bulk_stream::query._source_includes' + - $ref: '#/components/parameters/bulk_stream::query.batch_interval' + - $ref: '#/components/parameters/bulk_stream::query.batch_size' + - $ref: '#/components/parameters/bulk_stream::query.pipeline' + - $ref: '#/components/parameters/bulk_stream::query.refresh' + - $ref: '#/components/parameters/bulk_stream::query.require_alias' + - $ref: '#/components/parameters/bulk_stream::query.routing' + - $ref: '#/components/parameters/bulk_stream::query.timeout' + - $ref: '#/components/parameters/bulk_stream::query.type' + - $ref: '#/components/parameters/bulk_stream::query.wait_for_active_shards' requestBody: - $ref: '#/components/requestBodies/bulk' + $ref: '#/components/requestBodies/bulk_stream' responses: '200': - $ref: '#/components/responses/bulk@200' + $ref: '#/components/responses/bulk_stream@200' put: operationId: bulk_stream.1 x-operation-group: bulk_stream @@ -110,23 +110,23 @@ paths: externalDocs: url: https://opensearch.org/docs/latest/api-reference/document-apis/bulk-streaming/ parameters: - - $ref: '#/components/parameters/bulk::query._source' - - $ref: '#/components/parameters/bulk::query._source_excludes' - - $ref: '#/components/parameters/bulk::query._source_includes' - - $ref: '#/components/parameters/bulk::query.pipeline' - - $ref: '#/components/parameters/bulk::query.refresh' - - $ref: '#/components/parameters/bulk::query.require_alias' - - $ref: '#/components/parameters/bulk::query.routing' - - $ref: '#/components/parameters/bulk::query.timeout' - - $ref: '#/components/parameters/bulk::query.type' - - $ref: '#/components/parameters/bulk::query.wait_for_active_shards' - - $ref: '#/components/parameters/bulk::query.batch_size' - - $ref: '#/components/parameters/bulk::query.batch_interval' + - $ref: '#/components/parameters/bulk_stream::query._source' + - $ref: '#/components/parameters/bulk_stream::query._source_excludes' + - $ref: '#/components/parameters/bulk_stream::query._source_includes' + - $ref: '#/components/parameters/bulk_stream::query.batch_interval' + - $ref: '#/components/parameters/bulk_stream::query.batch_size' + - $ref: '#/components/parameters/bulk_stream::query.pipeline' + - $ref: '#/components/parameters/bulk_stream::query.refresh' + - $ref: '#/components/parameters/bulk_stream::query.require_alias' + - $ref: '#/components/parameters/bulk_stream::query.routing' + - $ref: '#/components/parameters/bulk_stream::query.timeout' + - $ref: '#/components/parameters/bulk_stream::query.type' + - $ref: '#/components/parameters/bulk_stream::query.wait_for_active_shards' requestBody: - $ref: '#/components/requestBodies/bulk' + $ref: '#/components/requestBodies/bulk_stream' responses: '200': - $ref: '#/components/responses/bulk@200' + $ref: '#/components/responses/bulk_stream@200' /_count: get: operationId: count.0 @@ -1110,52 +1110,52 @@ paths: x-version-added: '2.17' description: Allows to perform multiple index/update/delete operations using request response streaming. externalDocs: - url: https://opensearch.org/docs/latest/api-reference/document-apis/bulk/ + url: https://opensearch.org/docs/latest/api-reference/document-apis/bulk-streaming/ parameters: - - $ref: '#/components/parameters/bulk::path.index' - - $ref: '#/components/parameters/bulk::query._source' - - $ref: '#/components/parameters/bulk::query._source_excludes' - - $ref: '#/components/parameters/bulk::query._source_includes' - - $ref: '#/components/parameters/bulk::query.pipeline' - - $ref: '#/components/parameters/bulk::query.refresh' - - $ref: '#/components/parameters/bulk::query.require_alias' - - $ref: '#/components/parameters/bulk::query.routing' - - $ref: '#/components/parameters/bulk::query.timeout' - - $ref: '#/components/parameters/bulk::query.type' - - $ref: '#/components/parameters/bulk::query.wait_for_active_shards' - - $ref: '#/components/parameters/bulk::query.batch_size' - - $ref: '#/components/parameters/bulk::query.batch_interval' + - $ref: '#/components/parameters/bulk_stream::path.index' + - $ref: '#/components/parameters/bulk_stream::query._source' + - $ref: '#/components/parameters/bulk_stream::query._source_excludes' + - $ref: '#/components/parameters/bulk_stream::query._source_includes' + - $ref: '#/components/parameters/bulk_stream::query.batch_interval' + - $ref: '#/components/parameters/bulk_stream::query.batch_size' + - $ref: '#/components/parameters/bulk_stream::query.pipeline' + - $ref: '#/components/parameters/bulk_stream::query.refresh' + - $ref: '#/components/parameters/bulk_stream::query.require_alias' + - $ref: '#/components/parameters/bulk_stream::query.routing' + - $ref: '#/components/parameters/bulk_stream::query.timeout' + - $ref: '#/components/parameters/bulk_stream::query.type' + - $ref: '#/components/parameters/bulk_stream::query.wait_for_active_shards' requestBody: - $ref: '#/components/requestBodies/bulk' + $ref: '#/components/requestBodies/bulk_stream' responses: '200': - $ref: '#/components/responses/bulk@200' + $ref: '#/components/responses/bulk_stream@200' put: operationId: bulk_stream.3 x-operation-group: bulk_stream x-version-added: '2.17' description: Allows to perform multiple index/update/delete operations using request response streaming. externalDocs: - url: https://opensearch.org/docs/latest/api-reference/document-apis/bulk/ + url: https://opensearch.org/docs/latest/api-reference/document-apis/bulk-streaming/ parameters: - - $ref: '#/components/parameters/bulk::path.index' - - $ref: '#/components/parameters/bulk::query._source' - - $ref: '#/components/parameters/bulk::query._source_excludes' - - $ref: '#/components/parameters/bulk::query._source_includes' - - $ref: '#/components/parameters/bulk::query.pipeline' - - $ref: '#/components/parameters/bulk::query.refresh' - - $ref: '#/components/parameters/bulk::query.require_alias' - - $ref: '#/components/parameters/bulk::query.routing' - - $ref: '#/components/parameters/bulk::query.timeout' - - $ref: '#/components/parameters/bulk::query.type' - - $ref: '#/components/parameters/bulk::query.wait_for_active_shards' - - $ref: '#/components/parameters/bulk::query.batch_size' - - $ref: '#/components/parameters/bulk::query.batch_interval' + - $ref: '#/components/parameters/bulk_stream::path.index' + - $ref: '#/components/parameters/bulk_stream::query._source' + - $ref: '#/components/parameters/bulk_stream::query._source_excludes' + - $ref: '#/components/parameters/bulk_stream::query._source_includes' + - $ref: '#/components/parameters/bulk_stream::query.batch_interval' + - $ref: '#/components/parameters/bulk_stream::query.batch_size' + - $ref: '#/components/parameters/bulk_stream::query.pipeline' + - $ref: '#/components/parameters/bulk_stream::query.refresh' + - $ref: '#/components/parameters/bulk_stream::query.require_alias' + - $ref: '#/components/parameters/bulk_stream::query.routing' + - $ref: '#/components/parameters/bulk_stream::query.timeout' + - $ref: '#/components/parameters/bulk_stream::query.type' + - $ref: '#/components/parameters/bulk_stream::query.wait_for_active_shards' requestBody: - $ref: '#/components/requestBodies/bulk' + $ref: '#/components/requestBodies/bulk_stream' responses: '200': - $ref: '#/components/responses/bulk@200' + $ref: '#/components/responses/bulk_stream@200' /{index}/_count: get: operationId: count.2 @@ -2256,6 +2256,18 @@ components: - type: object description: The operation definition and data (action-data pairs), separated by newlines required: true + bulk_stream: + content: + application/x-ndjson: + schema: + type: array + items: + anyOf: + - $ref: '../schemas/_core.bulk.yaml#/components/schemas/OperationContainer' + - $ref: '../schemas/_core.bulk.yaml#/components/schemas/UpdateAction' + - type: object + description: The operation definition and data (action-data pairs), separated by newlines + required: true clear_scroll: content: application/json: @@ -2751,6 +2763,30 @@ components: - errors - items - took + bulk_stream@200: + content: + application/json: + schema: + type: object + properties: + errors: + type: boolean + items: + type: array + items: + type: object + additionalProperties: + $ref: '../schemas/_core.bulk.yaml#/components/schemas/ResponseItem' + minProperties: 1 + maxProperties: 1 + took: + type: number + ingest_took: + type: number + required: + - errors + - items + - took clear_scroll@200: content: application/json: @@ -3321,7 +3357,94 @@ components: $ref: '../schemas/_common.yaml#/components/schemas/WaitForActiveShards' style: form x-default: '1' - bulk::query.batch_size: + bulk_stream::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + bulk_stream::query._source: + in: query + name: _source + description: '`true` or `false` to return the `_source` field or not, or a list of fields to return.' + schema: + $ref: '../schemas/_core.search.yaml#/components/schemas/SourceConfigParam' + style: form + bulk_stream::query._source_excludes: + in: query + name: _source_excludes + description: A comma-separated list of source fields to exclude from the response. + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Fields' + style: form + bulk_stream::query._source_includes: + in: query + name: _source_includes + description: A comma-separated list of source fields to include in the response. + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Fields' + style: form + bulk_stream::query.pipeline: + in: query + name: pipeline + description: |- + ID of the pipeline to use to preprocess incoming documents. + If the index has a default ingest pipeline specified, then setting the value to `_none` disables the default ingest pipeline for this request. + If a final pipeline is configured it will always run, regardless of the value of this parameter. + schema: + type: string + style: form + bulk_stream::query.refresh: + in: query + name: refresh + description: |- + If `true`, OpenSearch refreshes the affected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` do nothing with refreshes. + Valid values: `true`, `false`, `wait_for`. + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Refresh' + style: form + bulk_stream::query.require_alias: + in: query + name: require_alias + description: If `true`, the request's actions must target an index alias. + schema: + type: boolean + default: false + style: form + bulk_stream::query.routing: + in: query + name: routing + description: Custom value used to route operations to a specific shard. + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Routing' + style: form + bulk_stream::query.timeout: + in: query + name: timeout + description: 'Period each action waits for the following operations: automatic index creation, dynamic mapping updates, waiting for active shards.' + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Duration' + style: form + bulk_stream::query.type: + name: type + in: query + description: Default document type for items which don't provide one. + schema: + type: string + description: Default document type for items which don't provide one. + bulk_stream::query.wait_for_active_shards: + in: query + name: wait_for_active_shards + description: |- + The number of shard copies that must be active before proceeding with the operation. + Set to all or any positive integer up to the total number of shards in the index (`number_of_replicas+1`). + schema: + $ref: '../schemas/_common.yaml#/components/schemas/WaitForActiveShards' + style: form + x-default: '1' + bulk_stream::query.batch_size: in: query name: batch_size description: |- @@ -3330,7 +3453,7 @@ components: $ref: '../schemas/_common.yaml#/components/schemas/BatchSize' style: form x-default: 1 - bulk::query.batch_interval: + bulk_stream::query.batch_interval: in: query name: batch_interval description: |- diff --git a/tests/default/_core/bulk_stream.yaml b/tests/default/_core/bulk/stream.yaml similarity index 96% rename from tests/default/_core/bulk_stream.yaml rename to tests/default/_core/bulk/stream.yaml index c3c7b278..26e457e2 100644 --- a/tests/default/_core/bulk_stream.yaml +++ b/tests/default/_core/bulk/stream.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test bulk streaming endpoint. epilogues: diff --git a/tests/default/indices/bulk_stream.yaml b/tests/default/indices/bulk/stream.yaml similarity index 100% rename from tests/default/indices/bulk_stream.yaml rename to tests/default/indices/bulk/stream.yaml