Added CBOR support. (#371)

Signed-off-by: dblock <[email protected]>

CHANGELOG.md

@@ -32,7 +32,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added support for `application/yaml` responses ([#363](https://github.com/opensearch-project/opensearch-api-specification/pull/363))
 - Added test for search with seq_no_primary_term ([#367](https://github.com/opensearch-project/opensearch-api-specification/pull/367))
 - Added a linter for parameter sorting ([#369](https://github.com/opensearch-project/opensearch-api-specification/pull/369))
-
+- Added support for `application/cbor` responses ([#371](https://github.com/opensearch-project/opensearch-api-specification/pull/371))
+
 ### Changed
 
 - Replaced Smithy with a native OpenAPI spec ([#189](https://github.com/opensearch-project/opensearch-api-specification/issues/189))

package.json

@@ -37,10 +37,12 @@
     "@eslint/eslintrc": "^3.0.2",
     "@eslint/js": "^9.1.1",
     "@stylistic/eslint-plugin": "^2.3.0",
+    "@types/cbor-js": "^0.1.1",
     "@types/jest": "^29.5.12",
     "@types/qs": "^6.9.15",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
     "ajv-errors": "^3.0.0",
+    "cbor": "^9.0.2",
     "eslint": "^8.57.0",
     "eslint-config-standard-with-typescript": "^43.0.1",
     "eslint-plugin-eslint-comments": "^3.2.0",
@@ -50,7 +52,6 @@
     "eslint-plugin-promise": "^6.1.1",
     "eslint-plugin-yml": "^1.14.0",
     "globals": "^15.0.0",
-    "jest": "^29.7.0",
     "json-diff-ts": "^4.0.1",
     "json-schema-to-typescript": "^14.0.4",
     "qs": "^6.12.1",

spec/namespaces/cat.yaml

@@ -853,6 +853,11 @@ components:
             type: array
             items:
               $ref: '../schemas/cat.health.yaml#/components/schemas/HealthRecord'
+        application/cbor:
+          schema:
+            type: array
+            items:
+              $ref: '../schemas/cat.health.yaml#/components/schemas/HealthRecord'
     cat.help@200:
       description: ''
       content:
@@ -874,6 +879,11 @@ components:
             type: array
             items:
               $ref: '../schemas/cat.indices.yaml#/components/schemas/IndicesRecord'
+        application/cbor:
+          schema:
+            type: array
+            items:
+              $ref: '../schemas/cat.indices.yaml#/components/schemas/IndicesRecord'
     cat.master@200:
       description: ''
       content:

tests/cat/health.yaml

@@ -69,3 +69,16 @@ chapters:
           status: yellow
           node.data: '1'
           discovered_cluster_manager: 'true'
+  - synopsis: Cat in different formats (format=cbor).
+    method: GET
+    path: /_cat/health
+    parameters:
+      format: cbor
+    response:
+      status: 200
+      content_type: application/yaml
+      payload:
+        - node.total: '1'
+          status: yellow
+          node.data: '1'
+          discovered_cluster_manager: 'true'

tests/cat/indices.yaml

@@ -70,3 +70,11 @@ chapters:
     response:
       status: 200
       content_type: application/yaml
+  - synopsis: Cat in different formats (format=cbor).
+    method: GET
+    path: /_cat/indices
+    parameters:
+      format: cbor
+    response:
+      status: 200
+      content_type: application/cbor

tools/src/OpenSearchHttpClient.ts

@@ -8,7 +8,7 @@
 */
 
 import { Option } from '@commander-js/extra-typings'
-import axios, { type AxiosInstance, type AxiosRequestConfig, type AxiosResponse } from 'axios'
+import axios, { type AxiosInstance, type AxiosRequestConfig, type AxiosResponse, type ResponseType } from 'axios'
 import * as https from 'node:https'
 import { sleep } from './helpers'
 
@@ -34,7 +34,8 @@ export interface OpenSearchHttpClientOptions {
   url?: string
   username?: string
   password?: string
-  insecure?: boolean
+  insecure?: boolean,
+  responseType: ResponseType | undefined
 }
 
 export type OpenSearchHttpClientCliOptions = { [K in keyof OpenSearchHttpClientOptions as `opensearch${Capitalize<K>}`]: OpenSearchHttpClientOptions[K] }
@@ -44,7 +45,8 @@ export function get_opensearch_opts_from_cli (opts: OpenSearchHttpClientCliOptio
     url: opts.opensearchUrl,
     username: opts.opensearchUsername,
     password: opts.opensearchPassword,
-    insecure: opts.opensearchInsecure
+    insecure: opts.opensearchInsecure,
+    responseType: opts.opensearchResponseType
   }
 }
 
@@ -79,7 +81,8 @@ export class OpenSearchHttpClient {
           password: opts.password
         }
         : undefined,
-      httpsAgent: new https.Agent({ rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE) })
+      httpsAgent: new https.Agent({ rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE) }),
+      responseType: opts?.responseType,
     })
   }
 

tools/src/dump-cluster-spec/dump-cluster-spec.ts

@@ -48,7 +48,7 @@ const command = new Command()
 
 const opts = command.opts()
 
-main({ output: opts.output, opensearch: get_opensearch_opts_from_cli(opts) })
+main({ output: opts.output, opensearch: get_opensearch_opts_from_cli({ opensearchResponseType: undefined, ...opts }) })
   .catch(e => {
     if (e instanceof Error) {
       console.error(`ERROR: ${e.stack}`)

tools/src/tester/ChapterEvaluator.ts

@@ -17,18 +17,21 @@ import type SchemaValidator from './SchemaValidator'
 import { type StoryOutputs } from './StoryOutputs'
 import { ChapterOutput } from './ChapterOutput'
 import { Operation, atomizeChangeset, diff } from 'json-diff-ts'
-import YAML from 'yaml'
 import _ from 'lodash'
+import { Logger } from 'Logger'
+import { to_json } from '../helpers'
 
 export default class ChapterEvaluator {
+  private readonly logger: Logger
   private readonly _operation_locator: OperationLocator
   private readonly _chapter_reader: ChapterReader
   private readonly _schema_validator: SchemaValidator
 
-  constructor(spec_parser: OperationLocator, chapter_reader: ChapterReader, schema_validator: SchemaValidator) {
+  constructor(spec_parser: OperationLocator, chapter_reader: ChapterReader, schema_validator: SchemaValidator, logger: Logger) {
     this._operation_locator = spec_parser
     this._chapter_reader = chapter_reader
     this._schema_validator = schema_validator
+    this.logger = logger
   }
 
   async evaluate(chapter: Chapter, skip: boolean, story_outputs: StoryOutputs): Promise<ChapterEvaluation> {
@@ -86,8 +89,8 @@ export default class ChapterEvaluator {
 
   #evaluate_payload_body(response: ActualResponse, expected_payload?: Payload): Evaluation {
     if (expected_payload == null) return { result: Result.PASSED }
-    const content_type = response.content_type ?? 'application/json'
-    const payload = this.#deserialize_payload(response.payload, content_type)
+    const payload = response.payload
+    this.logger.info(`${to_json(payload)}`)
     const delta = atomizeChangeset(diff(expected_payload, payload))
     const messages: string[] = _.compact(delta.map((value, _index, _array) => {
       switch (value.type) {
@@ -102,18 +105,12 @@ export default class ChapterEvaluator {
 
   #evaluate_payload_schema(response: ActualResponse, operation: ParsedOperation): Evaluation {
     const content_type = response.content_type ?? 'application/json'
-    const content = operation.responses[response.status]?.content[content_type]
-    const schema = content?.schema
+    const content = operation.responses[response.status]?.content
+    // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
+    const content_type_content = content ? content[content_type] : undefined
+    const schema = content_type_content?.schema
     if (schema == null && content != null) return { result: Result.PASSED }
     if (schema == null) return { result: Result.FAILED, message: `Schema for "${response.status}: ${response.content_type}" response not found in the spec.` }
-    return this._schema_validator.validate(schema, this.#deserialize_payload(response.payload, content_type))
-  }
-
-  #deserialize_payload(payload: any, content_type: string): any {
-    if (payload === undefined) return undefined
-    switch (content_type) {
-      case 'application/yaml': return YAML.parse(payload as string)
-      default: return payload
-    }
+    return this._schema_validator.validate(schema, response.payload)
   }
 }

tools/src/tester/ChapterReader.ts

@@ -13,6 +13,8 @@ import { type StoryOutputs } from './StoryOutputs'
 import { Logger } from 'Logger'
 import { to_json, to_ndjson } from '../helpers'
 import qs from 'qs'
+import YAML from 'yaml'
+import CBOR from 'cbor'
 
 export default class ChapterReader {
   private readonly _client: OpenSearchHttpClient
@@ -43,22 +45,23 @@ export default class ChapterReader {
         return qs.stringify(params, { arrayFormat: 'comma' })
       }
     }).then(r => {
-      this.logger.info(`<= ${r.status} (${r.headers['content-type']}) | ${to_json(r.data)}`)
       response.status = r.status
-      response.content_type = r.headers['content-type'].split(';')[0]
-      response.payload = r.data
+      response.content_type = r.headers['content-type']?.split(';')[0]
+      response.payload = this.#deserialize_payload(r.data, response.content_type)
+      this.logger.info(`<= ${r.status} (${r.headers['content-type']}) | ${to_json(response.payload)}`)
     }).catch(e => {
       if (e.response == null) {
         this.logger.info(`<= ERROR: ${e}`)
         throw e
       }
       response.status = e.response.status
-      response.content_type = e.response.headers['content-type'].split(';')[0]
-      response.payload = e.response.data?.error
-      response.message = e.response.data?.error?.reason ?? e.response.statusText
+      response.content_type = e.response.headers['content-type']?.split(';')[0]
+      const payload = this.#deserialize_payload(e.response.data, response.content_type)
+      response.payload = payload?.error
+      response.message = payload.error?.reason ?? e.response.statusText
       response.error = e
 
-      this.logger.info(`<= ${response.status} (${response.content_type}) | ${to_json(response.payload ?? response.message)}`)
+      this.logger.info(`<= ${response.status} (${response.content_type}) | ${response.payload ?? response.message}`)
     })
     return response as ActualResponse
   }
@@ -80,4 +83,20 @@ export default class ChapterReader {
     const query_params = Object.fromEntries(Object.entries(parameters).filter(([key]) => !path_params.has(key)))
     return [parsed_path, query_params]
   }
+
+  #deserialize_payload(payload: any, content_type: any): any {
+    if (payload === undefined) return undefined
+    if (content_type === undefined) return payload
+    switch (content_type) {
+      case 'text/plain': return this.#deserialize_payload_data(payload as string)
+      case 'application/json': return payload.length == 0 ? {} : JSON.parse(this.#deserialize_payload_data(payload as string))
+      case 'application/yaml': return payload.length == 0 ? {} : YAML.parse(this.#deserialize_payload_data(payload as string))
+      case 'application/cbor': return payload.length == 0 ? {} : CBOR.decode(payload as string)
+      default: return this.#deserialize_payload_data(payload as string)
+    }
+  }
+
+  #deserialize_payload_data(payload: string): string {
+    return Buffer.from(payload, 'binary').toString()
+  }
 }

tools/src/tester/test.ts

@@ -51,9 +51,9 @@ const opts = command.opts()
 const logger = new Logger(opts.verbose ? LogLevel.info : LogLevel.warn)
 
 const spec = (new MergedOpenApiSpec(opts.specPath, new Logger(LogLevel.error))).spec()
-const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli(opts))
+const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli({ opensearchResponseType: 'arraybuffer', ...opts }))
 const chapter_reader = new ChapterReader(http_client, logger)
-const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec), chapter_reader, new SchemaValidator(spec, logger))
+const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec), chapter_reader, new SchemaValidator(spec, logger), logger)
 const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader)
 const story_validator = new StoryValidator()
 const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapter_evaluator)