Added CBOR support.

Signed-off-by: dblock <[email protected]>

CHANGELOG.md

@@ -32,7 +32,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added support for `application/yaml` responses ([#363](https://github.com/opensearch-project/opensearch-api-specification/pull/363))
 - Added test for search with seq_no_primary_term ([#367](https://github.com/opensearch-project/opensearch-api-specification/pull/367))
 - Added a linter for parameter sorting ([#369](https://github.com/opensearch-project/opensearch-api-specification/pull/369))
-
+- Added support for `application/cbor` responses ([#371](https://github.com/opensearch-project/opensearch-api-specification/pull/371))
+
 ### Changed
 
 - Replaced Smithy with a native OpenAPI spec ([#189](https://github.com/opensearch-project/opensearch-api-specification/issues/189))

package.json

@@ -37,10 +37,12 @@
     "@eslint/eslintrc": "^3.0.2",
     "@eslint/js": "^9.1.1",
     "@stylistic/eslint-plugin": "^2.3.0",
+    "@types/cbor-js": "^0.1.1",
     "@types/jest": "^29.5.12",
     "@types/qs": "^6.9.15",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
     "ajv-errors": "^3.0.0",
+    "cbor": "^9.0.2",
     "eslint": "^8.57.0",
     "eslint-config-standard-with-typescript": "^43.0.1",
     "eslint-plugin-eslint-comments": "^3.2.0",

spec/namespaces/cat.yaml

@@ -853,6 +853,11 @@ components:
             type: array
             items:
               $ref: '../schemas/cat.health.yaml#/components/schemas/HealthRecord'
+        application/cbor:
+          schema:
+            type: array
+            items:
+              $ref: '../schemas/cat.health.yaml#/components/schemas/HealthRecord'
     cat.help@200:
       description: ''
       content:
@@ -874,6 +879,11 @@ components:
             type: array
             items:
               $ref: '../schemas/cat.indices.yaml#/components/schemas/IndicesRecord'
+        application/cbor:
+          schema:
+            type: array
+            items:
+              $ref: '../schemas/cat.indices.yaml#/components/schemas/IndicesRecord'
     cat.master@200:
       description: ''
       content:

tests/cat/health.yaml

@@ -69,3 +69,16 @@ chapters:
           status: yellow
           node.data: '1'
           discovered_cluster_manager: 'true'
+  - synopsis: Cat in different formats (format=cbor).
+    method: GET
+    path: /_cat/health
+    parameters:
+      format: cbor
+    response:
+      status: 200
+      content_type: application/yaml
+      payload:
+        - node.total: '1'
+          status: yellow
+          node.data: '1'
+          discovered_cluster_manager: 'true'

tests/cat/indices.yaml

@@ -70,3 +70,11 @@ chapters:
     response:
       status: 200
       content_type: application/yaml
+  - synopsis: Cat in different formats (format=cbor).
+    method: GET
+    path: /_cat/indices
+    parameters:
+      format: cbor
+    response:
+      status: 200
+      content_type: application/cbor

tools/src/OpenSearchHttpClient.ts

@@ -79,7 +79,8 @@ export class OpenSearchHttpClient {
           password: opts.password
         }
         : undefined,
-      httpsAgent: new https.Agent({ rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE) })
+      httpsAgent: new https.Agent({ rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE) }),
+      responseType: 'arraybuffer',
     })
   }
 

tools/src/tester/ChapterEvaluator.ts

@@ -19,16 +19,21 @@ import { ChapterOutput } from './ChapterOutput'
 import { Operation, atomizeChangeset, diff } from 'json-diff-ts'
 import YAML from 'yaml'
 import _ from 'lodash'
+import CBOR from 'cbor'
+import { Logger } from 'Logger'
+import { to_json } from '../helpers'
 
 export default class ChapterEvaluator {
+  private readonly logger: Logger
   private readonly _operation_locator: OperationLocator
   private readonly _chapter_reader: ChapterReader
   private readonly _schema_validator: SchemaValidator
 
-  constructor(spec_parser: OperationLocator, chapter_reader: ChapterReader, schema_validator: SchemaValidator) {
+  constructor(spec_parser: OperationLocator, chapter_reader: ChapterReader, schema_validator: SchemaValidator, logger: Logger) {
     this._operation_locator = spec_parser
     this._chapter_reader = chapter_reader
     this._schema_validator = schema_validator
+    this.logger = logger
   }
 
   async evaluate(chapter: Chapter, skip: boolean, story_outputs: StoryOutputs): Promise<ChapterEvaluation> {
@@ -88,6 +93,7 @@ export default class ChapterEvaluator {
     if (expected_payload == null) return { result: Result.PASSED }
     const content_type = response.content_type ?? 'application/json'
     const payload = this.#deserialize_payload(response.payload, content_type)
+    this.logger.info(`${to_json(payload)}`)
     const delta = atomizeChangeset(diff(expected_payload, payload))
     const messages: string[] = _.compact(delta.map((value, _index, _array) => {
       switch (value.type) {
@@ -111,9 +117,13 @@ export default class ChapterEvaluator {
 
   #deserialize_payload(payload: any, content_type: string): any {
     if (payload === undefined) return undefined
+    const payload_data = payload as string
     switch (content_type) {
-      case 'application/yaml': return YAML.parse(payload as string)
-      default: return payload
+      case 'text/plain': return Buffer.from(payload_data, 'binary').toString()
+      case 'application/json': return JSON.parse(Buffer.from(payload_data, 'binary').toString())
+      case 'application/yaml': return YAML.parse(Buffer.from(payload_data, 'binary').toString())
+      case 'application/cbor': return CBOR.decode(payload_data)
+      default: return Buffer.from(payload_data, 'binary').toString()
     }
   }
 }

tools/src/tester/ChapterReader.ts

@@ -43,7 +43,7 @@ export default class ChapterReader {
         return qs.stringify(params, { arrayFormat: 'comma' })
       }
     }).then(r => {
-      this.logger.info(`<= ${r.status} (${r.headers['content-type']}) | ${to_json(r.data)}`)
+      this.logger.info(`<= ${r.status} (${r.headers['content-type']}) | ${r.data?.length ?? 0} byte(s)`)
       response.status = r.status
       response.content_type = r.headers['content-type'].split(';')[0]
       response.payload = r.data
@@ -54,11 +54,21 @@ export default class ChapterReader {
       }
       response.status = e.response.status
       response.content_type = e.response.headers['content-type'].split(';')[0]
-      response.payload = e.response.data?.error
-      response.message = e.response.data?.error?.reason ?? e.response.statusText
+
+      try {
+        if (e.response.data !== undefined) {
+          const data = Buffer.from(e.response.data as string, 'binary').toString()
+          const payload = JSON.parse(data)
+          response.payload = payload?.error
+          response.message = payload.error?.reason ?? e.response.statusText
+        }
+      } catch {
+        // ignore
+      }
+
       response.error = e
 
-      this.logger.info(`<= ${response.status} (${response.content_type}) | ${to_json(response.payload ?? response.message)}`)
+      this.logger.info(`<= ${response.status} (${response.content_type}) | ${(response.payload ?? response.message ?? '').length} byte(s)`)
     })
     return response as ActualResponse
   }

tools/src/tester/test.ts

@@ -53,7 +53,7 @@ const logger = new Logger(opts.verbose ? LogLevel.info : LogLevel.warn)
 const spec = (new MergedOpenApiSpec(opts.specPath, new Logger(LogLevel.error))).spec()
 const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli(opts))
 const chapter_reader = new ChapterReader(http_client, logger)
-const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec), chapter_reader, new SchemaValidator(spec, logger))
+const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec), chapter_reader, new SchemaValidator(spec, logger), logger)
 const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader)
 const story_validator = new StoryValidator()
 const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapter_evaluator)

tools/tests/tester/fixtures/evals/passed.yaml

@@ -69,6 +69,22 @@ chapters:
         result: PASSED
       payload_schema:
         result: PASSED
+  - title: This GET /_cat/health chapter returns application/cbor and should pass.
+    overall:
+      result: PASSED
+    request:
+      parameters:
+        format:
+          result: PASSED
+      request_body:
+        result: PASSED
+    response:
+      status:
+        result: PASSED
+      payload_body:
+        result: PASSED
+      payload_schema:
+        result: PASSED
 epilogues:
   - title: DELETE /books
     overall:

tools/tests/tester/fixtures/specs/excerpt.yaml

@@ -101,6 +101,11 @@ components:
             type: array
             items:
               type: object
+        application/cbor:
+          schema:
+            type: array
+            items:
+              type: object
     indices.delete@200:
       description: ''
       content:

tools/tests/tester/fixtures/stories/passed.yaml

@@ -34,3 +34,13 @@ chapters:
       content_type: application/yaml
       payload:
         - node.total: '1'
+  - synopsis: This GET /_cat/health chapter returns application/cbor and should pass.
+    path: /_cat/health
+    parameters:
+      format: cbor
+    method: GET
+    response:
+      status: 200
+      content_type: application/cbor
+      payload:
+        - node.total: '1'

tools/tests/tester/helpers.ts

@@ -46,7 +46,7 @@ export function construct_tester_components (spec_path: string): {
   })
   const chapter_reader = new ChapterReader(opensearch_http_client, logger)
   const schema_validator = new SchemaValidator(specification, logger)
-  const chapter_evaluator = new ChapterEvaluator(operation_locator, chapter_reader, schema_validator)
+  const chapter_evaluator = new ChapterEvaluator(operation_locator, chapter_reader, schema_validator, logger)
   const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader)
   const story_validator = new StoryValidator()
   const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapter_evaluator)