Make TestsRunner.test.ts and TestsRunner.test.ts independent.

.github/workflows/test-tools.yml

@@ -46,11 +46,5 @@ jobs:
       - name: Lint
         run: npm run lint
 
-      - name: Test Spec Merger
-        run: npm run test -- tools/tests/merger
-
-      - name: Test Spec Linter
-        run: npm run test -- tools/tests/linter
-
-      - name: Test Spec Tester
-        run: npm run test -- tools/tests/tester --runInBand
+      - name: Test
+        run: npm run test

tools/tests/tester/StoryEvaluator.test.ts

@@ -9,43 +9,48 @@
 
 import { create_shared_resources, load_actual_evaluation, load_expected_evaluation } from './helpers'
 import { read_yaml } from '../../helpers'
-import { type OpenAPIV3 } from 'openapi-types'
 
-const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml')
-create_shared_resources(spec as OpenAPIV3.Document)
-
-test('passed', async () => {
-  const actual = await load_actual_evaluation('passed')
-  const expected = load_expected_evaluation('passed')
-  expect(actual).toEqual(expected)
-})
-
-test('skipped', async () => {
-  const actual = await load_actual_evaluation('skipped')
-  const expected = load_expected_evaluation('skipped')
-  expect(actual).toEqual(expected)
-})
-
-test('failed/not_found', async () => {
-  const actual = await load_actual_evaluation('failed/not_found')
-  const expected = load_expected_evaluation('failed/not_found')
-  expect(actual).toEqual(expected)
-})
-
-test('failed/invalid_data', async () => {
-  const actual = await load_actual_evaluation('failed/invalid_data')
-  const expected = load_expected_evaluation('failed/invalid_data')
-  expect(actual).toEqual(expected)
-})
-
-test('error/prologue_error', async () => {
-  const actual = await load_actual_evaluation('error/prologue_error')
-  const expected = load_expected_evaluation('error/prologue_error')
-  expect(actual).toEqual(expected)
-})
-
-test('error/chapter_error', async () => {
-  const actual = await load_actual_evaluation('error/chapter_error')
-  const expected = load_expected_evaluation('error/chapter_error')
-  expect(actual).toEqual(expected)
+describe('StoryEvaluator', () => {
+  beforeAll(() => {
+    // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml
+    process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!'
+    const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml')
+    create_shared_resources(spec)
+  })
+
+  test('passed', async () => {
+    const actual = await load_actual_evaluation('books/passed')
+    const expected = load_expected_evaluation('books/passed')
+    expect(actual).toEqual(expected)
+  })
+
+  test('skipped', async () => {
+    const actual = await load_actual_evaluation('books/skipped')
+    const expected = load_expected_evaluation('books/skipped')
+    expect(actual).toEqual(expected)
+  })
+
+  test('failed/not_found', async () => {
+    const actual = await load_actual_evaluation('books/failed/not_found')
+    const expected = load_expected_evaluation('books/failed/not_found')
+    expect(actual).toEqual(expected)
+  })
+
+  test('failed/invalid_data', async () => {
+    const actual = await load_actual_evaluation('books/failed/invalid_data')
+    const expected = load_expected_evaluation('books/failed/invalid_data')
+    expect(actual).toEqual(expected)
+  })
+
+  test('error/prologue_error', async () => {
+    const actual = await load_actual_evaluation('books/error/prologue_error')
+    const expected = load_expected_evaluation('books/error/prologue_error')
+    expect(actual).toEqual(expected)
+  })
+
+  test('error/chapter_error', async () => {
+    const actual = await load_actual_evaluation('books/error/chapter_error')
+    const expected = load_expected_evaluation('books/error/chapter_error')
+    expect(actual).toEqual(expected)
+  })
 })

tools/tests/tester/TestsRunner.test.ts

@@ -12,25 +12,33 @@ import TestsRunner from '../../src/tester/TestsRunner'
 import { type OpenAPIV3 } from 'openapi-types'
 import { load_expected_evaluation, scrub_errors } from './helpers'
 
-test('stories folder', async () => {
-  // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml
-  process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!'
-  const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml')
-  const runner = new TestsRunner(spec as OpenAPIV3.Document, 'tools/tests/tester/fixtures/stories', {})
-  const actual_evaluations = await runner.run(true) as any[]
-  for (const evaluation of actual_evaluations) scrub_errors(evaluation)
-  for (const evaluation of actual_evaluations) {
-    expect(evaluation.full_path.endsWith(evaluation.display_path)).toBeTruthy()
-    delete evaluation.full_path
-  }
+describe('TestsRunner', () => {
+  let spec: OpenAPIV3.Document
 
-  const skipped = load_expected_evaluation('skipped', true)
-  const passed = load_expected_evaluation('passed', true)
-  const not_found = load_expected_evaluation('failed/not_found', true)
-  const invalid_data = load_expected_evaluation('failed/invalid_data', true)
-  const chapter_error = load_expected_evaluation('error/chapter_error', true)
-  const prologue_error = load_expected_evaluation('error/prologue_error', true)
+  beforeAll(() => {
+    // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml
+    process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!'
+    spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml')
+  })
 
-  const expected_evaluations = [passed, skipped, chapter_error, prologue_error, invalid_data, not_found]
-  expect(actual_evaluations).toEqual(expected_evaluations)
+  test('run', async () => {
+    const runner = new TestsRunner(spec, 'tools/tests/tester/fixtures/stories/shoes', {})
+    const actual_evaluations = await runner.run(true) as any[]
+
+    for (const evaluation of actual_evaluations) {
+      scrub_errors(evaluation)
+      expect(evaluation.full_path.endsWith(evaluation.display_path)).toBeTruthy()
+      delete evaluation.full_path
+    }
+
+    const skipped = load_expected_evaluation('shoes/skipped', true)
+    const passed = load_expected_evaluation('shoes/passed', true)
+    const not_found = load_expected_evaluation('shoes/failed/not_found', true)
+    const invalid_data = load_expected_evaluation('shoes/failed/invalid_data', true)
+    const chapter_error = load_expected_evaluation('shoes/error/chapter_error', true)
+    const prologue_error = load_expected_evaluation('shoes/error/prologue_error', true)
+
+    const expected_evaluations = [passed, skipped, chapter_error, prologue_error, invalid_data, not_found]
+    expect(actual_evaluations).toEqual(expected_evaluations)
+  })
 })

tools/tests/tester/fixtures/evals/error/chapter_error.yaml → tools/tests/tester/fixtures/evals/books/error/chapter_error.yaml

@@ -1,5 +1,5 @@
-display_path: error/chapter_error.yaml
-full_path: tools/tests/tester/fixtures/stories/error/chapter_error.yaml
+display_path: books/error/chapter_error.yaml
+full_path: tools/tests/tester/fixtures/stories/books/error/chapter_error.yaml
 
 result: ERROR
 description: This story should failed due to missing info in the spec.

tools/tests/tester/fixtures/evals/error/prologue_error.yaml → tools/tests/tester/fixtures/evals/books/error/prologue_error.yaml

@@ -1,5 +1,5 @@
-display_path: error/prologue_error.yaml
-full_path: tools/tests/tester/fixtures/stories/error/prologue_error.yaml
+display_path: books/error/prologue_error.yaml
+full_path: tools/tests/tester/fixtures/stories/books/error/prologue_error.yaml
 
 result: ERROR
 description: This story should failed due to missing info in the spec.

tools/tests/tester/fixtures/evals/failed/invalid_data.yaml → tools/tests/tester/fixtures/evals/books/failed/invalid_data.yaml

@@ -1,5 +1,5 @@
-display_path: failed/invalid_data.yaml
-full_path: tools/tests/tester/fixtures/stories/failed/invalid_data.yaml
+display_path: books/failed/invalid_data.yaml
+full_path: tools/tests/tester/fixtures/stories/books/failed/invalid_data.yaml
 
 result: FAILED
 description: This story should failed due invalid data.

tools/tests/tester/fixtures/evals/books/failed/not_found.yaml

@@ -0,0 +1,65 @@
+display_path: books/failed/not_found.yaml
+full_path: tools/tests/tester/fixtures/stories/books/failed/not_found.yaml
+
+result: FAILED
+description: This story should failed due to missing info in the spec.
+
+prologues: []
+
+chapters:
+  - title: This chapter should fail because the operation is not defined in the spec.
+    overall:
+      result: FAILED
+      message: Operation "GET /_cat/health" not found in the spec.
+  - title: This chapter should fail because the parameter is not defined in the spec.
+    overall:
+      result: FAILED
+    request:
+      parameters:
+        index:
+          result: PASSED
+        timeout:
+          result: FAILED
+          message: Schema for "timeout" parameter not found.
+      request_body:
+        result: PASSED
+    response:
+      status:
+        result: PASSED
+      payload:
+        result: PASSED
+  - title: This chapter should fail because the request body is not defined in the spec.
+    overall:
+      result: FAILED
+    request:
+      parameters:
+        index:
+          result: PASSED
+      request_body:
+        result: FAILED
+        message: Schema for "application/json" request body not found in the spec.
+    response:
+      status:
+        result: PASSED
+      payload:
+        result: PASSED
+  - title: This chapter should fail because the response is not defined in the spec.
+    overall:
+      result: FAILED
+    request:
+      parameters:
+        index:
+          result: PASSED
+      request_body:
+        result: PASSED
+    response:
+      status:
+        result: PASSED
+      payload:
+        result: FAILED
+        message: 'Schema for "404: application/json" response not found in the spec.'
+
+epilogues:
+  - title: DELETE /books
+    overall:
+      result: PASSED

tools/tests/tester/fixtures/evals/books/passed.yaml

@@ -0,0 +1,28 @@
+display_path: books/passed.yaml
+full_path: tools/tests/tester/fixtures/stories/books/passed.yaml
+
+result: PASSED
+description: This story should pass.
+
+prologues: []
+
+chapters:
+  - title: This chapter should pass.
+    overall:
+      result: PASSED
+    request:
+      parameters:
+        index:
+          result: PASSED
+      request_body:
+        result: PASSED
+    response:
+      status:
+        result: PASSED
+      payload:
+        result: PASSED
+
+epilogues:
+  - title: DELETE /books
+    overall:
+      result: PASSED

tools/tests/tester/fixtures/evals/books/skipped.yaml

@@ -0,0 +1,7 @@
+display_path: books/skipped.yaml
+full_path: "tools/tests/tester/fixtures/stories/books/skipped.yaml"
+
+result: SKIPPED
+description: This story should be skipped.
+
+chapters: []

tools/tests/tester/fixtures/evals/shoes/error/chapter_error.yaml

@@ -0,0 +1,39 @@
+display_path: error/chapter_error.yaml
+full_path: tools/tests/tester/fixtures/stories/shoes/error/chapter_error.yaml
+
+result: ERROR
+description: This story should failed due to missing info in the spec.
+
+prologues:
+  - title: PUT /shoes
+    overall:
+      result: PASSED
+
+chapters:
+  - title: This chapter should fail.
+    overall:
+      result: FAILED
+      message: Operation "GET /{index}/settings" not found in the spec.
+  - title: This chapter show throw an error.
+    overall:
+      result: ERROR
+    request:
+      parameters: {}
+      request_body:
+        result: PASSED
+    response:
+      status:
+        result: ERROR
+        message: 'Expected status 200, but received 404: application/json. no such index
+            [undefined]'
+        error: Request failed with status code 404
+      payload:
+        result: SKIPPED
+  - title: This chapter should be skipped.
+    overall:
+      result: SKIPPED
+
+epilogues:
+  - title: DELETE /shoes
+    overall:
+      result: PASSED

tools/tests/tester/fixtures/evals/shoes/error/prologue_error.yaml

@@ -0,0 +1,28 @@
+display_path: error/prologue_error.yaml
+full_path: tools/tests/tester/fixtures/stories/shoes/error/prologue_error.yaml
+
+result: ERROR
+description: This story should failed due to missing info in the spec.
+
+prologues:
+  - title: PUT /shoes
+    overall:
+      result: PASSED
+  - title: DELETE /does_not_exists
+    overall:
+      result: ERROR
+      message: no such index [does_not_exists]
+      error: Request failed with status code 404
+
+chapters:
+  - title: This chapter be skipped.
+    overall:
+      result: SKIPPED
+  - title: This chapter be skipped.
+    overall:
+      result: SKIPPED
+
+epilogues:
+  - title: DELETE /shoes
+    overall:
+      result: PASSED