diff --git a/tools/tests/tester/StoryEvaluator.test.ts b/tools/tests/tester/StoryEvaluator.test.ts index 3cc57aa90..1ae2787c2 100644 --- a/tools/tests/tester/StoryEvaluator.test.ts +++ b/tools/tests/tester/StoryEvaluator.test.ts @@ -9,43 +9,48 @@ import { create_shared_resources, load_actual_evaluation, load_expected_evaluation } from './helpers' import { read_yaml } from '../../helpers' -import { type OpenAPIV3 } from 'openapi-types' -const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml') -create_shared_resources(spec as OpenAPIV3.Document) - -test('passed', async () => { - const actual = await load_actual_evaluation('passed') - const expected = await load_expected_evaluation('passed') - expect(actual).toEqual(expected) -}) - -test('skipped', async () => { - const actual = await load_actual_evaluation('skipped') - const expected = await load_expected_evaluation('skipped') - expect(actual).toEqual(expected) -}) - -test('failed/not_found', async () => { - const actual = await load_actual_evaluation('failed/not_found') - const expected = await load_expected_evaluation('failed/not_found') - expect(actual).toEqual(expected) -}) - -test('failed/invalid_data', async () => { - const actual = await load_actual_evaluation('failed/invalid_data') - const expected = await load_expected_evaluation('failed/invalid_data') - expect(actual).toEqual(expected) -}) - -test('error/prologue_error', async () => { - const actual = await load_actual_evaluation('error/prologue_error') - const expected = await load_expected_evaluation('error/prologue_error') - expect(actual).toEqual(expected) -}) - -test('error/chapter_error', async () => { - const actual = await load_actual_evaluation('error/chapter_error') - const expected = await load_expected_evaluation('error/chapter_error') - expect(actual).toEqual(expected) +describe('StoryEvaluator', () => { + beforeAll(() => { + // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml + process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!' + const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml') + create_shared_resources(spec) + }) + + test('passed', async () => { + const actual = await load_actual_evaluation('books/passed') + const expected = await load_expected_evaluation('books/passed') + expect(actual).toEqual(expected) + }) + + test('skipped', async () => { + const actual = await load_actual_evaluation('books/skipped') + const expected = await load_expected_evaluation('books/skipped') + expect(actual).toEqual(expected) + }) + + test('failed/not_found', async () => { + const actual = await load_actual_evaluation('books/failed/not_found') + const expected = await load_expected_evaluation('books/failed/not_found') + expect(actual).toEqual(expected) + }) + + test('failed/invalid_data', async () => { + const actual = await load_actual_evaluation('books/failed/invalid_data') + const expected = await load_expected_evaluation('books/failed/invalid_data') + expect(actual).toEqual(expected) + }) + + test('error/prologue_error', async () => { + const actual = await load_actual_evaluation('books/error/prologue_error') + const expected = await load_expected_evaluation('books/error/prologue_error') + expect(actual).toEqual(expected) + }) + + test('error/chapter_error', async () => { + const actual = await load_actual_evaluation('books/error/chapter_error') + const expected = await load_expected_evaluation('books/error/chapter_error') + expect(actual).toEqual(expected) + }) }) diff --git a/tools/tests/tester/TestsRunner.test.ts b/tools/tests/tester/TestsRunner.test.ts index d2290d808..07425557d 100644 --- a/tools/tests/tester/TestsRunner.test.ts +++ b/tools/tests/tester/TestsRunner.test.ts @@ -12,25 +12,33 @@ import TestsRunner from '../../src/tester/TestsRunner' import { type OpenAPIV3 } from 'openapi-types' import { load_expected_evaluation, scrub_errors } from './helpers' -test('stories folder', async () => { - // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml - process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!' - const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml') - const runner = new TestsRunner(spec as OpenAPIV3.Document, 'tools/tests/tester/fixtures/stories', {}) - const actual_evaluations = await runner.run(true) as any[] - for (const evaluation of actual_evaluations) scrub_errors(evaluation) - for (const evaluation of actual_evaluations) { - expect(evaluation.full_path.endsWith(evaluation.display_path)).toBeTruthy() - delete evaluation.full_path - } +describe('TestsRunner', () => { + let spec: OpenAPIV3.Document - const skipped = await load_expected_evaluation('skipped', true) - const passed = await load_expected_evaluation('passed', true) - const not_found = await load_expected_evaluation('failed/not_found', true) - const invalid_data = await load_expected_evaluation('failed/invalid_data', true) - const chapter_error = await load_expected_evaluation('error/chapter_error', true) - const prologue_error = await load_expected_evaluation('error/prologue_error', true) + beforeAll(() => { + // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml + process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!' + spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml') + }) - const expected_evaluations = [passed, skipped, chapter_error, prologue_error, invalid_data, not_found] - expect(actual_evaluations).toEqual(expected_evaluations) + test('run', async () => { + const runner = new TestsRunner(spec, 'tools/tests/tester/fixtures/stories/shoes', {}) + const actual_evaluations = await runner.run(true) as any[] + + for (const evaluation of actual_evaluations) { + scrub_errors(evaluation) + expect(evaluation.full_path.endsWith(evaluation.display_path)).toBeTruthy() + delete evaluation.full_path + } + + const skipped = await load_expected_evaluation('shoes/skipped', true) + const passed = await load_expected_evaluation('shoes/passed', true) + const not_found = await load_expected_evaluation('shoes/failed/not_found', true) + const invalid_data = await load_expected_evaluation('shoes/failed/invalid_data', true) + const chapter_error = await load_expected_evaluation('shoes/error/chapter_error', true) + const prologue_error = await load_expected_evaluation('shoes/error/prologue_error', true) + + const expected_evaluations = [passed, skipped, chapter_error, prologue_error, invalid_data, not_found] + expect(actual_evaluations).toEqual(expected_evaluations) + }) }) diff --git a/tools/tests/tester/fixtures/evals/error/chapter_error.yaml b/tools/tests/tester/fixtures/evals/books/error/chapter_error.yaml similarity index 87% rename from tools/tests/tester/fixtures/evals/error/chapter_error.yaml rename to tools/tests/tester/fixtures/evals/books/error/chapter_error.yaml index 5a3b6977c..8d3eff9d4 100644 --- a/tools/tests/tester/fixtures/evals/error/chapter_error.yaml +++ b/tools/tests/tester/fixtures/evals/books/error/chapter_error.yaml @@ -1,5 +1,5 @@ -display_path: error/chapter_error.yaml -full_path: tools/tests/tester/fixtures/stories/error/chapter_error.yaml +display_path: books/error/chapter_error.yaml +full_path: tools/tests/tester/fixtures/stories/books/error/chapter_error.yaml result: ERROR description: This story should failed due to missing info in the spec. diff --git a/tools/tests/tester/fixtures/evals/error/prologue_error.yaml b/tools/tests/tester/fixtures/evals/books/error/prologue_error.yaml similarity index 81% rename from tools/tests/tester/fixtures/evals/error/prologue_error.yaml rename to tools/tests/tester/fixtures/evals/books/error/prologue_error.yaml index 299fa9fee..487776293 100644 --- a/tools/tests/tester/fixtures/evals/error/prologue_error.yaml +++ b/tools/tests/tester/fixtures/evals/books/error/prologue_error.yaml @@ -1,5 +1,5 @@ -display_path: error/prologue_error.yaml -full_path: tools/tests/tester/fixtures/stories/error/prologue_error.yaml +display_path: books/error/prologue_error.yaml +full_path: tools/tests/tester/fixtures/stories/books/error/prologue_error.yaml result: ERROR description: This story should failed due to missing info in the spec. diff --git a/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml b/tools/tests/tester/fixtures/evals/books/failed/invalid_data.yaml similarity index 91% rename from tools/tests/tester/fixtures/evals/failed/invalid_data.yaml rename to tools/tests/tester/fixtures/evals/books/failed/invalid_data.yaml index 1cc11180a..c0e24da7b 100644 --- a/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml +++ b/tools/tests/tester/fixtures/evals/books/failed/invalid_data.yaml @@ -1,5 +1,5 @@ -display_path: failed/invalid_data.yaml -full_path: tools/tests/tester/fixtures/stories/failed/invalid_data.yaml +display_path: books/failed/invalid_data.yaml +full_path: tools/tests/tester/fixtures/stories/books/failed/invalid_data.yaml result: FAILED description: This story should failed due invalid data. diff --git a/tools/tests/tester/fixtures/evals/books/failed/not_found.yaml b/tools/tests/tester/fixtures/evals/books/failed/not_found.yaml new file mode 100644 index 000000000..6613f0862 --- /dev/null +++ b/tools/tests/tester/fixtures/evals/books/failed/not_found.yaml @@ -0,0 +1,65 @@ +display_path: books/failed/not_found.yaml +full_path: tools/tests/tester/fixtures/stories/books/failed/not_found.yaml + +result: FAILED +description: This story should failed due to missing info in the spec. + +prologues: [] + +chapters: + - title: This chapter should fail because the operation is not defined in the spec. + overall: + result: FAILED + message: Operation "GET /_cat/health" not found in the spec. + - title: This chapter should fail because the parameter is not defined in the spec. + overall: + result: FAILED + request: + parameters: + index: + result: PASSED + timeout: + result: FAILED + message: Schema for "timeout" parameter not found. + request_body: + result: PASSED + response: + status: + result: PASSED + payload: + result: PASSED + - title: This chapter should fail because the request body is not defined in the spec. + overall: + result: FAILED + request: + parameters: + index: + result: PASSED + request_body: + result: FAILED + message: Schema for "application/json" request body not found in the spec. + response: + status: + result: PASSED + payload: + result: PASSED + - title: This chapter should fail because the response is not defined in the spec. + overall: + result: FAILED + request: + parameters: + index: + result: PASSED + request_body: + result: PASSED + response: + status: + result: PASSED + payload: + result: FAILED + message: 'Schema for "404: application/json" response not found in the spec.' + +epilogues: + - title: DELETE /books + overall: + result: PASSED \ No newline at end of file diff --git a/tools/tests/tester/fixtures/evals/books/passed.yaml b/tools/tests/tester/fixtures/evals/books/passed.yaml new file mode 100644 index 000000000..1b5fc7497 --- /dev/null +++ b/tools/tests/tester/fixtures/evals/books/passed.yaml @@ -0,0 +1,28 @@ +display_path: books/passed.yaml +full_path: tools/tests/tester/fixtures/stories/books/passed.yaml + +result: PASSED +description: This story should pass. + +prologues: [] + +chapters: + - title: This chapter should pass. + overall: + result: PASSED + request: + parameters: + index: + result: PASSED + request_body: + result: PASSED + response: + status: + result: PASSED + payload: + result: PASSED + +epilogues: + - title: DELETE /books + overall: + result: PASSED \ No newline at end of file diff --git a/tools/tests/tester/fixtures/evals/books/skipped.yaml b/tools/tests/tester/fixtures/evals/books/skipped.yaml new file mode 100644 index 000000000..16edaaffd --- /dev/null +++ b/tools/tests/tester/fixtures/evals/books/skipped.yaml @@ -0,0 +1,7 @@ +display_path: books/skipped.yaml +full_path: "tools/tests/tester/fixtures/stories/books/skipped.yaml" + +result: SKIPPED +description: This story should be skipped. + +chapters: [] \ No newline at end of file diff --git a/tools/tests/tester/fixtures/evals/shoes/error/chapter_error.yaml b/tools/tests/tester/fixtures/evals/shoes/error/chapter_error.yaml new file mode 100644 index 000000000..53612b975 --- /dev/null +++ b/tools/tests/tester/fixtures/evals/shoes/error/chapter_error.yaml @@ -0,0 +1,39 @@ +display_path: error/chapter_error.yaml +full_path: tools/tests/tester/fixtures/stories/shoes/error/chapter_error.yaml + +result: ERROR +description: This story should failed due to missing info in the spec. + +prologues: + - title: PUT /shoes + overall: + result: PASSED + +chapters: + - title: This chapter should fail. + overall: + result: FAILED + message: Operation "GET /{index}/settings" not found in the spec. + - title: This chapter show throw an error. + overall: + result: ERROR + request: + parameters: {} + request_body: + result: PASSED + response: + status: + result: ERROR + message: 'Expected status 200, but received 404: application/json. no such index + [undefined]' + error: Request failed with status code 404 + payload: + result: SKIPPED + - title: This chapter should be skipped. + overall: + result: SKIPPED + +epilogues: + - title: DELETE /shoes + overall: + result: PASSED \ No newline at end of file diff --git a/tools/tests/tester/fixtures/evals/shoes/error/prologue_error.yaml b/tools/tests/tester/fixtures/evals/shoes/error/prologue_error.yaml new file mode 100644 index 000000000..bbaf5c99e --- /dev/null +++ b/tools/tests/tester/fixtures/evals/shoes/error/prologue_error.yaml @@ -0,0 +1,28 @@ +display_path: error/prologue_error.yaml +full_path: tools/tests/tester/fixtures/stories/shoes/error/prologue_error.yaml + +result: ERROR +description: This story should failed due to missing info in the spec. + +prologues: + - title: PUT /shoes + overall: + result: PASSED + - title: DELETE /does_not_exists + overall: + result: ERROR + message: no such index [does_not_exists] + error: Request failed with status code 404 + +chapters: + - title: This chapter be skipped. + overall: + result: SKIPPED + - title: This chapter be skipped. + overall: + result: SKIPPED + +epilogues: + - title: DELETE /shoes + overall: + result: PASSED \ No newline at end of file diff --git a/tools/tests/tester/fixtures/evals/shoes/failed/invalid_data.yaml b/tools/tests/tester/fixtures/evals/shoes/failed/invalid_data.yaml new file mode 100644 index 000000000..0ec407a29 --- /dev/null +++ b/tools/tests/tester/fixtures/evals/shoes/failed/invalid_data.yaml @@ -0,0 +1,62 @@ +display_path: failed/invalid_data.yaml +full_path: tools/tests/tester/fixtures/stories/shoes/failed/invalid_data.yaml + +result: FAILED +description: This story should failed due invalid data. + +prologues: [] + +chapters: + - title: This chapter should fail because the parameter is invalid. + overall: + result: FAILED + request: + parameters: + index: + result: FAILED + message: data must be string + request_body: + result: PASSED + response: + status: + result: PASSED + payload: + result: PASSED + - title: This chapter should fail because the request body is invalid. + overall: + result: FAILED + request: + parameters: + index: + result: PASSED + request_body: + result: FAILED + message: data must NOT have additional properties + response: + status: + result: PASSED + payload: + result: PASSED + - title: This chapter should fail because the response is invalid. + overall: + result: FAILED + request: + parameters: + index: + result: PASSED + request_body: + result: PASSED + response: + status: + result: PASSED + payload: + result: FAILED + message: data must NOT have additional properties + +epilogues: + - title: DELETE /shoes + overall: + result: PASSED + - title: DELETE /40 + overall: + result: PASSED \ No newline at end of file diff --git a/tools/tests/tester/fixtures/evals/failed/not_found.yaml b/tools/tests/tester/fixtures/evals/shoes/failed/not_found.yaml similarity index 94% rename from tools/tests/tester/fixtures/evals/failed/not_found.yaml rename to tools/tests/tester/fixtures/evals/shoes/failed/not_found.yaml index 06d186c73..d3191f068 100644 --- a/tools/tests/tester/fixtures/evals/failed/not_found.yaml +++ b/tools/tests/tester/fixtures/evals/shoes/failed/not_found.yaml @@ -1,5 +1,5 @@ display_path: failed/not_found.yaml -full_path: tools/tests/tester/fixtures/stories/failed/not_found.yaml +full_path: tools/tests/tester/fixtures/stories/shoes/failed/not_found.yaml result: FAILED description: This story should failed due to missing info in the spec. @@ -60,6 +60,6 @@ chapters: message: 'Schema for "404: application/json" response not found in the spec.' epilogues: - - title: DELETE /books + - title: DELETE /shoes overall: result: PASSED \ No newline at end of file diff --git a/tools/tests/tester/fixtures/evals/passed.yaml b/tools/tests/tester/fixtures/evals/shoes/passed.yaml similarity index 82% rename from tools/tests/tester/fixtures/evals/passed.yaml rename to tools/tests/tester/fixtures/evals/shoes/passed.yaml index e7ec69ba3..566347e98 100644 --- a/tools/tests/tester/fixtures/evals/passed.yaml +++ b/tools/tests/tester/fixtures/evals/shoes/passed.yaml @@ -1,5 +1,5 @@ display_path: passed.yaml -full_path: tools/tests/tester/fixtures/stories/passed.yaml +full_path: tools/tests/tester/fixtures/stories/shoes/passed.yaml result: PASSED description: This story should pass. @@ -23,6 +23,6 @@ chapters: result: PASSED epilogues: - - title: DELETE /books + - title: DELETE /shoes overall: result: PASSED \ No newline at end of file diff --git a/tools/tests/tester/fixtures/evals/skipped.yaml b/tools/tests/tester/fixtures/evals/shoes/skipped.yaml similarity index 59% rename from tools/tests/tester/fixtures/evals/skipped.yaml rename to tools/tests/tester/fixtures/evals/shoes/skipped.yaml index 663cb1acf..60b83138a 100644 --- a/tools/tests/tester/fixtures/evals/skipped.yaml +++ b/tools/tests/tester/fixtures/evals/shoes/skipped.yaml @@ -1,5 +1,5 @@ display_path: skipped.yaml -full_path: "tools/tests/tester/fixtures/stories/skipped.yaml" +full_path: "tools/tests/tester/fixtures/stories/shoes/skipped.yaml" result: SKIPPED description: This story should be skipped. diff --git a/tools/tests/tester/fixtures/stories/error/chapter_error.yaml b/tools/tests/tester/fixtures/stories/books/error/chapter_error.yaml similarity index 86% rename from tools/tests/tester/fixtures/stories/error/chapter_error.yaml rename to tools/tests/tester/fixtures/stories/books/error/chapter_error.yaml index 2272a666d..1f5f55e17 100644 --- a/tools/tests/tester/fixtures/stories/error/chapter_error.yaml +++ b/tools/tests/tester/fixtures/stories/books/error/chapter_error.yaml @@ -1,4 +1,4 @@ -$schema: ../../../../../../json_schemas/test_story.schema.yaml +$schema: ../../../../../../../json_schemas/test_story.schema.yaml description: This story should failed due to missing info in the spec. prologues: diff --git a/tools/tests/tester/fixtures/stories/error/prologue_error.yaml b/tools/tests/tester/fixtures/stories/books/error/prologue_error.yaml similarity index 85% rename from tools/tests/tester/fixtures/stories/error/prologue_error.yaml rename to tools/tests/tester/fixtures/stories/books/error/prologue_error.yaml index 8b11deb92..ef2f4cfdb 100644 --- a/tools/tests/tester/fixtures/stories/error/prologue_error.yaml +++ b/tools/tests/tester/fixtures/stories/books/error/prologue_error.yaml @@ -1,4 +1,4 @@ -$schema: ../../../../../../json_schemas/test_story.schema.yaml +$schema: ../../../../../../../json_schemas/test_story.schema.yaml description: This story should failed due to missing info in the spec. prologues: diff --git a/tools/tests/tester/fixtures/stories/failed/invalid_data.yaml b/tools/tests/tester/fixtures/stories/books/failed/invalid_data.yaml similarity index 91% rename from tools/tests/tester/fixtures/stories/failed/invalid_data.yaml rename to tools/tests/tester/fixtures/stories/books/failed/invalid_data.yaml index fbb6573b2..81f15f862 100644 --- a/tools/tests/tester/fixtures/stories/failed/invalid_data.yaml +++ b/tools/tests/tester/fixtures/stories/books/failed/invalid_data.yaml @@ -1,4 +1,4 @@ -$schema: ../../../../../../json_schemas/test_story.schema.yaml +$schema: ../../../../../../../json_schemas/test_story.schema.yaml description: This story should failed due invalid data. epilogues: diff --git a/tools/tests/tester/fixtures/stories/failed/not_found.yaml b/tools/tests/tester/fixtures/stories/books/failed/not_found.yaml similarity index 92% rename from tools/tests/tester/fixtures/stories/failed/not_found.yaml rename to tools/tests/tester/fixtures/stories/books/failed/not_found.yaml index b210f74e9..ed7f32c1f 100644 --- a/tools/tests/tester/fixtures/stories/failed/not_found.yaml +++ b/tools/tests/tester/fixtures/stories/books/failed/not_found.yaml @@ -1,4 +1,4 @@ -$schema: ../../../../../../json_schemas/test_story.schema.yaml +$schema: ../../../../../../../json_schemas/test_story.schema.yaml description: This story should failed due to missing info in the spec. epilogues: diff --git a/tools/tests/tester/fixtures/stories/passed.yaml b/tools/tests/tester/fixtures/stories/books/passed.yaml similarity index 78% rename from tools/tests/tester/fixtures/stories/passed.yaml rename to tools/tests/tester/fixtures/stories/books/passed.yaml index 4a7fbfea3..939b279e8 100644 --- a/tools/tests/tester/fixtures/stories/passed.yaml +++ b/tools/tests/tester/fixtures/stories/books/passed.yaml @@ -1,4 +1,4 @@ -$schema: ../../../../../json_schemas/test_story.schema.yaml +$schema: ../../../../../../json_schemas/test_story.schema.yaml description: This story should pass. epilogues: diff --git a/tools/tests/tester/fixtures/stories/skipped.yaml b/tools/tests/tester/fixtures/stories/books/skipped.yaml similarity index 85% rename from tools/tests/tester/fixtures/stories/skipped.yaml rename to tools/tests/tester/fixtures/stories/books/skipped.yaml index 256d65094..ed731a616 100644 --- a/tools/tests/tester/fixtures/stories/skipped.yaml +++ b/tools/tests/tester/fixtures/stories/books/skipped.yaml @@ -1,4 +1,4 @@ -$schema: ../../../../../json_schemas/test_story.schema.yaml +$schema: ../../../../../../json_schemas/test_story.schema.yaml skip: true description: This story should be skipped. diff --git a/tools/tests/tester/fixtures/stories/shoes/error/chapter_error.yaml b/tools/tests/tester/fixtures/stories/shoes/error/chapter_error.yaml new file mode 100644 index 000000000..e09d8e75f --- /dev/null +++ b/tools/tests/tester/fixtures/stories/shoes/error/chapter_error.yaml @@ -0,0 +1,20 @@ +$schema: ../../../../../../../../json_schemas/test_story.schema.yaml + +description: This story should failed due to missing info in the spec. +prologues: + - path: /shoes + method: PUT +epilogues: + - path: /shoes + method: DELETE +chapters: + - synopsis: This chapter should fail. + path: /{index}/settings + method: GET + - synopsis: This chapter show throw an error. + path: /{index} + method: DELETE + - synopsis: This chapter should be skipped. + path: /_cat/indices + method: GET + diff --git a/tools/tests/tester/fixtures/stories/shoes/error/prologue_error.yaml b/tools/tests/tester/fixtures/stories/shoes/error/prologue_error.yaml new file mode 100644 index 000000000..dbc780482 --- /dev/null +++ b/tools/tests/tester/fixtures/stories/shoes/error/prologue_error.yaml @@ -0,0 +1,19 @@ +$schema: ../../../../../../../json_schemas/test_story.schema.yaml + +description: This story should failed due to missing info in the spec. +prologues: + - path: /shoes + method: PUT + - path: /does_not_exists + method: DELETE +epilogues: + - path: /shoes + method: DELETE +chapters: + - synopsis: This chapter be skipped. + path: /_cat/health + method: GET + - synopsis: This chapter be skipped. + path: /_cat/indices + method: GET + diff --git a/tools/tests/tester/fixtures/stories/shoes/failed/invalid_data.yaml b/tools/tests/tester/fixtures/stories/shoes/failed/invalid_data.yaml new file mode 100644 index 000000000..0b540e180 --- /dev/null +++ b/tools/tests/tester/fixtures/stories/shoes/failed/invalid_data.yaml @@ -0,0 +1,33 @@ +$schema: ../../../../../../../json_schemas/test_story.schema.yaml + +description: This story should failed due invalid data. +epilogues: + - path: /shoes + method: DELETE + status: [200, 404] + - path: /40 + method: DELETE + status: [200, 404] +chapters: + - synopsis: This chapter should fail because the parameter is invalid. + path: /{index} + method: PUT + parameters: + index: 40 + - synopsis: This chapter should fail because the request body is invalid. + path: /{index} + method: PUT + parameters: + index: shoes + request_body: + payload: + aliases: {} + - synopsis: This chapter should fail because the response is invalid. + path: /{index} + method: DELETE + parameters: + index: shoes + response: + status: 200 + payload: + shards_acknowledged: true \ No newline at end of file diff --git a/tools/tests/tester/fixtures/stories/shoes/failed/not_found.yaml b/tools/tests/tester/fixtures/stories/shoes/failed/not_found.yaml new file mode 100644 index 000000000..c864ee00e --- /dev/null +++ b/tools/tests/tester/fixtures/stories/shoes/failed/not_found.yaml @@ -0,0 +1,31 @@ +$schema: ../../../../../../../json_schemas/test_story.schema.yaml + +description: This story should failed due to missing info in the spec. +epilogues: + - path: /shoes + method: DELETE + status: [200, 404] +chapters: + - synopsis: This chapter should fail because the operation is not defined in the spec. + path: /_cat/health + method: GET + - synopsis: This chapter should fail because the parameter is not defined in the spec. + path: /{index} + method: PUT + parameters: + index: shoes + timeout: 30s + - synopsis: This chapter should fail because the request body is not defined in the spec. + path: /{index} + method: HEAD + parameters: + index: shoes + request_body: + payload: {} + - synopsis: This chapter should fail because the response is not defined in the spec. + path: /{index} + method: DELETE + parameters: + index: cats + response: + status: 404 diff --git a/tools/tests/tester/fixtures/stories/shoes/passed.yaml b/tools/tests/tester/fixtures/stories/shoes/passed.yaml new file mode 100644 index 000000000..7f50e91bf --- /dev/null +++ b/tools/tests/tester/fixtures/stories/shoes/passed.yaml @@ -0,0 +1,13 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: This story should pass. +epilogues: + - path: /shoes + method: DELETE + status: [200, 404] +chapters: + - synopsis: This chapter should pass. + path: /{index} + method: PUT + parameters: + index: shoes diff --git a/tools/tests/tester/fixtures/stories/shoes/skipped.yaml b/tools/tests/tester/fixtures/stories/shoes/skipped.yaml new file mode 100644 index 000000000..ed731a616 --- /dev/null +++ b/tools/tests/tester/fixtures/stories/shoes/skipped.yaml @@ -0,0 +1,17 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +skip: true +description: This story should be skipped. +prologues: + - path: /_cluster/settings + method: GET +epilogues: + - path: /_cluster/settings + method: PUT +chapters: + - synopsis: This chapter should not be executed. + path: /_cat/health + method: GET + - synopsis: This chapter should not be executed. + path: /_cat/indices + method: GET \ No newline at end of file diff --git a/tools/tests/tester/helpers.test.ts b/tools/tests/tester/helpers.test.ts new file mode 100644 index 000000000..8383edb42 --- /dev/null +++ b/tools/tests/tester/helpers.test.ts @@ -0,0 +1,25 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import { overall_result } from '../../src/tester/helpers' +import { type Evaluation, Result } from '../../src/tester/types/eval.types' + +describe('helpers', () => { + function e (...results: Result[]): Evaluation[] { + return results.map(result => ({ result })) + } + + test('overall_result', () => { + expect(overall_result(e(Result.PASSED, Result.SKIPPED, Result.FAILED, Result.ERROR))).toBe(Result.ERROR) + expect(overall_result(e(Result.PASSED, Result.SKIPPED, Result.FAILED))).toBe(Result.FAILED) + expect(overall_result(e(Result.PASSED, Result.SKIPPED))).toBe(Result.SKIPPED) + expect(overall_result(e(Result.PASSED))).toBe(Result.PASSED) + expect(overall_result(e())).toBe(Result.PASSED) + }) +}) diff --git a/tools/tests/tester/overall_result.test.ts b/tools/tests/tester/overall_result.test.ts deleted file mode 100644 index 7e2bb2427..000000000 --- a/tools/tests/tester/overall_result.test.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* -* Copyright OpenSearch Contributors -* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -*/ - -import { overall_result } from '../../src/tester/helpers' -import { type Evaluation, Result } from '../../src/tester/types/eval.types' - -function e (...results: Result[]): Evaluation[] { - return results.map(result => ({ result })) -} -test('overall_result', () => { - expect(overall_result(e(Result.PASSED, Result.SKIPPED, Result.FAILED, Result.ERROR))).toBe(Result.ERROR) - expect(overall_result(e(Result.PASSED, Result.SKIPPED, Result.FAILED))).toBe(Result.FAILED) - expect(overall_result(e(Result.PASSED, Result.SKIPPED))).toBe(Result.SKIPPED) - expect(overall_result(e(Result.PASSED))).toBe(Result.PASSED) - expect(overall_result(e())).toBe(Result.PASSED) -})