From 382883848467c7f2edef81c074446c73e88bb80f Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Mon, 12 Aug 2024 13:03:20 -0400 Subject: [PATCH] Catch response deserialization errors. (#478) * Catch unexpected errors. Signed-off-by: dblock * Eat the error instead of catching it. Signed-off-by: dblock * Distinguish errors and errors. Signed-off-by: dblock --------- Signed-off-by: dblock --- tools/src/tester/ChapterEvaluator.ts | 10 +++-- tools/src/tester/ChapterReader.ts | 18 ++++---- .../fixtures/evals/error/chapter_error.yaml | 1 - .../fixtures/evals/error/prologue_error.yaml | 1 - .../fixtures/stories/failed/not_found.yaml | 1 - .../tests/tester/integ/StoryEvaluator.test.ts | 44 +++++++++++++++++++ 6 files changed, 59 insertions(+), 16 deletions(-) diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts index 0be147f4..ae73f90a 100644 --- a/tools/src/tester/ChapterEvaluator.ts +++ b/tools/src/tester/ChapterEvaluator.ts @@ -122,15 +122,17 @@ export default class ChapterEvaluator { #evaluate_status(chapter: Chapter, response: ActualResponse): Evaluation { const expected_status = chapter.response?.status ?? 200 - if (response.status === expected_status) return { result: Result.PASSED } + if (response.status === expected_status && response.error === undefined) return { result: Result.PASSED } - const result: Evaluation = { + let result: Evaluation = { result: Result.ERROR, message: _.join(_.compact([ - `Expected status ${expected_status}, but received ${response.status}: ${response.content_type}.`, + expected_status == response.status ? + `Received ${response.status ?? 'none'}: ${response.content_type ?? 'unknown'}.` : + `Expected status ${expected_status}, but received ${response.status ?? 'none'}: ${response.content_type ?? 'unknown'}.`, response.message ]), ' ') - }; + } if (response.error !== undefined) { result.error = response.error as Error diff --git a/tools/src/tester/ChapterReader.ts b/tools/src/tester/ChapterReader.ts index ae26e992..26dd0275 100644 --- a/tools/src/tester/ChapterReader.ts +++ b/tools/src/tester/ChapterReader.ts @@ -56,16 +56,16 @@ export default class ChapterReader { }).catch(e => { if (e.response == null) { this.logger.info(`<= ERROR: ${e}`) - throw e + response.message = e.message + response.error = e + } else { + response.status = e.response.status + response.content_type = e.response.headers['content-type']?.split(';')[0] + const payload = this.#deserialize_payload(e.response.data, response.content_type) + if (payload !== undefined) response.payload = payload.error + response.message = payload.error?.reason ?? e.response.statusText + this.logger.info(`<= ${response.status} (${response.content_type}) | ${response.payload !== undefined ? to_json(response.payload) : response.message}`) } - response.status = e.response.status - response.content_type = e.response.headers['content-type']?.split(';')[0] - const payload = this.#deserialize_payload(e.response.data, response.content_type) - if (payload !== undefined) response.payload = payload.error - response.message = payload.error?.reason ?? e.response.statusText - response.error = e - - this.logger.info(`<= ${response.status} (${response.content_type}) | ${response.payload !== undefined ? to_json(response.payload) : response.message}`) }) return response as ActualResponse } diff --git a/tools/tests/tester/fixtures/evals/error/chapter_error.yaml b/tools/tests/tester/fixtures/evals/error/chapter_error.yaml index 450d9c5a..fab1f0cc 100644 --- a/tools/tests/tester/fixtures/evals/error/chapter_error.yaml +++ b/tools/tests/tester/fixtures/evals/error/chapter_error.yaml @@ -33,7 +33,6 @@ chapters: result: ERROR message: 'Expected status 200, but received 404: application/json. no such index [undefined]' - error: Request failed with status code 404 payload_body: result: SKIPPED payload_schema: diff --git a/tools/tests/tester/fixtures/evals/error/prologue_error.yaml b/tools/tests/tester/fixtures/evals/error/prologue_error.yaml index 5ff63bb8..d91beb82 100644 --- a/tools/tests/tester/fixtures/evals/error/prologue_error.yaml +++ b/tools/tests/tester/fixtures/evals/error/prologue_error.yaml @@ -16,7 +16,6 @@ prologues: overall: result: ERROR message: no such index [does_not_exists] - error: Request failed with status code 404 chapters: - title: This chapter be skipped. diff --git a/tools/tests/tester/fixtures/stories/failed/not_found.yaml b/tools/tests/tester/fixtures/stories/failed/not_found.yaml index 0b61b03a..2bedb9c6 100644 --- a/tools/tests/tester/fixtures/stories/failed/not_found.yaml +++ b/tools/tests/tester/fixtures/stories/failed/not_found.yaml @@ -29,4 +29,3 @@ chapters: index: movies response: status: 404 - diff --git a/tools/tests/tester/integ/StoryEvaluator.test.ts b/tools/tests/tester/integ/StoryEvaluator.test.ts index 7abc9dbf..ac5fc60e 100644 --- a/tools/tests/tester/integ/StoryEvaluator.test.ts +++ b/tools/tests/tester/integ/StoryEvaluator.test.ts @@ -7,6 +7,7 @@ * compatible open source license. */ +import { Result } from 'tester/types/eval.types' import { construct_tester_components, load_actual_evaluation, load_expected_evaluation } from '../helpers' const { story_evaluator, opensearch_http_client } = construct_tester_components('tools/tests/tester/fixtures/specs/excerpt.yaml') @@ -16,6 +17,10 @@ beforeAll(async () => { expect(info.version).toBeDefined() }) +afterEach(() => { + jest.resetAllMocks() +}) + test('passed', async () => { const actual = await load_actual_evaluation(story_evaluator, 'passed') const expected = load_expected_evaluation('passed') @@ -57,3 +62,42 @@ test('skipped/semver', async () => { const expected = load_expected_evaluation('skipped/semver') expect(actual).toEqual(expected) }) + +test('with an unexpected error deserializing data', async () => { + opensearch_http_client.request = jest.fn().mockRejectedValue(new Error('This was unexpected.')) + const actual = await load_actual_evaluation(story_evaluator, 'passed') + expect(actual.result).toEqual(Result.ERROR) + expect(actual.chapters && actual.chapters[0]).toEqual({ + title: "This PUT /{index} chapter should pass.", + path: 'PUT /{index}', + overall: { + result: Result.ERROR + }, + request: { + parameters: { + index: { + result: Result.PASSED + }, + }, + request: { + result: Result.PASSED + } + }, + response: { + output_values: { + result: Result.SKIPPED + }, + payload_body: { + result: Result.SKIPPED + }, + payload_schema: { + result: Result.SKIPPED + }, + status: { + error: 'This was unexpected.', + message: 'Expected status 200, but received none: unknown. This was unexpected.', + result: Result.ERROR + } + } + }) +}) \ No newline at end of file