Support outputs on tests

tools/src/tester/ChapterEvaluator.ts

@@ -10,11 +10,12 @@
 import { type Chapter, type ActualResponse } from './types/story.types'
 import { type ChapterEvaluation, type Evaluation, Result } from './types/eval.types'
 import { type ParsedOperation } from './types/spec.types'
-import { extract_output_values, overall_result } from './helpers'
+import { overall_result } from './helpers'
 import type ChapterReader from './ChapterReader'
 import type OperationLocator from './OperationLocator'
 import type SchemaValidator from './SchemaValidator'
 import { type StoryOutputs } from './StoryOutputs'
+import { ChapterOutput } from './ChapterOutput'
 
 export default class ChapterEvaluator {
   private readonly _operation_locator: OperationLocator
@@ -36,7 +37,7 @@ export default class ChapterEvaluator {
     const request_body = this.#evaluate_request_body(chapter, operation)
     const status = this.#evaluate_status(chapter, response)
     const payload = status.result === Result.PASSED ? this.#evaluate_payload(response, operation) : { result: Result.SKIPPED }
-    const output_values = extract_output_values(response, chapter.output)
+    const output_values = ChapterOutput.extract_output_values(response, chapter.output)
     return {
       title: chapter.synopsis,
       overall: { result: overall_result(Object.values(params).concat([request_body, status, payload]).concat(output_values ? [output_values] : [])) },

tools/src/tester/ChapterOutput.ts

@@ -7,7 +7,9 @@
 * compatible open source license.
 */
 
-import { type Output } from './types/story.types'
+import { EvaluationWithOutput, Result } from './types/eval.types'
+import { ActualResponse, type Output } from './types/story.types'
+import _ from 'lodash'
 
 export class ChapterOutput {
   private outputs: Record<string, any>
@@ -33,4 +35,27 @@ export class ChapterOutput {
   static create_dummy_from_output (output: Output): ChapterOutput {
     return new ChapterOutput(output)
   }
+
+  static extract_output_values (response: ActualResponse, output?: Output): EvaluationWithOutput | undefined {
+    if (!output) return undefined
+  const chapter_output = new ChapterOutput({})
+  for (const [name, path] of Object.entries(output)) {
+    const [source, ...rest] = path.split('.')
+    const keys = rest.join('.')
+    let value: any
+    if (source === 'payload') {
+      if (response.payload === undefined) {
+        return { result: Result.ERROR, message: 'No payload found in response, but expected output: ' + path }
+      }
+      value = keys.length === 0 ? response.payload : _.get(response.payload, keys)
+      if (value === undefined) {
+        return { result: Result.ERROR, message: `Expected to find non undefined value at \`${path}\`.` }
+      }
+    } else {
+      return { result: Result.ERROR, message: 'Unknown output source: ' + source }
+    }
+    chapter_output.set_output(name, value)
+  }
+  return { result: Result.PASSED, output: chapter_output }
+  }
 }

tools/src/tester/SupplementalChapterEvaluator.ts

@@ -7,9 +7,10 @@
 * compatible open source license.
 */
 
+import { ChapterOutput } from "./ChapterOutput";
 import ChapterReader from "./ChapterReader";
 import { StoryOutputs } from "./StoryOutputs";
-import { extract_output_values, overall_result } from "./helpers";
+import { overall_result } from "./helpers";
 import { ChapterEvaluation, Result } from "./types/eval.types";
 import { SupplementalChapter } from "./types/story.types";
 
@@ -24,7 +25,7 @@ export default class SupplementalChapterEvaluator {
         const title = `${chapter.method} ${chapter.path}`
         const response = await this._chapter_reader.read(chapter, story_outputs)
         const status = chapter.status ?? [200, 201]
-        const output_values = extract_output_values(response)
+        const output_values = ChapterOutput.extract_output_values(response, chapter.output)
         let response_evaluation: ChapterEvaluation
         const passed_evaluation = { title, overall: { result: Result.PASSED } }
         if (status.includes(response.status)) {

tools/src/tester/helpers.ts

@@ -7,37 +7,11 @@
 * compatible open source license.
 */
 
-import { ChapterOutput } from './ChapterOutput'
-import { type Evaluation, Result, type EvaluationWithOutput } from './types/eval.types'
-import { type ActualResponse, type Output } from './types/story.types'
-import _ from 'lodash'
+import { type Evaluation, Result } from './types/eval.types'
 
 export function overall_result(evaluations: Evaluation[]): Result {
   if (evaluations.some(e => e.result === Result.ERROR)) return Result.ERROR
   if (evaluations.some(e => e.result === Result.FAILED)) return Result.FAILED
   if (evaluations.some(e => e.result === Result.SKIPPED)) return Result.SKIPPED
   return Result.PASSED
 }
-
-export function extract_output_values(response: ActualResponse, chapter_output?: Output): EvaluationWithOutput | undefined {
-  if (!chapter_output) return undefined
-  const output = new ChapterOutput({})
-  for (const [name, path] of Object.entries(chapter_output)) {
-    const [source, ...rest] = path.split('.')
-    const keys = rest.join('.')
-    let value: any
-    if (source === 'payload') {
-      if (response.payload === undefined) {
-        return { result: Result.ERROR, message: 'No payload found in response, but expected output: ' + path }
-      }
-      value = keys.length === 0 ? response.payload : _.get(response.payload, keys)
-      if (value === undefined) {
-        return { result: Result.ERROR, message: `Expected to find non undefined value at \`${path}\`.` }
-      }
-    } else {
-      return { result: Result.ERROR, message: 'Unknown output source: ' + source }
-    }
-    output.set_output(name, value)
-  }
-  return { result: Result.PASSED, output }
-}

tools/tests/tester/start.test.ts

@@ -10,7 +10,6 @@
 import { spawnSync } from 'child_process'
 import * as ansi from 'tester/Ansi'
 import * as path from 'path'
-import { extract_output_values } from '../../src/tester/helpers'
 import { type Chapter, type ChapterRequest, type Output, type RequestBody, type ActualResponse, Story } from 'tester/types/story.types'
 import { type EvaluationWithOutput, Result, ChapterEvaluation, StoryEvaluation } from 'tester/types/eval.types'
 import { ChapterOutput } from 'tester/ChapterOutput'
@@ -78,15 +77,15 @@ test('extract_output_values', () => {
     d: 'payload.a.arr[0].d',
     e: 'payload.a.arr[1].e'
   }
-  expect(extract_output_values(response, output1)).toEqual(passed_output({
+  expect(ChapterOutput.extract_output_values(response, output1)).toEqual(passed_output({
     c: 1,
     d: 2,
     e: 3
   }))
-  expect(extract_output_values(response, { x: 'payload' })).toEqual(
+  expect(ChapterOutput.extract_output_values(response, { x: 'payload' })).toEqual(
     passed_output({ x: response.payload })
   )
-  expect(extract_output_values(response, { x: 'payload.a.b.x[0]' })).toEqual({
+  expect(ChapterOutput.extract_output_values(response, { x: 'payload.a.b.x[0]' })).toEqual({
     result: Result.ERROR,
     message: 'Expected to find non undefined value at `payload.a.b.x[0]`.'
   })