# ChapterEvaluator.ts

Signed-off-by: Theo Truong <[email protected]>

json_schemas/story.schema.yaml

@@ -93,6 +93,8 @@ definitions:
     properties:
       status:
         type: integer
+        description: The expected HTTP status code. Default to 200.
+        default: 200
       content_type:
         type: string
       payload:

tools/src/tester/ChapterEvaluator.ts

@@ -1,15 +1,67 @@
-import { type Chapter } from './types/story.types'
-import { type ChapterEvaluation, Result } from './types/eval.types'
+import { type Chapter, type FullResponse } from './types/story.types'
+import { type ChapterEvaluation, type Evaluation, Result } from './types/eval.types'
+import { type ParsedOperation } from './types/spec.types'
 
 export default class ChapterEvaluator {
   chapter: Chapter
+  result: Result = Result.PASSED
 
   constructor (chapter: Chapter) {
     this.chapter = chapter
   }
 
-  evaluate (skipped: boolean): ChapterEvaluation {
-    const synopsis = this.chapter.synopsis
-    if (skipped) return { result: Result.SKIPPED, synopsis }
+  async evaluate (skipped: boolean): Promise<ChapterEvaluation> {
+    try {
+      if (skipped) return { result: Result.SKIPPED, synopsis: this.chapter.synopsis }
+      const operation = globalThis.spec_parser.locate_operation(this.chapter)
+      const response = await globalThis.chapter_reader.read(this.chapter, true)
+      return {
+        synopsis: this.chapter.synopsis,
+        request: {
+          parameters: this.#evaluate_parameters(operation),
+          requestBody: this.#evaluate_request_body(operation)
+        },
+        response: {
+          status: this.#evaluate_status(response),
+          payload: this.#evaluate_payload(operation, response)
+        },
+        result: this.result
+      }
+    } catch (error) {
+      return { result: Result.ERROR, synopsis: this.chapter.synopsis, message: (error as Error).message }
+    }
+  }
+
+  #evaluate_parameters (operation: ParsedOperation): Record<string, Evaluation> {
+    return Object.fromEntries(Object.entries(this.chapter.parameters ?? {}).map(([name, parameter]) => {
+      const schema = operation.parameters[name]?.schema
+      const evaluation = globalThis.schema_validator.validate(schema, parameter)
+      if (evaluation.result === Result.FAILED) this.result = Result.FAILED
+      return [name, evaluation]
+    }))
+  }
+
+  #evaluate_request_body (operation: ParsedOperation): Evaluation | undefined {
+    if (!this.chapter.requestBody) return undefined
+    const schema = operation.requestBody?.content[this.chapter.requestBody?.content_type ?? '']?.schema
+    if (schema == null) return { result: Result.FAILED, message: `Schema for "${this.chapter.requestBody.content_type}" request body not found.` }
+    const evaluation = globalThis.schema_validator.validate(schema, this.chapter.requestBody?.payload ?? {})
+    if (evaluation.result === Result.FAILED) this.result = Result.FAILED
+    return evaluation
+  }
+
+  #evaluate_status (response: FullResponse): Evaluation {
+    const expected_status = this.chapter.response?.status ?? 200
+    return response.status === expected_status
+      ? { result: Result.PASSED }
+      : { result: Result.FAILED, message: `Expected status ${expected_status}, but received ${response.status}` }
+  }
+
+  #evaluate_payload (operation: ParsedOperation, response: FullResponse): Evaluation {
+    const schema = operation.responses[response.status]?.content[response.content_type]?.schema
+    if (schema == null) return { result: Result.FAILED, message: `Schema for "${response.status}: ${response.content_type}" response not found.` }
+    const evaluation = globalThis.schema_validator.validate(schema, response.payload)
+    if (evaluation.result === Result.FAILED) this.result = Result.FAILED
+    return evaluation
   }
 }

tools/src/tester/SchemaValidator.ts

@@ -1,7 +1,7 @@
 import AJV from 'ajv'
 import addFormats from 'ajv-formats'
 import { type OpenAPIV3 } from 'openapi-types'
-import { type Evaluation } from './types/eval.types'
+import { type Evaluation, Result } from './types/eval.types'
 
 export default class SchemaValidator {
   private readonly ajv: AJV
@@ -14,7 +14,7 @@ export default class SchemaValidator {
     const validate = this.ajv.compile(schema)
     const valid = validate(data)
     return {
-      result: valid ? 'PASSED' : 'FAILED',
+      result: valid ? Result.PASSED : Result.FAILED,
       message: valid ? undefined : this.ajv.errorsText(validate.errors)
     }
   }

tools/src/tester/SpecParser.ts

@@ -18,7 +18,7 @@ export default class SpecParser {
     const cache_key = path + method
     if (this.cached_operations[cache_key] != null) return this.cached_operations[cache_key]
     const operation = this.spec.paths[path]?.[method]
-    if (operation == null) throw new Error(`Operation "${method} ${path}" not found in the spec.`)
+    if (operation == null) throw new Error(`Operation "${method.toUpperCase()} ${path}" not found in the spec.`)
     this.#deref(operation)
     const parameters = _.keyBy(operation.parameters ?? [], 'name')
     // eslint-disable-next-line @typescript-eslint/consistent-type-assertions

tools/src/tester/types/eval.types.ts

@@ -13,8 +13,8 @@ export interface StoryEvaluation {
 export interface ChapterEvaluation {
   result: Result
   synopsis: string
+  message?: string
   request?: {
-    operation: Evaluation
     parameters?: Record<string, Evaluation>
     requestBody?: Evaluation
   }

tools/src/tester/types/story.types.ts

@@ -85,6 +85,9 @@ export interface RequestBody {
  * via the `definition` "Response".
  */
 export interface Response {
+  /**
+   * The expected HTTP status code. Default to 200.
+   */
   status: number;
   content_type?: string;
   payload?: Payload;