# error formatting

Signed-off-by: Theo Truong <[email protected]>

tests/index_lifecycle.yaml

@@ -5,7 +5,7 @@ prologues: []
 epilogues:
   - path: /books,movies,games
     method: DELETE
-    ignore_errors: true
+    ignore_errors: false
 chapters:
   - synopsis: Create an index named `books` with mappings and settings.
     path: /{index}

tools/src/tester/ChapterEvaluator.ts

@@ -1,6 +1,7 @@
 import { type Chapter, type FullResponse } from './types/story.types'
 import { type ChapterEvaluation, type Evaluation, Result } from './types/eval.types'
 import { type ParsedOperation } from './types/spec.types'
+import YAML from 'yaml'
 
 export default class ChapterEvaluator {
   chapter: Chapter
@@ -28,15 +29,15 @@ export default class ChapterEvaluator {
         result: this.result
       }
     } catch (error) {
-      return { result: Result.ERROR, synopsis: this.chapter.synopsis, message: (error as Error).message }
+      return { result: Result.ERROR, synopsis: this.chapter.synopsis, message: error as string }
     }
   }
 
   #evaluate_parameters (operation: ParsedOperation): Record<string, Evaluation> {
     return Object.fromEntries(Object.entries(this.chapter.parameters ?? {}).map(([name, parameter]) => {
       const schema = operation.parameters[name]?.schema
       const evaluation = globalThis.schema_validator.validate(schema, parameter)
-      if (evaluation.result === Result.FAILED) this.result = Result.FAILED
+      if (evaluation.result !== Result.PASSED) this.result = Result.FAILED
       return [name, evaluation]
     }))
   }
@@ -46,7 +47,7 @@ export default class ChapterEvaluator {
     const schema = operation.requestBody?.content[this.chapter.requestBody?.content_type ?? '']?.schema
     if (schema == null) return { result: Result.FAILED, message: `Schema for "${this.chapter.requestBody.content_type}" request body not found.` }
     const evaluation = globalThis.schema_validator.validate(schema, this.chapter.requestBody?.payload ?? {})
-    if (evaluation.result === Result.FAILED) this.result = Result.FAILED
+    if (evaluation.result !== Result.PASSED) this.result = Result.FAILED
     return evaluation
   }
 
@@ -61,7 +62,7 @@ export default class ChapterEvaluator {
     const schema = operation.responses[response.status]?.content[response.content_type]?.schema
     if (schema == null) return { result: Result.FAILED, message: `Schema for "${response.status}: ${response.content_type}" response not found.` }
     const evaluation = globalThis.schema_validator.validate(schema, response.payload)
-    if (evaluation.result === Result.FAILED) this.result = Result.FAILED
+    if (evaluation.result !== Result.PASSED) this.result = Result.FAILED
     return evaluation
   }
 }

tools/src/tester/ChapterReader.ts

@@ -21,7 +21,7 @@ export default class ChapterReader {
       response.content_type = r.headers['content-type'].split(';')[0]
       response.payload = r.data
     }).catch(e => {
-      if (!ignore_errors) throw new Error(e.response.data.error as string)
+      if (!ignore_errors) throw new Error(e.response.data.error.reason as string)
       response.status = e.response.status
       response.content_type = e.response.headers['content-type'].split(';')[0]
       response.payload = e.response.data.error

tools/src/tester/StoryEvaluator.ts

@@ -50,8 +50,8 @@ export default class StoryEvaluator {
     for (const chapter of chapters) {
       const evaluator = new ChapterEvaluator(chapter)
       const evaluation = await evaluator.evaluate(skipped)
-      skipped = skipped || evaluation.result === Result.FAILED
-      if (evaluation.result === Result.FAILED) this.result = Result.FAILED
+      skipped = skipped || evaluation.result !== Result.PASSED
+      if (evaluation.result !== Result.PASSED) this.result = Result.FAILED
       evaluations.push(evaluation)
     }
 
@@ -65,6 +65,8 @@ export default class StoryEvaluator {
       }
       return { result: Result.OK }
     } catch (error) {
+      this.result = Result.ERROR
+      this.skipped = true
       return { result: Result.ERROR, message: (error as Error).message }
     }
   }

tools/src/tester/TestsRunner.ts

@@ -7,22 +7,31 @@ import fs from 'fs'
 import { type Story } from './types/story.types'
 import { read_yaml } from '../../helpers'
 
+declare global {
+  // eslint-disable-next-line no-var
+  var chapter_reader: ChapterReader
+  // eslint-disable-next-line no-var
+  var schema_validator: SchemaValidator
+  // eslint-disable-next-line no-var
+  var spec_parser: SpecParser
+}
+
 export default class TestsRunner {
   path: string // Path to a story file or a directory containing story files
 
   constructor (spec: OpenAPIV3.Document, path: string) {
     // TODO: Grab server URL from environment variable and add authentication.
-    globalThis.chapter_reader = new ChapterReader('http://localhost:9200')
-    globalThis.spec_parser = new SpecParser(spec)
-    globalThis.schema_validator = new SchemaValidator()
+    global.chapter_reader = new ChapterReader('http://localhost:9200')
+    global.spec_parser = new SpecParser(spec)
+    global.schema_validator = new SchemaValidator()
     this.path = path
   }
 
   async run (): Promise<void> {
     const story_files = this.#collect_story_files(`${process.cwd()}/${this.path}`, '', '')
     for (const story_file of story_files) {
       const evaluator = new StoryEvaluator(story_file)
-      await evaluator.evaluate()
+      console.log(await evaluator.evaluate())
     }
   }