From f695bcafe61e709556a658895b7cb7ddceab1ad9 Mon Sep 17 00:00:00 2001 From: Theo Truong Date: Sun, 19 May 2024 13:32:14 -0600 Subject: [PATCH] # Signed-off-by: Theo Truong --- json_schemas/story.schema.yaml | 4 +++ tools/src/tester/ChapterEvaluator.ts | 15 ++++++++ tools/src/tester/SchemaValidator.ts | 2 +- tools/src/tester/StoryEvaluator.ts | 36 +++++++++++++++++-- .../tester/SupplementalChapterEvaluator.ts | 13 +++++++ tools/src/tester/TestsRunner.ts | 18 ++++++---- tools/src/tester/types/eval.types.ts | 22 ++++++++---- tools/src/tester/types/story.types.ts | 4 +++ 8 files changed, 99 insertions(+), 15 deletions(-) create mode 100644 tools/src/tester/ChapterEvaluator.ts create mode 100644 tools/src/tester/SupplementalChapterEvaluator.ts diff --git a/json_schemas/story.schema.yaml b/json_schemas/story.schema.yaml index 582fb0498..c1b627722 100644 --- a/json_schemas/story.schema.yaml +++ b/json_schemas/story.schema.yaml @@ -4,6 +4,10 @@ type: object properties: $schema: type: string + skipped: + type: boolean + description: If true, the story will be skipped. + default: false description: type: string prologues: diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts new file mode 100644 index 000000000..aad276be0 --- /dev/null +++ b/tools/src/tester/ChapterEvaluator.ts @@ -0,0 +1,15 @@ +import { type Chapter } from './types/story.types' +import { type ChapterEvaluation, Result } from './types/eval.types' + +export default class ChapterEvaluator { + chapter: Chapter + + constructor (chapter: Chapter) { + this.chapter = chapter + } + + evaluate (skipped: boolean): ChapterEvaluation { + const synopsis = this.chapter.synopsis + if (skipped) return { result: Result.SKIPPED, synopsis } + } +} diff --git a/tools/src/tester/SchemaValidator.ts b/tools/src/tester/SchemaValidator.ts index d58ff81ba..f80014746 100644 --- a/tools/src/tester/SchemaValidator.ts +++ b/tools/src/tester/SchemaValidator.ts @@ -14,7 +14,7 @@ export default class SchemaValidator { const validate = this.ajv.compile(schema) const valid = validate(data) return { - result: valid ? 'PASS' : 'FAIL', + result: valid ? 'PASSED' : 'FAILED', message: valid ? undefined : this.ajv.errorsText(validate.errors) } } diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index 7020422d3..b0bb61e68 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -1,5 +1,7 @@ import { type Chapter, type Story, type SupplementalChapter } from './types/story.types' -import { type ChapterEvaluation, type Evaluation, type StoryEvaluation } from './types/eval.types' +import { type ChapterEvaluation, type Evaluation, Result, type StoryEvaluation } from './types/eval.types' +import ChapterEvaluator from './ChapterEvaluator' +import SupplementalChapterEvaluator from './SupplementalChapterEvaluator' export interface StoryFile { display_path: string @@ -11,6 +13,8 @@ export default class StoryEvaluator { story: Story display_path: string full_path: string + skipped: boolean = false + result: Result = Result.PASSED constructor (story_file: StoryFile) { this.story = story_file.story @@ -19,21 +23,49 @@ export default class StoryEvaluator { } evaluate (): StoryEvaluation { + if (this.story.skipped) { + return { + result: Result.SKIPPED, + display_path: this.display_path, + full_path: this.full_path, + description: this.story.description + } + } return { display_path: this.display_path, full_path: this.full_path, description: this.story.description, prologues: this.#evaluate_supplemental_chapters(this.story.prologues), chapters: this.#evaluate_chapters(this.story.chapters), - epilogues: this.#evaluate_supplemental_chapters(this.story.epilogues) + epilogues: this.#evaluate_supplemental_chapters(this.story.epilogues), + result: this.result } } #evaluate_chapters (chapters: Chapter[]): ChapterEvaluation[] { + let skipped: boolean = this.skipped + if (skipped) return [] + return chapters.map(chapter => { + const evaluator = new ChapterEvaluator(chapter) + const evaluation = evaluator.evaluate(skipped) + skipped = skipped || evaluation.result === Result.FAILED + if (evaluation.result === Result.FAILED) this.result = Result.FAILED + return evaluation + }) } #evaluate_supplemental_chapters (chapters: SupplementalChapter[]): Evaluation { + chapters.forEach(chapter => { + const evaluator = new SupplementalChapterEvaluator(chapter) + const evaluation = evaluator.evaluate() + if (evaluation.result === Result.ERROR) { + this.result = Result.ERROR + this.skipped = true + return evaluation + } + }) + return { result: Result.OK } } } diff --git a/tools/src/tester/SupplementalChapterEvaluator.ts b/tools/src/tester/SupplementalChapterEvaluator.ts new file mode 100644 index 000000000..2ad61e9f0 --- /dev/null +++ b/tools/src/tester/SupplementalChapterEvaluator.ts @@ -0,0 +1,13 @@ +import { type SupplementalChapter } from './types/story.types' +import { type Evaluation, Result } from './types/eval.types' + +export default class SupplementalChapterEvaluator { + chapter: SupplementalChapter + + constructor (chapter: SupplementalChapter) { + this.chapter = chapter + } + + evaluate (): Evaluation { + } +} diff --git a/tools/src/tester/TestsRunner.ts b/tools/src/tester/TestsRunner.ts index 4454a652c..0c99ca5cd 100644 --- a/tools/src/tester/TestsRunner.ts +++ b/tools/src/tester/TestsRunner.ts @@ -7,17 +7,23 @@ import fs from 'fs' import { type Story } from './types/story.types' import { read_yaml } from '../../helpers' +declare global { + // eslint-disable-next-line no-var + var chapter_reader: ChapterReader + // eslint-disable-next-line no-var + var schema_validator: SchemaValidator + // eslint-disable-next-line no-var + var spec_parser: SpecParser +} + export default class TestsRunner { - schema_validator: SchemaValidator - spec_parser: SpecParser - chapter_reader: ChapterReader path: string // Path to a story file or a directory containing story files constructor (spec: OpenAPIV3.Document, path: string) { // TODO: Grab server URL from environment variable and add authentication. - this.chapter_reader = new ChapterReader('http://localhost:9200') - this.spec_parser = new SpecParser(spec) - this.schema_validator = new SchemaValidator() + globalThis.chapter_reader = new ChapterReader('http://localhost:9200') + globalThis.spec_parser = new SpecParser(spec) + globalThis.schema_validator = new SchemaValidator() this.path = path } diff --git a/tools/src/tester/types/eval.types.ts b/tools/src/tester/types/eval.types.ts index 1b1dac5f7..da464e8e9 100644 --- a/tools/src/tester/types/eval.types.ts +++ b/tools/src/tester/types/eval.types.ts @@ -1,28 +1,38 @@ export type LibraryEvaluation = StoryEvaluation[] export interface StoryEvaluation { + result: Result display_path: string full_path: string description: string - chapters: ChapterEvaluation[] - epilogues: Evaluation - prologues: Evaluation + chapters?: ChapterEvaluation[] + epilogues?: Evaluation + prologues?: Evaluation } export interface ChapterEvaluation { + result: Result synopsis: string - request: { + request?: { operation: Evaluation parameters?: Record requestBody?: Evaluation } - response: { + response?: { status: Evaluation payload: Evaluation } } export interface Evaluation { - result: 'PASS' | 'FAIL' | 'SKIPPED' + result: Result message?: string } + +export enum Result { + PASSED = 'PASSED', + FAILED = 'FAILED', + SKIPPED = 'SKIPPED', + ERROR = 'ERROR', + OK = 'OK' +} diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts index 72505e363..d2da4b06b 100644 --- a/tools/src/tester/types/story.types.ts +++ b/tools/src/tester/types/story.types.ts @@ -48,6 +48,10 @@ export type ReadChapter = Chapter & { export interface Story { $schema?: string; + /** + * If true, the story will be skipped. + */ + skipped?: boolean; description: string; prologues: SupplementalChapter[]; epilogues: SupplementalChapter[];