Added tools/linter along with jest tests.

tools/README.md

@@ -0,0 +1,25 @@
+# OpenSearch OpenAPI Tools
+This folder contains tools for the repo:
+- Merger: Merges multiple OpenAPI files into one
+- Linter: Validates files in the spec folder
+
+## Setup
+1. Install [Node.js](https://nodejs.org/en/learn/getting-started/how-to-install-nodejs)
+2. Run `npm install` in the `tools` folder
+
+## Merger
+The merger tool merges the multi-file OpenSearch spec into a single file for programmatic use. It takes 2 values as input:
+- The path to the root folder of the multi-file spec
+- The path to the output file
+Example:
+```bash
+npm run merge -- ../spec ../build/opensearch-openapi.latest.yaml
+```
+
+## Linter
+The linter tool validates the OpenSearch spec files in the `spec` folder:
+```bash
+npm run lint
+```
+It will print out all the errors and warnings in the spec files. This tool in still in development, and it will be integrated into the CI/CD pipeline and run automatically with every PR.
+```

tools/helpers.ts

@@ -1,8 +1,42 @@
+import fs from "fs";
+import YAML from "yaml";
+import _ from "lodash";
+
 export function resolve(ref: string, root: Record<string, any>) {
     const paths = ref.replace('#/', '').split('/');
     for(const p of paths) {
         root = root[p];
         if(root === undefined) break;
     }
     return root;
+}
+
+export function sortByKey(obj: Record<string, any>, priorities: string[] = []) {
+    const orders = _.fromPairs(priorities.map((k, i) => [k, i+1]));
+    const sorted = _.entries(obj).sort((a,b) => {
+        const order_a = orders[a[0]];
+        const order_b = orders[b[0]];
+        if(order_a && order_b) return order_a - order_b;
+        if(order_a) return 1;
+        if(order_b) return -1;
+        return a[0].localeCompare(b[0]);
+    });
+    sorted.forEach(([k, v]) => {
+        delete obj[k];
+        obj[k] = v;
+    });
+}
+
+export function write2file(file_path: string, content: Record<string, any>): void {
+    fs.writeFileSync(file_path, quoteRefs(YAML.stringify(content, {lineWidth: 0, singleQuote: true})));
+}
+
+function quoteRefs(str: string): string {
+    return str.split('\n').map((line) => {
+        if(line.includes('$ref')) {
+            const [key, value] = line.split(': ');
+            if(!value.startsWith("'")) line = `${key}: '${value}'`;
+        }
+        return line
+    }).join('\n');
 }

tools/jest.config.js

@@ -0,0 +1,5 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+};

tools/linter/PathRefsValidator.ts

@@ -0,0 +1,76 @@
+import {ValidationError} from "../types";
+import RootFile from "./components/RootFile";
+import NamespacesFolder from "./components/NamespacesFolder";
+
+export default class PathRefsValidator {
+    root_file: RootFile;
+    namespaces_folder: NamespacesFolder;
+
+    referenced_paths: Record<string, Set<string>> = {}; // file -> paths
+    available_paths: Record<string, Set<string>> = {}; // file -> paths
+
+    constructor(root_file: RootFile, namespaces_folder: NamespacesFolder) {
+        this.root_file = root_file;
+        this.namespaces_folder = namespaces_folder;
+        this.#build_referenced_paths();
+        this.#build_available_paths();
+    }
+
+    #build_referenced_paths() {
+        for (const [path, spec] of Object.entries(this.root_file.spec().paths)) {
+            const ref = spec!.$ref!;
+            const file = ref.split('#')[0];
+            if(!this.referenced_paths[file]) this.referenced_paths[file] = new Set();
+            this.referenced_paths[file].add(path);
+        }
+    }
+
+    #build_available_paths() {
+        for (const file of this.namespaces_folder.files) {
+            this.available_paths[file.file] = new Set(Object.keys(file.spec().paths || {}));
+        }
+    }
+
+    validate(): ValidationError[] {
+        return [
+            ...this.validate_unresolved_refs(),
+            ...this.validate_unreferenced_paths(),
+        ];
+    }
+
+    validate_unresolved_refs(): ValidationError[] {
+        return Object.entries(this.referenced_paths).flatMap(([ref_file, ref_paths]) => {
+            const available = this.available_paths[ref_file];
+            if(!available) return {
+                file: this.root_file.file,
+                location: `Paths: ${[...ref_paths].join(' , ')}`,
+                message: `Unresolved path reference: Namespace file ${ref_file} does not exist.`,
+            };
+
+            return Array.from(ref_paths).map((path) => {
+                if(!available.has(path)) return {
+                    file: this.root_file.file,
+                    location: `Path: ${path}`,
+                    message: `Unresolved path reference: Path ${path} does not exist in namespace file ${ref_file}`,
+                };
+            }).filter((e) => e) as ValidationError[];
+        });
+    }
+
+    validate_unreferenced_paths(): ValidationError[] {
+        return Object.entries(this.available_paths).flatMap(([ns_file, ns_paths]) => {
+            const referenced = this.referenced_paths[ns_file];
+            if(!referenced) return {
+                file: ns_file,
+                message: `Unreferenced paths: No paths are referenced in the root file.`,
+            };
+            return Array.from(ns_paths).map((path) => {
+                if(!referenced || !referenced.has(path)) return {
+                    file: ns_file,
+                    location: `Path: ${path}`,
+                    message: `Unreferenced path: Path ${path} is not referenced in the root file.`,
+                };
+            }).filter((e) => e) as ValidationError[];
+        });
+    }
+}

tools/linter/SchemaRefsValidator.ts

@@ -0,0 +1,100 @@
+import NamespacesFolder from "./components/NamespacesFolder";
+import SchemasFolder from "./components/SchemasFolder";
+import {ValidationError} from "../types";
+
+export default class SchemaRefsValidator {
+    namespaces_folder: NamespacesFolder;
+    schemas_folder: SchemasFolder;
+
+    referenced_schemas: Record<string, Set<string>> = {}; // file -> schemas
+    available_schemas: Record<string, Set<string>> = {}; // file -> schemas
+
+    constructor(namespaces_folder: NamespacesFolder, schemas_folder: SchemasFolder) {
+        this.namespaces_folder = namespaces_folder;
+        this.schemas_folder = schemas_folder;
+        this.#find_refs_in_namespaces_folder();
+        this.#find_refs_in_schemas_folder();
+        this.#build_available_schemas();
+    }
+
+    #find_refs_in_namespaces_folder() {
+        const search = (obj: Record<string, any>) => {
+            const ref = obj.$ref;
+            if(ref) {
+                const file = ref.split('#')[0].replace("../", "");
+                const name = ref.split('/').pop();
+                if(!this.referenced_schemas[file]) this.referenced_schemas[file] = new Set();
+                this.referenced_schemas[file].add(name);
+            }
+            for (const key in obj)
+                if(typeof obj[key] === 'object') search(obj[key]);
+        };
+
+        this.namespaces_folder.files.forEach((file) => { search(file.spec().components || {}) });
+    }
+
+    #find_refs_in_schemas_folder() {
+        const search = (obj: Record<string, any>, ref_file: string) => {
+            const ref = obj.$ref;
+            if(ref) {
+                const file = ref.startsWith('#') ? ref_file : ref.split('#')[0].replace("./", "schemas/");
+                const name = ref.split('/').pop();
+                if(!this.referenced_schemas[file]) this.referenced_schemas[file] = new Set();
+                this.referenced_schemas[file].add(name);
+            }
+            for (const key in obj)
+                if(typeof obj[key] === 'object') search(obj[key], ref_file);
+        }
+
+        this.schemas_folder.files.forEach((file) => { search(file.spec().components?.schemas  || {}, file.file) });
+    }
+
+    #build_available_schemas() {
+        this.schemas_folder.files.forEach((file) => {
+            this.available_schemas[file.file] = new Set(Object.keys(file.spec().components?.schemas || {}));
+        });
+    }
+
+    validate(): ValidationError[] {
+        return [
+            ...this.validate_unresolved_refs(),
+            ...this.validate_unreferenced_schemas(),
+        ];
+    }
+
+    validate_unresolved_refs(): ValidationError[] {
+        return Object.entries(this.referenced_schemas).flatMap(([ref_file, ref_schemas]) => {
+            const available = this.available_schemas[ref_file];
+            if(!available) return {
+                file: this.namespaces_folder.file,
+                message: `Unresolved schema reference: Schema file ${ref_file} is referenced but does not exist.`,
+            };
+
+            return Array.from(ref_schemas).map((schema) => {
+                if(!available.has(schema)) return {
+                    file: ref_file,
+                    location: `#/components/schemas/${schema}`,
+                    message: `Unresolved schema reference: Schema ${schema} is referenced but does not exist.`,
+                };
+            }).filter((e) => e) as ValidationError[];
+        });
+    }
+
+    validate_unreferenced_schemas(): ValidationError[] {
+        return Object.entries(this.available_schemas).flatMap(([file, schemas]) => {
+            const referenced = this.referenced_schemas[file];
+            if(!referenced) return {
+                file: file,
+                message: `Unreferenced schema: Schema file ${file} is not referenced anywhere.`,
+            };
+
+            return Array.from(schemas).map((schema) => {
+                if(!referenced.has(schema)) return {
+                    file: file,
+                    location: `#/components/schemas/${schema}`,
+                    message: `Unreferenced schema: Schema ${schema} is not referenced anywhere.`,
+                };
+            }).filter((e) => e) as ValidationError[];
+        });
+    }
+}

tools/linter/SpecValidator.ts

@@ -0,0 +1,36 @@
+import SchemasFolder from "./components/SchemasFolder";
+import NamespacesFolder from "./components/NamespacesFolder";
+import RootFile from "./components/RootFile";
+import {ValidationError} from "../types";
+import PathRefsValidator from "./PathRefsValidator";
+import SchemaRefsValidator from "./SchemaRefsValidator";
+
+export default class SpecValidator {
+    root_file: RootFile;
+    namespaces_folder: NamespacesFolder;
+    schemas_folder: SchemasFolder;
+    path_refs_validator: PathRefsValidator;
+    schema_refs_validator: SchemaRefsValidator;
+
+    constructor(root_folder: string) {
+        this.root_file = new RootFile(`${root_folder}/opensearch-openapi.yaml`);
+        this.namespaces_folder = new NamespacesFolder(`${root_folder}/namespaces`);
+        this.schemas_folder = new SchemasFolder(`${root_folder}/schemas`);
+        this.path_refs_validator = new PathRefsValidator(this.root_file, this.namespaces_folder);
+        this.schema_refs_validator = new SchemaRefsValidator(this.namespaces_folder, this.schemas_folder);
+    }
+
+    validate(): ValidationError[] {
+        const component_errors = [
+            ...this.root_file.validate(),
+            ...this.namespaces_folder.validate(),
+            ...this.schemas_folder.validate(),
+        ];
+        if(component_errors.length) return component_errors;
+
+        return [
+            ...this.path_refs_validator.validate(),
+            ...this.schema_refs_validator.validate()
+        ]
+    }
+}

tools/linter/components/NamespaceFile.ts

@@ -0,0 +1,99 @@
+import {OpenAPIV3} from "openapi-types";
+import {OperationSpec, ValidationError} from "../../types";
+import OperationGroup from "./OperationGroup";
+import _ from "lodash";
+import Operation from "./Operation";
+import {resolve} from "../../helpers"
+import FileValidator from "./base/FileValidator";
+
+const HTTP_METHODS = ['get', 'put', 'post', 'delete', 'options', 'head', 'patch', 'trace'];
+const NAME_REGEX = /^[a-z]+[a-z_]*[a-z]+$/;
+
+export default class NamespaceFile extends FileValidator {
+    namespace: string;
+    _operation_groups: OperationGroup[] | undefined;
+    _refs: Set<string> | undefined;
+
+    constructor(file_path: string) {
+        super(file_path);
+        this.namespace = file_path.split('/').slice(-1)[0].replace('.yaml', '');
+    }
+
+    validate_file(): ValidationError[] {
+        const name_error = this.validate_name();
+        if(name_error) return [name_error];
+        const group_errors = this.operation_groups().flatMap((group) => group.validate());
+        if(group_errors.length > 0) return group_errors;
+
+        return [
+            this.validate_schemas(),
+            ...this.validate_unresolved_refs(),
+            ...this.validate_unused_refs(),
+            ...this.validate_parameter_refs(),
+        ].filter((e) => e) as ValidationError[]
+    }
+
+    operation_groups(): OperationGroup[] {
+        if(this._operation_groups) return this._operation_groups;
+        const ops: Operation[] = _.entries(this.spec().paths).flatMap(([path, ops]) => {
+            return _.entries(_.pick(ops, HTTP_METHODS)).map(([verb, op]) => {
+                return new Operation(this.file, path, verb, op as OperationSpec);
+            });
+        });
+
+        return this._operation_groups = _.entries(_.groupBy(ops, (op) => op.group)).map(([group, ops]) => {
+            return new OperationGroup(this.file, group, ops);
+        });
+    }
+
+    refs(): Set<string> {
+        if(this._refs) return this._refs;
+        this._refs = new Set<string>();
+        const find_refs = (obj: Record<string, any>) => {
+            if(obj.$ref) this._refs!.add(obj.$ref);
+            _.values(obj).forEach((value) => { if(typeof value === 'object') find_refs(value); });
+        }
+        find_refs(this.spec().paths || {});
+        return this._refs;
+    }
+
+    validate_name(name = this.namespace): ValidationError | void {
+        if(name === '_core') return;
+        if(!name.match(NAME_REGEX))
+            return this.error(`Invalid namespace name '${name}'. Must match regex: ${NAME_REGEX.source}`, 'File Name');
+        return;
+    }
+
+    validate_schemas(): ValidationError | void {
+        if(this.spec().components?.schemas)
+            return this.error(`components/schemas is not allowed in namespace files`, '#/components/schemas');
+    }
+
+    validate_unresolved_refs(): ValidationError[] {
+        return Array.from(this.refs()).map((ref) => {
+            if(resolve(ref, this.spec()) === undefined) return this.error(`Unresolved reference: ${ref}`, ref);
+        }).filter((e) => e) as ValidationError[];
+    }
+
+    validate_unused_refs(): ValidationError[] {
+        return _.entries(this.spec().components || {}).flatMap(([type, collection]) => {
+            return _.keys(collection).map((name) => {
+                if (!this.refs().has(`#/components/${type}/${name}`))
+                    return this.error(`Unused ${type} component: ${name}`, `#/components/${type}/${name}`);
+            })
+        }).filter((e) => e) as ValidationError[];
+    }
+
+    validate_parameter_refs(): ValidationError[] {
+        const parameters = this.spec().components?.parameters as Record<string, OpenAPIV3.ParameterObject>
+        if(!parameters) return [];
+        return _.entries(parameters).map(([name, p]) => {
+            const group = name.split('::')[0];
+            const expected = `${group}::${p.in}.${p.name}`;
+            if(name !== expected)
+                return this.error(
+                    `Parameter component '${name}' must be named '${expected}' since it is a ${p.in} parameter named '${p.name}'.`,
+                    `#/components/parameters/#${name}`);
+        }).filter((e) => e) as ValidationError[];
+    }
+}