issue #451 - Adding validation for the architecture only

cli/src/cli.spec.ts

@@ -161,6 +161,27 @@ describe('CLI Integration Tests', () => {
             done();
         });
     });
+
+    test('example validate command - fails when neither an architecture or a pattern is provided', (done) => {
+        const calmValidateCommand = 'calm validate';
+        exec(calmValidateCommand, (error, _stdout, stderr) => {
+            expect(error).not.toBeNull();
+            expect(stderr).toContain('error: one of the required options \'-p, --pattern <file>\' or \'-a, --architecture <file>\' was not specified');
+            done();
+        });
+    });
+
+    test('example validate command - validates an architecture only', (done) => {
+        const calmValidateArchitectureOnlyCommand = 'calm validate -a ../calm/samples/api-gateway-architecture.json';
+        exec(calmValidateArchitectureOnlyCommand, (error, stdout, _stderr) => {
+            const expectedFilePath = path.join(__dirname, '../test_fixtures/validate_architecture_only_output.json');
+            const expectedOutput = fs.readFileSync(expectedFilePath, 'utf-8');
+            expect(error).toBeNull();
+            expect(stdout).toContain(expectedOutput);
+            done();
+        });
+    });
+
 });
 
 

cli/src/index.ts

@@ -53,7 +53,7 @@ program
 program
     .command('validate')
     .description('Validate that an architecture conforms to a given CALM pattern.')
-    .requiredOption(PATTERN_OPTION, 'Path to the pattern file to use. May be a file path or a URL.')
+    .option(PATTERN_OPTION, 'Path to the pattern file to use. May be a file path or a URL.')
     .option(ARCHITECTURE_OPTION, 'Path to the architecture file to use. May be a file path or a URL.')
     .option(SCHEMAS_OPTION, 'Path to the directory containing the meta schemas to use.', CALM_META_SCHEMA_DIRECTORY)
     .option(STRICT_OPTION, 'When run in strict mode, the CLI will fail if any warnings are reported.', false)
@@ -65,6 +65,9 @@ program
     .option(OUTPUT_OPTION, 'Path location at which to output the generated file.')
     .option(VERBOSE_OPTION, 'Enable verbose logging.', false)
     .action(async (options) => {
+        if(!options.pattern && !options.architecture) {
+            program.error(`error: one of the required options '${PATTERN_OPTION}' or '${ARCHITECTURE_OPTION}' was not specified`);
+        }
         const outcome = await validate(options.architecture, options.pattern, options.schemaDirectory, options.verbose);
         const content = getFormattedOutput(outcome, options.format);
         writeOutputFile(options.output, content);

cli/test_fixtures/validate_architecture_only_output.json

@@ -0,0 +1,18 @@
+{
+    "jsonSchemaValidationOutputs": [],
+    "spectralSchemaValidationOutputs": [
+        {
+            "code": "architecture-has-no-placeholder-properties-numerical",
+            "severity": "warning",
+            "message": "Numerical placeholder (-1) detected in architecture.",
+            "path": "/nodes/2/interfaces/0/port",
+            "schemaPath": "",
+            "line_start": 32,
+            "line_end": 32,
+            "character_start": 18,
+            "character_end": 20
+        }
+    ],
+    "hasErrors": false,
+    "hasWarnings": true
+}

shared/src/commands/validate/validate.spec.ts

@@ -57,13 +57,18 @@ describe('validate-all', () => {
             fetchMock.restore();
         });
 
+        it('returns error when the the Pattern and the Architecture are undefined or an empty string', async () => {
+            await expect(validate('', undefined, metaSchemaLocation, debugDisabled))
+                .rejects
+                .toThrow();
+            expect(mockExit).toHaveBeenCalledWith(1);
+        });
 
         it('returns validation error when the JSON Schema pattern cannot be found in the input path', async () => {
             await expect(validate('../test_fixtures/api-gateway-implementation.json', 'thisFolderDoesNotExist/api-gateway.json', metaSchemaLocation, debugDisabled))
                 .rejects
                 .toThrow();
             expect(mockExit).toHaveBeenCalledWith(1);
-
         });
 
         it('returns validation error when the architecture file cannot be found in the input path', async () => {
@@ -495,6 +500,75 @@ describe('validate-all', () => {
                 .toHaveBeenCalledWith(1);
         });
     });
+
+    describe('validate - architecture only', () => {
+
+        let mockExit;
+
+        beforeEach(() => {
+            mockRunFunction.mockReturnValue([]);
+            mockExit = jest.spyOn(process, 'exit')
+                .mockImplementation((code) => {
+                    if (code != 0) {
+                        throw new Error('Expected successful run, code was nonzero: ' + code);
+                    }
+                    return undefined as never;
+                });
+        });
+
+        afterEach(() => {
+            fetchMock.restore();
+        });
+
+        it('exits with non zero exit code when the architecture cannot be found', async () => {
+            fetchMock.mock('http://exist/api-gateway-implementation.json', 404);
+
+            await expect(validateAndExitConditionally('http://exist/api-gateway-implementation.json', '', metaSchemaLocation, debugDisabled))
+                .rejects
+                .toThrow();
+
+            expect(mockExit)
+                .toHaveBeenCalledWith(1);
+        });
+
+        it('exits with non zero exit code when the architecture does not pass all the spectral validations ', async () => {
+            const expectedSpectralOutput: ISpectralDiagnostic[] = [
+                {
+                    code: 'example-error',
+                    message: 'Example error',
+                    severity: 0,
+                    path: ['/nodes'],
+                    range: { start: { line: 1, character: 1 }, end: { line: 2, character: 1 } }
+                }
+            ];
+
+            mockRunFunction.mockReturnValue(expectedSpectralOutput);
+
+            const apiGateway = readFileSync(path.resolve(__dirname, '../../../test_fixtures/api-gateway-implementation.json'), 'utf8');
+            fetchMock.mock('http://exist/api-gateway-implementation.json', apiGateway);
+
+            await expect(validateAndExitConditionally('http://exist/api-gateway-implementation.json', '', metaSchemaLocation, debugDisabled))
+                .rejects
+                .toThrow();
+
+            expect(mockExit)
+                .toHaveBeenCalledWith(1);
+        });
+
+        it('exits with zero exit code when the architecture passes all the spectral validations ', async () => {
+            const expectedSpectralOutput: ISpectralDiagnostic[] = [];
+
+            mockRunFunction.mockReturnValue(expectedSpectralOutput);
+
+            const apiGateway = readFileSync(path.resolve(__dirname, '../../../test_fixtures/api-gateway-implementation.json'), 'utf8');
+            fetchMock.mock('http://exist/api-gateway-implementation.json', apiGateway);
+
+            await expect(validateAndExitConditionally('http://exist/api-gateway-implementation.json', undefined, metaSchemaLocation, debugDisabled));
+
+            expect(mockExit)
+                .toHaveBeenCalledWith(0);
+        });
+    });
 });
 
 

shared/src/commands/validate/validate.ts

@@ -267,71 +267,105 @@ export async function validate(
     debug: boolean = false): Promise<ValidationOutcome> {
 
     logger = initLogger(debug);
-    let errors = false;
-    let warnings = false;
     try {
-        const ajv = buildAjv2020(debug);
-
-        await loadMetaSchemas(ajv, metaSchemaPath);
-
-        logger.info(`Loading pattern from : ${jsonSchemaLocation}`);
-        const jsonSchema = await getFileFromUrlOrPath(jsonSchemaLocation);
-
-        const spectralResultForPattern: SpectralResult = await runSpectralValidations(stripRefs(jsonSchema), validationRulesForPattern);
-
-        if (jsonSchemaArchitectureLocation === undefined) {
-            return validatePatternOnly(spectralResultForPattern, jsonSchema, ajv);
+        if (jsonSchemaArchitectureLocation && jsonSchemaLocation) {
+            return await validateArchitectureAgainstPattern(jsonSchemaArchitectureLocation, jsonSchemaLocation, metaSchemaPath, debug);
+        } else if (jsonSchemaLocation) {
+            return await validatePatternOnly(jsonSchemaLocation, metaSchemaPath, debug);
+        } else if (jsonSchemaArchitectureLocation) {
+            return await validateArchitectureOnly(jsonSchemaArchitectureLocation);
+        } else {
+            logger.debug('You must provide at least an architecture or a pattern');
+            throw new Error('You must provide at least an architecture or a pattern');
         }
+    } catch (error) {
+        logger.error('An error occured:', error);
+        process.exit(1);
+    }
+}
 
-        const validateSchema = await ajv.compileAsync(jsonSchema);
+/**
+ * Run the spectral rules for the pattern and the architecture, and then compile the pattern and validate the architecture against it.
+ * 
+ * @param jsonSchemaArchitectureLocation the location of the architecture to validate.
+ * @param jsonSchemaLocation the location of the pattern to validate against.
+ * @param metaSchemaPath the path of the meta schemas to use for ajv.
+ * @param debug the flag to enable debug logging.
+ * @returns the validation outcome with the results of the spectral and json schema validations.
+ */
+async function validateArchitectureAgainstPattern(jsonSchemaArchitectureLocation:string, jsonSchemaLocation:string, metaSchemaPath:string, debug: boolean): Promise<ValidationOutcome>{
+    const ajv = buildAjv2020(debug);
+    await loadMetaSchemas(ajv, metaSchemaPath);
 
-        logger.info(`Loading architecture from : ${jsonSchemaArchitectureLocation}`);
-        const jsonSchemaArchitecture = await getFileFromUrlOrPath(jsonSchemaArchitectureLocation);
+    logger.info(`Loading pattern from : ${jsonSchemaLocation}`);
+    const jsonSchema = await getFileFromUrlOrPath(jsonSchemaLocation);
+    const spectralResultForPattern: SpectralResult = await runSpectralValidations(stripRefs(jsonSchema), validationRulesForPattern);
+    const validateSchema = await ajv.compileAsync(jsonSchema);
 
-        const spectralResultForArchitecture: SpectralResult = await runSpectralValidations(jsonSchemaArchitecture, validationRulesForArchitecture);
+    logger.info(`Loading architecture from : ${jsonSchemaArchitectureLocation}`);
+    const jsonSchemaArchitecture = await getFileFromUrlOrPath(jsonSchemaArchitectureLocation);
 
-        const spectralResult = mergeSpectralResults(spectralResultForPattern, spectralResultForArchitecture);
+    const spectralResultForArchitecture: SpectralResult = await runSpectralValidations(jsonSchemaArchitecture, validationRulesForArchitecture);
 
-        errors = spectralResult.errors;
-        warnings = spectralResult.warnings;
+    const spectralResult = mergeSpectralResults(spectralResultForPattern, spectralResultForArchitecture);
 
-        let jsonSchemaValidations = [];
-        if (!validateSchema(jsonSchemaArchitecture)) {
-            logger.debug(`JSON Schema validation raw output: ${prettifyJson(validateSchema.errors)}`);
-            errors = true;
-            jsonSchemaValidations = convertJsonSchemaIssuesToValidationOutputs(validateSchema.errors);
-        }
+    let errors = spectralResult.errors;
+    const warnings = spectralResult.warnings;
 
-        return new ValidationOutcome(jsonSchemaValidations, spectralResult.spectralIssues, errors, warnings);
-    } catch (error) {
-        logger.error('An error occured:', error);
-        process.exit(1);
+    let jsonSchemaValidations = [];
+
+    if (!validateSchema(jsonSchemaArchitecture)) {
+        logger.debug(`JSON Schema validation raw output: ${prettifyJson(validateSchema.errors)}`);
+        errors = true;
+        jsonSchemaValidations = convertJsonSchemaIssuesToValidationOutputs(validateSchema.errors);
     }
+
+    return new ValidationOutcome(jsonSchemaValidations, spectralResult.spectralIssues, errors, warnings);
 }
 
 /**
  * Run validations for the case where only the pattern is provided. 
- * This essentially tries to compile the pattern, and returns the errors thrown if it fails.
+ * This essentially runs the spectral validations and tries to compile the pattern.
  * 
- * @param spectralValidationResults The results from running Spectral on the pattern.
- * @param patternSchema The pattern as a JS object, parsed from the file.
- * @param ajv The AJV instance to compile with.
- * @param failOnWarnings Whether or not to treat a warning as a failure in the validation process.
+ * @param jsonSchemaLocation the location of the patterns JSON Schema to validate.
+ * @param metaSchemaPath the path of the meta schemas to use for ajv.
+ * @param debug the flag to enable debug logging.
+ * @returns the validation outcome with the results of the spectral validation and the pattern compilation.
  */
-function validatePatternOnly(spectralValidationResults: SpectralResult, patternSchema: object, ajv: Ajv2020): ValidationOutcome {
-    logger.debug('Architecture was not provided, only the JSON Schema will be validated');
+async function validatePatternOnly(jsonSchemaLocation: string, metaSchemaPath: string, debug: boolean): Promise<ValidationOutcome> {
+    logger.debug('Architecture was not provided, only the Pattern Schema will be validated');
+    const ajv = buildAjv2020(debug);
+    await loadMetaSchemas(ajv, metaSchemaPath);
+
+    const patternSchema = await getFileFromUrlOrPath(jsonSchemaLocation);
+    const spectralValidationResults: SpectralResult = await runSpectralValidations(stripRefs(patternSchema), validationRulesForPattern);
+
     let errors = spectralValidationResults.errors;
     const warnings = spectralValidationResults.warnings;
     const jsonSchemaErrors = [];
 
     try {
-        ajv.compile(patternSchema);
+        await ajv.compileAsync(patternSchema); 
     } catch (error) {
         errors = true;
         jsonSchemaErrors.push(new ValidationOutput('json-schema', 'error', error.message, '/'));
     }
 
-    return new ValidationOutcome(jsonSchemaErrors, [], errors, warnings);
+    return new ValidationOutcome(jsonSchemaErrors, spectralValidationResults.spectralIssues, errors, warnings);// added spectral to return object
+}
+
+/**
+ * Run the spectral validations for the case where only the architecture is provided.
+ * 
+ * @param architectureSchemaLocation The location of the architecture schema.
+ * @returns the validation outcome with the results of the spectral validation.
+ */
+async function validateArchitectureOnly(architectureSchemaLocation: string): Promise<ValidationOutcome> {
+    logger.debug('Pattern was not provided, only the Architecture will be validated');
+
+    const jsonSchemaArchitecture = await getFileFromUrlOrPath(architectureSchemaLocation);
+    const spectralResultForArchitecture: SpectralResult = await runSpectralValidations(jsonSchemaArchitecture, validationRulesForArchitecture); 
+    return new ValidationOutcome([], spectralResultForArchitecture.spectralIssues, spectralResultForArchitecture.errors, spectralResultForArchitecture.warnings);
 }
 
 function extractSpectralRuleNames(): string[] {