From 1c5e62c7dbfe0d3d8fd3ae5c93696b4b7f210f08 Mon Sep 17 00:00:00 2001 From: lyubick Date: Wed, 15 Nov 2023 13:01:48 +0200 Subject: [PATCH 01/14] ISSUE #11: Ignore empty file option added ISSUE #7: All files will be checked before crashing with error --- action.yaml | 4 ++++ validator.py | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/action.yaml b/action.yaml index ad87e1d..1974eda 100644 --- a/action.yaml +++ b/action.yaml @@ -26,6 +26,10 @@ inputs: otherwise empty files will be ignored. required: false default: 'true' + schema-mapping: + description: ... + required: false + default: '' runs: using: 'docker' image: 'Dockerfile' diff --git a/validator.py b/validator.py index e89d925..1c7ff8e 100644 --- a/validator.py +++ b/validator.py @@ -75,5 +75,11 @@ def validate_files(yaml_files: list, json_schema: json): args = sys.argv[1:] schema = load_schema(args[0]) - files = get_yaml_json_files_list(args[1], args[2].lower() == 'true') + + files = get_yaml_json_files_list( + files_paths_list=args[1], + is_recursive=args[2].lower() == 'true', + ignore_empty_files=args[3].lower() == 'true' + ) + validate_files(files, schema) From 842a9c444fd3842b84c795d829e1f2931fbc909d Mon Sep 17 00:00:00 2001 From: lyubick Date: Wed, 15 Nov 2023 13:06:41 +0200 Subject: [PATCH 02/14] ISSUE #11: Ignore empty file option added ISSUE #7: All files will be checked before crashing with error --- validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator.py b/validator.py index 1c7ff8e..80f90b4 100644 --- a/validator.py +++ b/validator.py @@ -42,7 +42,7 @@ def get_yaml_json_files_list(files_paths_list: str, is_recursive: bool, ignore_e if ignore_empty_files: yaml_files = list(filter(lambda f: os.path.getsize(f) > 0, yaml_files)) - return yaml_files + return sorted(yaml_files) def validate_files(yaml_files: list, json_schema: json): From 1b81595792295e0a5bdc88fd0a5e3c6a3babb846 Mon Sep 17 00:00:00 2001 From: lyubick Date: Wed, 15 Nov 2023 13:09:15 +0200 Subject: [PATCH 03/14] ISSUE #11: Ignore empty file option added ISSUE #7: All files will be checked before crashing with error --- test/JSONs/{invalid.json => invalid1.json} | 0 test/JSONs/{invalid_2.json => invalid2.json} | 0 test/test_sanity.py | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) rename test/JSONs/{invalid.json => invalid1.json} (100%) rename test/JSONs/{invalid_2.json => invalid2.json} (100%) diff --git a/test/JSONs/invalid.json b/test/JSONs/invalid1.json similarity index 100% rename from test/JSONs/invalid.json rename to test/JSONs/invalid1.json diff --git a/test/JSONs/invalid_2.json b/test/JSONs/invalid2.json similarity index 100% rename from test/JSONs/invalid_2.json rename to test/JSONs/invalid2.json diff --git a/test/test_sanity.py b/test/test_sanity.py index 0455566..b7707b0 100644 --- a/test/test_sanity.py +++ b/test/test_sanity.py @@ -69,8 +69,8 @@ def test8_validate_folder_json(self): except Exception as exc: assert len(exc.args) == 1 assert len(exc.args[0]) == 2 - assert exc.args[0][1][1] == {'field2': 'Value2'} - assert exc.args[0][0][1] == {'field2': 'Value2_2'} + assert exc.args[0][0][1] == {'field2': 'Value2'} + assert exc.args[0][1][1] == {'field2': 'Value2_2'} def test9_validate_empty_json(self): schema = validator.load_schema(f'{self.abs_path}/schema/json_schema.json') From dd98828bcf98ab5741bde3ec5690f9930432ecf8 Mon Sep 17 00:00:00 2001 From: lyubick Date: Wed, 15 Nov 2023 13:09:46 +0200 Subject: [PATCH 04/14] ISSUE #11: Ignore empty file option added ISSUE #7: All files will be checked before crashing with error --- test/test_sanity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_sanity.py b/test/test_sanity.py index b7707b0..4aa1ba6 100644 --- a/test/test_sanity.py +++ b/test/test_sanity.py @@ -40,7 +40,7 @@ def test5_validate_valid_file_json(self): def test6_validate_invalid_file_json(self): schema = validator.load_schema(f'{self.abs_path}/schema/json_schema.json') - files = validator.get_yaml_json_files_list(f'{self.abs_path}/JSONs/invalid.json', False) + files = validator.get_yaml_json_files_list(f'{self.abs_path}/JSONs/invalid1.json', False) try: validator.validate_files(files, schema) assert False From 0ae9bad54f903e46b79574c284d6c557fadf0634 Mon Sep 17 00:00:00 2001 From: lyubick Date: Mon, 4 Dec 2023 13:24:37 +0200 Subject: [PATCH 05/14] ISSUE #5: Enable primitive schema mapping --- .github/workflows/self-test.yaml | 8 ++- action.yaml | 5 +- test/test_sanity.py | 75 ++++++++++++-------- validator.py | 115 ++++++++++++++++++++++--------- 4 files changed, 141 insertions(+), 62 deletions(-) diff --git a/.github/workflows/self-test.yaml b/.github/workflows/self-test.yaml index 9f1c52a..3da3334 100644 --- a/.github/workflows/self-test.yaml +++ b/.github/workflows/self-test.yaml @@ -24,4 +24,10 @@ jobs: uses: ./ with: json-schema-file: test/schema/json_schema.json - yaml-json-file-dir: test/emptyJSONs,test/emptyYAMLSs, + yaml-json-file-dir: test/emptyJSONs,test/emptyYAMLSs + - name: Run Action (check mapping) + uses: ./ + with: + json-schema-file: test/schema/json_schema.json + yaml-json-file-dir: test/emptyJSONs,test/emptyYAMLSs + schema-mapping: 'test/emptyJSONs:test/schema/json_schema.json,test/emptyYAMLSs:test/schema/json_schema.json' diff --git a/action.yaml b/action.yaml index 1974eda..1de59b8 100644 --- a/action.yaml +++ b/action.yaml @@ -27,7 +27,10 @@ inputs: required: false default: 'true' schema-mapping: - description: ... + description: | + File/Directory and Schema mapping, provided in a one of formats: + 1. /path/to/file:/path/to/schema.json + 2. /path/to/dir/:/path/to/schema.json required: false default: '' runs: diff --git a/test/test_sanity.py b/test/test_sanity.py index 4aa1ba6..880d8b9 100644 --- a/test/test_sanity.py +++ b/test/test_sanity.py @@ -9,24 +9,27 @@ class Test: abs_path = Path(__file__).parent def test1_load_schema(self): - schema = validator.load_schema(f'{self.abs_path}/schema/json_schema.json') - assert schema['title'] == 'TestConfig' - assert schema['description'] == 'Test Basic YAML File' + schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + assert len(schemas.keys()) == 1 + assert list(schemas.values())[0]['title'] == 'TestConfig' + assert list(schemas.values())[0]['description'] == 'Test Basic YAML File' def test2_get_files(self): - files = validator.get_yaml_json_files_list(f'{self.abs_path}/YAMLs', is_recursive=True) + files = validator.get_testing_filenames(f'{self.abs_path}/YAMLs', is_recursive=True) assert len(files) == 2 def test3_validate_valid_file(self): - schema = validator.load_schema(f'{self.abs_path}/schema/json_schema.json') - files = validator.get_yaml_json_files_list(f'{self.abs_path}/YAMLs/valid.yaml', False) - assert validator.validate_files(files, schema) + schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + files = validator.get_testing_filenames(f'{self.abs_path}/YAMLs/valid.yaml', False) + files_schemas = validator.get_filenames_with_schema(files, schemas, None) + assert validator.validate_files(files_schemas) def test4_validate_invalid_file(self): - schema = validator.load_schema(f'{self.abs_path}/schema/json_schema.json') - files = validator.get_yaml_json_files_list(f'{self.abs_path}/YAMLs/invalid.yaml', False) + schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + files = validator.get_testing_filenames(f'{self.abs_path}/YAMLs/invalid.yaml', False) + files_schemas = validator.get_filenames_with_schema(files, schemas, None) try: - validator.validate_files(files, schema) + validator.validate_files(files_schemas) assert False except Exception as exc: assert len(exc.args) == 1 @@ -34,15 +37,17 @@ def test4_validate_invalid_file(self): assert exc.args[0][0][1] == {'field2': 'Value2'} def test5_validate_valid_file_json(self): - schema = validator.load_schema(f'{self.abs_path}/schema/json_schema.json') - files = validator.get_yaml_json_files_list(f'{self.abs_path}/JSONs/valid.json', False) - assert validator.validate_files(files, schema) + schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + files = validator.get_testing_filenames(f'{self.abs_path}/JSONs/valid.json', False) + files_schemas = validator.get_filenames_with_schema(files, schemas, None) + assert validator.validate_files(files_schemas) def test6_validate_invalid_file_json(self): - schema = validator.load_schema(f'{self.abs_path}/schema/json_schema.json') - files = validator.get_yaml_json_files_list(f'{self.abs_path}/JSONs/invalid1.json', False) + schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + files = validator.get_testing_filenames(f'{self.abs_path}/JSONs/invalid1.json', False) + files_schemas = validator.get_filenames_with_schema(files, schemas, None) try: - validator.validate_files(files, schema) + validator.validate_files(files_schemas) assert False except Exception as exc: assert len(exc.args) == 1 @@ -50,10 +55,11 @@ def test6_validate_invalid_file_json(self): assert exc.args[0][0][1] == {'field2': 'Value2'} def test7_validate_folder_yaml(self): - schema = validator.load_schema(f'{self.abs_path}/schema/json_schema.json') - files = validator.get_yaml_json_files_list(f'{self.abs_path}/YAMLs/', False) + schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + files = validator.get_testing_filenames(f'{self.abs_path}/YAMLs/', False) + files_schemas = validator.get_filenames_with_schema(files, schemas, None) try: - validator.validate_files(files, schema) + validator.validate_files(files_schemas) assert False except Exception as exc: assert len(exc.args) == 1 @@ -61,10 +67,11 @@ def test7_validate_folder_yaml(self): assert exc.args[0][0][1] == {'field2': 'Value2'} def test8_validate_folder_json(self): - schema = validator.load_schema(f'{self.abs_path}/schema/json_schema.json') - files = validator.get_yaml_json_files_list(f'{self.abs_path}/JSONs/', False) + schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + files = validator.get_testing_filenames(f'{self.abs_path}/JSONs/', False) + files_schemas = validator.get_filenames_with_schema(files, schemas, None) try: - validator.validate_files(files, schema) + validator.validate_files(files_schemas) assert False except Exception as exc: assert len(exc.args) == 1 @@ -73,15 +80,27 @@ def test8_validate_folder_json(self): assert exc.args[0][1][1] == {'field2': 'Value2_2'} def test9_validate_empty_json(self): - schema = validator.load_schema(f'{self.abs_path}/schema/json_schema.json') - files = validator.get_yaml_json_files_list(f'{self.abs_path}/emptyJSONs/empty.json', False, True) - validator.validate_files(files, schema) + schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + files = validator.get_testing_filenames(f'{self.abs_path}/emptyJSONs/empty.json', False, True) + files_schemas = validator.get_filenames_with_schema(files, schemas, None) + validator.validate_files(files_schemas) def test9_validate_empty_json_fail(self): - schema = validator.load_schema(f'{self.abs_path}/schema/json_schema.json') - files = validator.get_yaml_json_files_list(f'{self.abs_path}/emptyJSONs/empty.json', False, False) + schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + files = validator.get_testing_filenames(f'{self.abs_path}/emptyJSONs/empty.json', False, False) + files_schemas = validator.get_filenames_with_schema(files, schemas, None) try: - validator.validate_files(files, schema) + validator.validate_files(files_schemas) + assert False + except json.JSONDecodeError as exc: + assert True + + def test10_validate_file_with_schema_map(self): + schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + files = validator.get_testing_filenames(f'{self.abs_path}/emptyJSONs/empty.json', False, False) + files_schemas = validator.get_filenames_with_schema(files, schemas, f"{self.abs_path}/emptyJSONs/empty.json:{self.abs_path}/schema/json_schema.json") + try: + validator.validate_files(files_schemas) assert False except json.JSONDecodeError as exc: assert True diff --git a/validator.py b/validator.py index 80f90b4..3f246bf 100644 --- a/validator.py +++ b/validator.py @@ -1,43 +1,53 @@ import json +import logging import os.path import pathlib +import re import sys +from typing import List, Tuple, Optional import yaml from jsonschema import validate from jsonschema.exceptions import ValidationError -def load_schema(schema_file_path: str) -> json: - if os.path.exists(schema_file_path): - if os.path.isfile(schema_file_path): - with open(schema_file_path, 'r') as stream: - return json.loads("".join(stream.readlines())) - else: - raise f'Provided JSON Schema is not a file! Please provide legit JSON Schema file.' - else: - raise f'Provided JSON Schema file does not exist!' - - -def get_yaml_json_files_list(files_paths_list: str, is_recursive: bool, ignore_empty_files: bool = False) -> list[str]: - yaml_input = list(files_paths_list.split(',')) - - yaml_files = [] - for yaml_object in yaml_input: - if os.path.isdir(yaml_object): - yaml_files.extend( - list( - map( - lambda f: str(f), # Convert all paths to string, instead of Posix Path - filter( - lambda f: str(f).endswith('.yaml') or str(f).endswith('.yml') or str(f).endswith('.json'), - pathlib.Path(yaml_object).glob('**/*' if is_recursive else '*') +def get_all_filenames(input_path: str, endings: List[str], is_recursive: bool) -> List[str]: + paths = list(input_path.split(',')) + + regex_endings = f'.*\\.({"|".join(endings)})' + + output_files = [] + + for path in paths: + if os.path.exists(path): + if os.path.isdir(path): + output_files.extend( + list( + map( + lambda f: str(f), # Convert all paths to string, instead of Posix Path + filter( + lambda f: len(re.findall(regex_endings, str(f))) > 0, + pathlib.Path(path).glob('**/*' if is_recursive else '*') + ) ) ) ) - ) - elif os.path.isfile(yaml_object): - yaml_files.append(yaml_object) + elif os.path.isfile(path): + output_files.append(path) + else: + continue + + return output_files + + +def get_all_schemas(schema_file_path: str) -> dict[str, json]: + schema_files = get_all_filenames(schema_file_path, endings=['json'], is_recursive=False) + schemas = list(map(lambda x: (x, json.loads(''.join(open(x, 'r').readlines()))), schema_files)) + return dict(schemas) + + +def get_testing_filenames(files_paths_list: str, is_recursive: bool, ignore_empty_files: bool = False) -> list[str]: + yaml_files = get_all_filenames(files_paths_list, endings=['json', 'yaml', 'yml'], is_recursive=is_recursive) if ignore_empty_files: yaml_files = list(filter(lambda f: os.path.getsize(f) > 0, yaml_files)) @@ -45,9 +55,33 @@ def get_yaml_json_files_list(files_paths_list: str, is_recursive: bool, ignore_e return sorted(yaml_files) -def validate_files(yaml_files: list, json_schema: json): +def get_filenames_with_schema(test_files: List[str], schemas: dict[str, json], mapping_str: Optional[str]) -> List[Tuple[str, json]]: + def tuple_split(inp: str, separator: str) -> Tuple[str, str]: + values = inp.split(separator) + return values[0], values[1] + + def map_schema(filename: str, schema_map: dict[str, str]) -> Tuple[str, str]: + for s in schema_map.keys(): + if filename in get_all_filenames(input_path=s, endings=['yaml', 'json'], is_recursive=True): + return filename, schemas[schema_map[s]] + + logging.error(f'{filename} does not match any schema!') + raise 'Schema mapping is invalid - not all files matches schema' + + if mapping_str: + mapping = dict(list(map(lambda x: tuple_split(x, ':'), list(mapping_str.split(','))))) + files_schema = list(map(lambda x: map_schema(x, mapping), test_files)) + else: + files_schema = list(map(lambda x: (x, list(schemas.values())[0]), test_files)) + + return files_schema + + +def validate_files(files_with_schema: List[Tuple[str, json]]): failed = [] - for yaml_file in yaml_files: + for file_with_schema in files_with_schema: + yaml_file = file_with_schema[0] + json_schema = file_with_schema[1] if os.path.exists(yaml_file): if os.path.isfile(yaml_file): with open(yaml_file, 'r') as stream: @@ -72,14 +106,31 @@ def validate_files(yaml_files: list, json_schema: json): if __name__ == '__main__': + """ + json-schema-file = args[0] + yaml-json-file-dir = args[1] + recursive = args[2] + ignore-empty = args[3] + mapping = args[4] + + /path/to/file.json:/path/to/schema.json,/path/to/*:/path/to/schema.json,... + """ args = sys.argv[1:] - schema = load_schema(args[0]) + input_schemas = get_all_schemas(args[0]) - files = get_yaml_json_files_list( + input_files = get_testing_filenames( files_paths_list=args[1], is_recursive=args[2].lower() == 'true', ignore_empty_files=args[3].lower() == 'true' ) - validate_files(files, schema) + input_mapping = args[4] + + input_files_with_schema = get_filenames_with_schema( + test_files=input_files, + schemas=input_schemas, + mapping_str=input_mapping + ) + + validate_files(input_files_with_schema) From 251f08292d2e71a5ef43e5bf5c41eef1ff4c5310 Mon Sep 17 00:00:00 2001 From: lyubick Date: Mon, 4 Dec 2023 13:27:50 +0200 Subject: [PATCH 06/14] ISSUE #5: Enable primitive schema mapping --- .github/workflows/self-test.yaml | 2 +- test/test_sanity.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/self-test.yaml b/.github/workflows/self-test.yaml index 3da3334..3c8c5b2 100644 --- a/.github/workflows/self-test.yaml +++ b/.github/workflows/self-test.yaml @@ -7,7 +7,7 @@ name: Self Test jobs: runValidation: - runs-on: ubuntu-latest + runs-on: python:3.11-slim steps: - uses: actions/checkout@v2 - name: Run PyTest(s) diff --git a/test/test_sanity.py b/test/test_sanity.py index 880d8b9..678e3c6 100644 --- a/test/test_sanity.py +++ b/test/test_sanity.py @@ -2,7 +2,6 @@ from pathlib import Path import validator -import jsonschema class Test: From b25b813c271bc67b432d8c977f89dcbb395d1e27 Mon Sep 17 00:00:00 2001 From: lyubick Date: Mon, 4 Dec 2023 14:47:16 +0200 Subject: [PATCH 07/14] ISSUE #5: Enable primitive schema mapping --- .github/workflows/self-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/self-test.yaml b/.github/workflows/self-test.yaml index 3c8c5b2..3da3334 100644 --- a/.github/workflows/self-test.yaml +++ b/.github/workflows/self-test.yaml @@ -7,7 +7,7 @@ name: Self Test jobs: runValidation: - runs-on: python:3.11-slim + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Run PyTest(s) From b47aa99a8a5a0f3f3ffd9bffb4c11bf3c46e5e89 Mon Sep 17 00:00:00 2001 From: lyubick Date: Mon, 4 Dec 2023 14:54:44 +0200 Subject: [PATCH 08/14] ISSUE #5: Enable primitive schema mapping --- validator.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/validator.py b/validator.py index 3f246bf..e85ab77 100644 --- a/validator.py +++ b/validator.py @@ -40,7 +40,7 @@ def get_all_filenames(input_path: str, endings: List[str], is_recursive: bool) - return output_files -def get_all_schemas(schema_file_path: str) -> dict[str, json]: +def get_all_schemas(schema_file_path: str) -> dict[str, dict]: schema_files = get_all_filenames(schema_file_path, endings=['json'], is_recursive=False) schemas = list(map(lambda x: (x, json.loads(''.join(open(x, 'r').readlines()))), schema_files)) return dict(schemas) @@ -55,12 +55,15 @@ def get_testing_filenames(files_paths_list: str, is_recursive: bool, ignore_empt return sorted(yaml_files) -def get_filenames_with_schema(test_files: List[str], schemas: dict[str, json], mapping_str: Optional[str]) -> List[Tuple[str, json]]: +def get_filenames_with_schema( + test_files: List[str], + schemas: dict[str, dict], + mapping_str: Optional[str]) -> List[Tuple[str, dict]]: def tuple_split(inp: str, separator: str) -> Tuple[str, str]: values = inp.split(separator) return values[0], values[1] - def map_schema(filename: str, schema_map: dict[str, str]) -> Tuple[str, str]: + def map_schema(filename: str, schema_map: dict[str, str]) -> Tuple[str, json]: for s in schema_map.keys(): if filename in get_all_filenames(input_path=s, endings=['yaml', 'json'], is_recursive=True): return filename, schemas[schema_map[s]] @@ -77,7 +80,7 @@ def map_schema(filename: str, schema_map: dict[str, str]) -> Tuple[str, str]: return files_schema -def validate_files(files_with_schema: List[Tuple[str, json]]): +def validate_files(files_with_schema: List[Tuple[str, dict]]): failed = [] for file_with_schema in files_with_schema: yaml_file = file_with_schema[0] @@ -106,15 +109,6 @@ def validate_files(files_with_schema: List[Tuple[str, json]]): if __name__ == '__main__': - """ - json-schema-file = args[0] - yaml-json-file-dir = args[1] - recursive = args[2] - ignore-empty = args[3] - mapping = args[4] - - /path/to/file.json:/path/to/schema.json,/path/to/*:/path/to/schema.json,... - """ args = sys.argv[1:] input_schemas = get_all_schemas(args[0]) From 5ca2fecfa04437327f72ac576cd441d3e23c0f11 Mon Sep 17 00:00:00 2001 From: lyubick Date: Mon, 4 Dec 2023 14:56:06 +0200 Subject: [PATCH 09/14] ISSUE #5: Enable primitive schema mapping --- validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator.py b/validator.py index e85ab77..83fc404 100644 --- a/validator.py +++ b/validator.py @@ -63,7 +63,7 @@ def tuple_split(inp: str, separator: str) -> Tuple[str, str]: values = inp.split(separator) return values[0], values[1] - def map_schema(filename: str, schema_map: dict[str, str]) -> Tuple[str, json]: + def map_schema(filename: str, schema_map: dict[str, str]) -> Tuple[str, dict]: for s in schema_map.keys(): if filename in get_all_filenames(input_path=s, endings=['yaml', 'json'], is_recursive=True): return filename, schemas[schema_map[s]] From 0041c090234a878d76880dd04a32fcb52d581d6a Mon Sep 17 00:00:00 2001 From: lyubick Date: Mon, 4 Dec 2023 15:00:28 +0200 Subject: [PATCH 10/14] ISSUE #5: Enable primitive schema mapping --- action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yaml b/action.yaml index 1de59b8..aaa837a 100644 --- a/action.yaml +++ b/action.yaml @@ -32,7 +32,6 @@ inputs: 1. /path/to/file:/path/to/schema.json 2. /path/to/dir/:/path/to/schema.json required: false - default: '' runs: using: 'docker' image: 'Dockerfile' @@ -41,3 +40,4 @@ runs: - ${{ inputs.yaml-json-file-dir }} - ${{ inputs.recursive }} - ${{ inputs.ignore-empty }} + - ${{ inputs.schema-mapping }} From 2b415fd54452efa8af656bdc062cb83583613286 Mon Sep 17 00:00:00 2001 From: lyubick Date: Wed, 6 Dec 2023 15:58:56 +0200 Subject: [PATCH 11/14] ISSUE #5: Enable primitive schema mapping Take schemas from mapping use a default schema --- .github/workflows/self-test.yaml | 4 ++- action.yaml | 5 +++- test/test_sanity.py | 50 +++++++++++++++++++++++++------- validator.py | 37 ++++++++++++++++------- 4 files changed, 73 insertions(+), 23 deletions(-) diff --git a/.github/workflows/self-test.yaml b/.github/workflows/self-test.yaml index 3da3334..753a40c 100644 --- a/.github/workflows/self-test.yaml +++ b/.github/workflows/self-test.yaml @@ -30,4 +30,6 @@ jobs: with: json-schema-file: test/schema/json_schema.json yaml-json-file-dir: test/emptyJSONs,test/emptyYAMLSs - schema-mapping: 'test/emptyJSONs:test/schema/json_schema.json,test/emptyYAMLSs:test/schema/json_schema.json' + schema-mapping: + - test/emptyJSONs:test/schema/json_schema.json + - test/emptyYAMLSs:test/schema/json_schema.json diff --git a/action.yaml b/action.yaml index aaa837a..c2325f9 100644 --- a/action.yaml +++ b/action.yaml @@ -7,7 +7,9 @@ branding: color: green inputs: json-schema-file: - description: 'JSON Schema file to validate against' + description: | + JSON Schema file to validate against. In case mapping is provided, schema + provided here will be used as a default (fallback) schema. required: true yaml-json-file-dir: description: | @@ -32,6 +34,7 @@ inputs: 1. /path/to/file:/path/to/schema.json 2. /path/to/dir/:/path/to/schema.json required: false + default: '' runs: using: 'docker' image: 'Dockerfile' diff --git a/test/test_sanity.py b/test/test_sanity.py index 678e3c6..2e9685e 100644 --- a/test/test_sanity.py +++ b/test/test_sanity.py @@ -8,7 +8,10 @@ class Test: abs_path = Path(__file__).parent def test1_load_schema(self): - schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + schemas = validator.get_all_schemas( + schema_file_path='', + default_schema_path=f'{self.abs_path}/schema/json_schema.json' + ) assert len(schemas.keys()) == 1 assert list(schemas.values())[0]['title'] == 'TestConfig' assert list(schemas.values())[0]['description'] == 'Test Basic YAML File' @@ -18,13 +21,19 @@ def test2_get_files(self): assert len(files) == 2 def test3_validate_valid_file(self): - schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + schemas = validator.get_all_schemas( + schema_file_path='', + default_schema_path=f'{self.abs_path}/schema/json_schema.json' + ) files = validator.get_testing_filenames(f'{self.abs_path}/YAMLs/valid.yaml', False) files_schemas = validator.get_filenames_with_schema(files, schemas, None) assert validator.validate_files(files_schemas) def test4_validate_invalid_file(self): - schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + schemas = validator.get_all_schemas( + schema_file_path='', + default_schema_path=f'{self.abs_path}/schema/json_schema.json' + ) files = validator.get_testing_filenames(f'{self.abs_path}/YAMLs/invalid.yaml', False) files_schemas = validator.get_filenames_with_schema(files, schemas, None) try: @@ -36,13 +45,19 @@ def test4_validate_invalid_file(self): assert exc.args[0][0][1] == {'field2': 'Value2'} def test5_validate_valid_file_json(self): - schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + schemas = validator.get_all_schemas( + schema_file_path='', + default_schema_path=f'{self.abs_path}/schema/json_schema.json' + ) files = validator.get_testing_filenames(f'{self.abs_path}/JSONs/valid.json', False) files_schemas = validator.get_filenames_with_schema(files, schemas, None) assert validator.validate_files(files_schemas) def test6_validate_invalid_file_json(self): - schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + schemas = validator.get_all_schemas( + schema_file_path='', + default_schema_path=f'{self.abs_path}/schema/json_schema.json' + ) files = validator.get_testing_filenames(f'{self.abs_path}/JSONs/invalid1.json', False) files_schemas = validator.get_filenames_with_schema(files, schemas, None) try: @@ -54,7 +69,10 @@ def test6_validate_invalid_file_json(self): assert exc.args[0][0][1] == {'field2': 'Value2'} def test7_validate_folder_yaml(self): - schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + schemas = validator.get_all_schemas( + schema_file_path='', + default_schema_path=f'{self.abs_path}/schema/json_schema.json' + ) files = validator.get_testing_filenames(f'{self.abs_path}/YAMLs/', False) files_schemas = validator.get_filenames_with_schema(files, schemas, None) try: @@ -66,7 +84,10 @@ def test7_validate_folder_yaml(self): assert exc.args[0][0][1] == {'field2': 'Value2'} def test8_validate_folder_json(self): - schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + schemas = validator.get_all_schemas( + schema_file_path='', + default_schema_path=f'{self.abs_path}/schema/json_schema.json' + ) files = validator.get_testing_filenames(f'{self.abs_path}/JSONs/', False) files_schemas = validator.get_filenames_with_schema(files, schemas, None) try: @@ -79,13 +100,19 @@ def test8_validate_folder_json(self): assert exc.args[0][1][1] == {'field2': 'Value2_2'} def test9_validate_empty_json(self): - schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + schemas = validator.get_all_schemas( + schema_file_path='', + default_schema_path=f'{self.abs_path}/schema/json_schema.json' + ) files = validator.get_testing_filenames(f'{self.abs_path}/emptyJSONs/empty.json', False, True) files_schemas = validator.get_filenames_with_schema(files, schemas, None) validator.validate_files(files_schemas) def test9_validate_empty_json_fail(self): - schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + schemas = validator.get_all_schemas( + schema_file_path='', + default_schema_path=f'{self.abs_path}/schema/json_schema.json' + ) files = validator.get_testing_filenames(f'{self.abs_path}/emptyJSONs/empty.json', False, False) files_schemas = validator.get_filenames_with_schema(files, schemas, None) try: @@ -95,7 +122,10 @@ def test9_validate_empty_json_fail(self): assert True def test10_validate_file_with_schema_map(self): - schemas = validator.get_all_schemas(f'{self.abs_path}/schema/json_schema.json') + schemas = validator.get_all_schemas( + schema_file_path=f'{self.abs_path}/schema/json_schema.json', + default_schema_path=f'{self.abs_path}/schema/json_schema.json' + ) files = validator.get_testing_filenames(f'{self.abs_path}/emptyJSONs/empty.json', False, False) files_schemas = validator.get_filenames_with_schema(files, schemas, f"{self.abs_path}/emptyJSONs/empty.json:{self.abs_path}/schema/json_schema.json") try: diff --git a/validator.py b/validator.py index 83fc404..f6c03eb 100644 --- a/validator.py +++ b/validator.py @@ -40,9 +40,10 @@ def get_all_filenames(input_path: str, endings: List[str], is_recursive: bool) - return output_files -def get_all_schemas(schema_file_path: str) -> dict[str, dict]: +def get_all_schemas(schema_file_path: str, default_schema_path: str) -> dict[str, dict]: schema_files = get_all_filenames(schema_file_path, endings=['json'], is_recursive=False) schemas = list(map(lambda x: (x, json.loads(''.join(open(x, 'r').readlines()))), schema_files)) + schemas.append(('default', json.loads(''.join(open(default_schema_path, 'r').readlines())))) return dict(schemas) @@ -59,23 +60,19 @@ def get_filenames_with_schema( test_files: List[str], schemas: dict[str, dict], mapping_str: Optional[str]) -> List[Tuple[str, dict]]: - def tuple_split(inp: str, separator: str) -> Tuple[str, str]: - values = inp.split(separator) - return values[0], values[1] def map_schema(filename: str, schema_map: dict[str, str]) -> Tuple[str, dict]: for s in schema_map.keys(): - if filename in get_all_filenames(input_path=s, endings=['yaml', 'json'], is_recursive=True): + if filename in get_all_filenames(input_path=s, endings=['yaml', 'json', 'yml'], is_recursive=True): return filename, schemas[schema_map[s]] - logging.error(f'{filename} does not match any schema!') - raise 'Schema mapping is invalid - not all files matches schema' + return filename, schemas['default'] if mapping_str: mapping = dict(list(map(lambda x: tuple_split(x, ':'), list(mapping_str.split(','))))) files_schema = list(map(lambda x: map_schema(x, mapping), test_files)) else: - files_schema = list(map(lambda x: (x, list(schemas.values())[0]), test_files)) + files_schema = list(map(lambda x: (x, schemas['default']), test_files)) return files_schema @@ -108,10 +105,30 @@ def validate_files(files_with_schema: List[Tuple[str, dict]]): return True +def tuple_split(inp: str, separator: str) -> Tuple[str, str]: + """ + + :param inp: String in a form of , that will be split into (key, value) tuple. + :param separator: String representing separator. + :return: filename: str, schema: str + """ + values = inp.split(separator) + return values[0], values[1] + + if __name__ == '__main__': args = sys.argv[1:] - input_schemas = get_all_schemas(args[0]) + input_mapping = args[4] + + input_schemas = {} + + if input_mapping: + logging.error(input_mapping) + input_mapped_schemas = ','.join(list(map(lambda x: tuple_split(x, ':')[1], input_mapping.split(',')))) + input_schemas = get_all_schemas(input_mapped_schemas) + + input_schemas['default'] = list(get_all_schemas(args[0]).values())[0] input_files = get_testing_filenames( files_paths_list=args[1], @@ -119,8 +136,6 @@ def validate_files(files_with_schema: List[Tuple[str, dict]]): ignore_empty_files=args[3].lower() == 'true' ) - input_mapping = args[4] - input_files_with_schema = get_filenames_with_schema( test_files=input_files, schemas=input_schemas, From d4f19c804d9d74391db40e2cdcdccde751abd6b8 Mon Sep 17 00:00:00 2001 From: lyubick Date: Wed, 6 Dec 2023 16:00:51 +0200 Subject: [PATCH 12/14] ISSUE #5: Enable primitive schema mapping Take schemas from mapping use a default schema --- .github/workflows/self-test.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/self-test.yaml b/.github/workflows/self-test.yaml index 753a40c..6f1f5ef 100644 --- a/.github/workflows/self-test.yaml +++ b/.github/workflows/self-test.yaml @@ -30,6 +30,4 @@ jobs: with: json-schema-file: test/schema/json_schema.json yaml-json-file-dir: test/emptyJSONs,test/emptyYAMLSs - schema-mapping: - - test/emptyJSONs:test/schema/json_schema.json - - test/emptyYAMLSs:test/schema/json_schema.json + schema-mapping: "test/emptyJSONs:test/schema/json_schema.json,test/emptyYAMLSs:test/schema/json_schema.json" From c7675e09044dc7cfaa2cc677943b6145e270977b Mon Sep 17 00:00:00 2001 From: lyubick Date: Wed, 6 Dec 2023 16:03:42 +0200 Subject: [PATCH 13/14] ISSUE #5: Enable primitive schema mapping Take schemas from mapping use a default schema --- validator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validator.py b/validator.py index f6c03eb..5bccb9d 100644 --- a/validator.py +++ b/validator.py @@ -123,12 +123,12 @@ def tuple_split(inp: str, separator: str) -> Tuple[str, str]: input_schemas = {} + input_mapped_schemas = '' if input_mapping: logging.error(input_mapping) input_mapped_schemas = ','.join(list(map(lambda x: tuple_split(x, ':')[1], input_mapping.split(',')))) - input_schemas = get_all_schemas(input_mapped_schemas) - input_schemas['default'] = list(get_all_schemas(args[0]).values())[0] + input_schemas = get_all_schemas(schema_file_path=input_mapped_schemas, default_schema_path=args[0]) input_files = get_testing_filenames( files_paths_list=args[1], From 0e5249512e30265c76444a23934b53783afb1658 Mon Sep 17 00:00:00 2001 From: lyubick Date: Wed, 6 Dec 2023 16:08:24 +0200 Subject: [PATCH 14/14] ISSUE #5: Enable primitive schema mapping Docs. --- README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 17b058f..a4c1109 100644 --- a/README.md +++ b/README.md @@ -21,13 +21,17 @@ jobs: json-schema-file: path/to/my/cool/schema.json yaml-json-file-dir: path/to/my/cool/yaml/file.yaml recursive: false + ignore-empty: false + schema-mapping: 'path/to/my/cool/yaml/file.yaml:path/to/my/cool/schema.json' ``` One should provide two parameters: -- `json-schema-file`, points to legit JSON Schema file -- `yaml-json-file-dir`, is a comma separated list that contains +- `json-schema-file`, required, points to legit JSON Schema files. In case of mapping this schema will be used as default (fallback) schema. +- `yaml-json-file-dir`, is a comma separated list that contains: - Single YAML or JSON files - Directories that will be parsed for `.yaml` or `.yml` or `.json` files -- `recursive`, True/False depending on if recursive scan for YAML or JSON files in directory required +- `recursive`, optional, True/False depending on if recursive scan for YAML or JSON files in directory required. Default is False. +- `ignore-empty`, optional, True/False depends if one want to cause failure on empty files or not, default is True. +- `schema-mapping`, optional, one can provide file-to-schema mapping, if specific files require specific schema. ## Results ### Success @@ -95,5 +99,5 @@ On instance: {'field2': 'Value2'} ``` -## Supported & Used by -* [Printify](https://github.com/printify) +## Organisations Use +* [Printify](https://printify.com)