From ab5101a8d1021ae4a7bc8d4375cfcc6ad8797bde Mon Sep 17 00:00:00 2001 From: Elango Cheran Date: Sun, 28 Apr 2024 23:11:47 -0700 Subject: [PATCH 1/5] Validate test cases against provided schema --- .github/workflows/validate_tests.yml | 20 ++++++++++ test/validate.py | 59 ++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 .github/workflows/validate_tests.yml create mode 100755 test/validate.py diff --git a/.github/workflows/validate_tests.yml b/.github/workflows/validate_tests.yml new file mode 100644 index 000000000..93e472706 --- /dev/null +++ b/.github/workflows/validate_tests.yml @@ -0,0 +1,20 @@ +name: Validate test data + +on: + push: + branches: + - main + pull_request: + branches: '**' + +jobs: + run_all: + name: Validate tests for specific schema version + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Validate tests for schema v0 + run: | + cd test + python3 validate.py --schema-dir ./schemas/v0/ diff --git a/test/validate.py b/test/validate.py new file mode 100755 index 000000000..398197330 --- /dev/null +++ b/test/validate.py @@ -0,0 +1,59 @@ +#!/bin/env python3 + +import argparse +import json +import jsonschema +import logging +import os +import sys + + +def parse_args(args): + parser = argparse.ArgumentParser( + prog='Test File Schema Validator', + description='Validate all of the test case JSON files against the JSON Schema file' + ) + + parser.add_argument('--schema-dir', + help='The directory containing the JSON schema file for the test cases (dir named according to a semantic version)') + parser.add_argument('--test-dir', + default='tests', + help='The base directory in which all test files are contained') + + # ignore the executable's name from the arg list, which is in the first position + return parser.parse_args(args[1:]) + +def main(args): + # initialize logging + logging.basicConfig(format='[%(asctime)s] [%(levelname)s] %(message)s', level=logging.INFO) + + # parse args + parsed_args = parse_args(args) + + # get schema file from the schema dir + schema_dir = parsed_args.schema_dir + schema_file_paths = os.listdir(path = schema_dir) + assert len(schema_file_paths) == 1, "there should only be 1 schema(s) definition file for a given schema version" + schema_file_path = os.path.join(schema_dir, schema_file_paths[0]) + logging.debug("schema file path: %s", schema_file_path) + schema_file = open(schema_file_path, encoding='utf-8', mode='r') + schema = json.load(schema_file) + + # iterate over all files in the test case file dir and validate against the schema + test_dir = parsed_args.test_dir + for (dirpath, dirnames, filenames) in os.walk(test_dir): + for test_filename in filenames: + test_file_path = os.path.join(dirpath, test_filename) + logging.debug("test file path: %s", test_file_path) + test_file = open(test_file_path, encoding='utf-8', mode='r') + logging.debug("verifying test case file: %s", test_file_path) + tests = json.load(test_file) + jsonschema.validate(tests, schema) + + test_file.close() + schema_file.close() + + logging.info("Completed verification of all test case files successfully") + +if __name__ == "__main__": + main(sys.argv) From bc16a301bd18fc1e0e05de67e6892f7d55fe17bb Mon Sep 17 00:00:00 2001 From: Elango Cheran Date: Wed, 1 May 2024 08:18:54 -0700 Subject: [PATCH 2/5] Use command line JSON Schema validator from NPM --- .github/workflows/validate_tests.yml | 10 +++-- test/validate.py | 59 ---------------------------- 2 files changed, 6 insertions(+), 63 deletions(-) delete mode 100755 test/validate.py diff --git a/.github/workflows/validate_tests.yml b/.github/workflows/validate_tests.yml index 93e472706..8e441a7ac 100644 --- a/.github/workflows/validate_tests.yml +++ b/.github/workflows/validate_tests.yml @@ -9,12 +9,14 @@ on: jobs: run_all: - name: Validate tests for specific schema version + name: Validate tests using schema runs-on: ubuntu-latest steps: - name: Checkout repo uses: actions/checkout@v4 - - name: Validate tests for schema v0 + - name: Validate tests using the latest schema version run: | - cd test - python3 validate.py --schema-dir ./schemas/v0/ + ajv validate --spec=draft2020 \ + -s $(ls -1v schemas/*/*schema.json | tail -1) \ + -d 'tests/**/*.json' + working-directory: ./test diff --git a/test/validate.py b/test/validate.py deleted file mode 100755 index 398197330..000000000 --- a/test/validate.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/env python3 - -import argparse -import json -import jsonschema -import logging -import os -import sys - - -def parse_args(args): - parser = argparse.ArgumentParser( - prog='Test File Schema Validator', - description='Validate all of the test case JSON files against the JSON Schema file' - ) - - parser.add_argument('--schema-dir', - help='The directory containing the JSON schema file for the test cases (dir named according to a semantic version)') - parser.add_argument('--test-dir', - default='tests', - help='The base directory in which all test files are contained') - - # ignore the executable's name from the arg list, which is in the first position - return parser.parse_args(args[1:]) - -def main(args): - # initialize logging - logging.basicConfig(format='[%(asctime)s] [%(levelname)s] %(message)s', level=logging.INFO) - - # parse args - parsed_args = parse_args(args) - - # get schema file from the schema dir - schema_dir = parsed_args.schema_dir - schema_file_paths = os.listdir(path = schema_dir) - assert len(schema_file_paths) == 1, "there should only be 1 schema(s) definition file for a given schema version" - schema_file_path = os.path.join(schema_dir, schema_file_paths[0]) - logging.debug("schema file path: %s", schema_file_path) - schema_file = open(schema_file_path, encoding='utf-8', mode='r') - schema = json.load(schema_file) - - # iterate over all files in the test case file dir and validate against the schema - test_dir = parsed_args.test_dir - for (dirpath, dirnames, filenames) in os.walk(test_dir): - for test_filename in filenames: - test_file_path = os.path.join(dirpath, test_filename) - logging.debug("test file path: %s", test_file_path) - test_file = open(test_file_path, encoding='utf-8', mode='r') - logging.debug("verifying test case file: %s", test_file_path) - tests = json.load(test_file) - jsonschema.validate(tests, schema) - - test_file.close() - schema_file.close() - - logging.info("Completed verification of all test case files successfully") - -if __name__ == "__main__": - main(sys.argv) From 9f50b310d3578fd98a3ec85044151cd89d714a11 Mon Sep 17 00:00:00 2001 From: Elango Cheran Date: Wed, 1 May 2024 08:20:22 -0700 Subject: [PATCH 3/5] Insert omitted step to install JSON Schema validator tool --- .github/workflows/validate_tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/validate_tests.yml b/.github/workflows/validate_tests.yml index 8e441a7ac..e3facf804 100644 --- a/.github/workflows/validate_tests.yml +++ b/.github/workflows/validate_tests.yml @@ -14,6 +14,8 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v4 + - name: Install CLI tool for JSON Schema validation + run: npm install --global ajv-cli - name: Validate tests using the latest schema version run: | ajv validate --spec=draft2020 \ From 2511e3439078f1980eb5f34f8f3ad7470dc00d74 Mon Sep 17 00:00:00 2001 From: Elango Cheran Date: Thu, 2 May 2024 09:33:33 -0700 Subject: [PATCH 4/5] Apply suggestions from code review Co-authored-by: Eemeli Aro --- .github/workflows/validate_tests.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate_tests.yml b/.github/workflows/validate_tests.yml index e3facf804..b7b34b816 100644 --- a/.github/workflows/validate_tests.yml +++ b/.github/workflows/validate_tests.yml @@ -4,6 +4,8 @@ on: push: branches: - main + paths: + - test/** pull_request: branches: '**' @@ -17,8 +19,8 @@ jobs: - name: Install CLI tool for JSON Schema validation run: npm install --global ajv-cli - name: Validate tests using the latest schema version - run: | - ajv validate --spec=draft2020 \ - -s $(ls -1v schemas/*/*schema.json | tail -1) \ + run: > + ajv validate --spec=draft2020 + -s $(ls -1v schemas/*/*schema.json | tail -1) -d 'tests/**/*.json' working-directory: ./test From f80c0f95584e03acb42e3eaa4e4efa6528104ac7 Mon Sep 17 00:00:00 2001 From: Elango Cheran Date: Thu, 2 May 2024 15:13:37 -0700 Subject: [PATCH 5/5] Add constraint for runs on PRs based on modified paths --- .github/workflows/validate_tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/validate_tests.yml b/.github/workflows/validate_tests.yml index b7b34b816..7d8ed254e 100644 --- a/.github/workflows/validate_tests.yml +++ b/.github/workflows/validate_tests.yml @@ -8,6 +8,8 @@ on: - test/** pull_request: branches: '**' + paths: + - test/** jobs: run_all: