diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 1148883..c7ca99a 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -11,30 +11,25 @@ on: jobs: check-semantic-version: if: "!contains(github.event.head_commit.message, 'skipci')" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: octue/check-semantic-version@1.0.0.beta-9 - with: - path: setup.py - breaking_change_indicated_by: minor + uses: octue/workflows/.github/workflows/check-semantic-version.yml@main + with: + path: setup.py + breaking_change_indicated_by: minor run-tests: if: "!contains(github.event.head_commit.message, 'skipci')" runs-on: ubuntu-latest env: - USING_COVERAGE: '3.8' + USING_COVERAGE: '3.9' strategy: matrix: - python: [3.8] + python: ['3.8', '3.9'] steps: - name: Checkout Repository uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} @@ -45,10 +40,11 @@ jobs: run: tox - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: files: coverage.xml fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} publish: if: "!contains(github.event.head_commit.message, 'skipci')" @@ -62,9 +58,9 @@ jobs: uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: "3.9" - name: Make package run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 59c262a..667b70f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,16 +12,16 @@ jobs: if: "github.event.pull_request.merged == true" runs-on: ubuntu-latest env: - USING_COVERAGE: '3.8' + USING_COVERAGE: '3.9' strategy: matrix: - python: [ 3.8 ] + python: ['3.8', '3.9'] steps: - name: Checkout Repository uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} @@ -32,10 +32,11 @@ jobs: run: tox - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: files: coverage.xml fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} release: needs: run-tests @@ -68,7 +69,7 @@ jobs: uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.8 diff --git a/.github/workflows/update-pull-request.yml b/.github/workflows/update-pull-request.yml index 1efc58f..d77ca38 100644 --- a/.github/workflows/update-pull-request.yml +++ b/.github/workflows/update-pull-request.yml @@ -1,30 +1,18 @@ # This workflow updates the pull request description with an auto-generated section containing the categorised commit -# message headers of the commits since the last pull request merged into main. The auto generated section is enveloped -# between two comments: "" and "". Anything -# outside these in the description is left untouched. Auto-generated updates can be skipped for a commit if +# message headers of the pull request's commits. The auto generated section is enveloped between two comments: +# "" and "". Anything outside these in the +# description is left untouched. Auto-generated updates can be skipped for a commit if # "" is added to the pull request description. name: update-pull-request -# Only trigger for pull requests into main branch. -on: - pull_request: - branches: - - main +on: [pull_request] jobs: description: - if: "!contains(github.event.pull_request.body, '')" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: octue/generate-pull-request-description@1.0.0.beta-2 - id: pr-description - with: - pull_request_url: ${{ github.event.pull_request.url }} - api_token: ${{ secrets.GITHUB_TOKEN }} - - name: Update pull request body - uses: riskledger/update-pr-description@v2 - with: - body: ${{ steps.pr-description.outputs.pull_request_description }} - token: ${{ secrets.GITHUB_TOKEN }} + uses: octue/workflows/.github/workflows/generate-pull-request-description.yml@main + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + permissions: + contents: read + pull-requests: write diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d348530..6957616 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,17 +41,16 @@ repos: - id: pydocstyle - repo: https://github.com/thclark/pre-commit-sphinx - rev: 0.0.1 + rev: 0.0.3 hooks: - id: build-docs language_version: python3 additional_dependencies: - - 'Sphinx>=2,<3' - - 'sphinx-rtd-theme==0.5.0' - - 'sphinx-tabs==1.2.1' - - 'sphinx-charts==0.0.4' - - 'scipy~=1.5.2' - - 'jsonschema~=3.2.0' + - 'Sphinx' + - 'sphinx-rtd-theme' + - 'sphinx-tabs' + - 'sphinx-charts' + - 'jsonschema' - repo: https://github.com/windpioneers/pre-commit-hooks rev: 0.0.5 diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..71a0bcb --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,16 @@ +# Read the Docs configuration file for Sphinx projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.10" + +sphinx: + configuration: docs/source/conf.py + +python: + install: + - requirements: docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt index 075a2e5..6b1cd11 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,8 +1,6 @@ - # Required by the python script for building documentation -Sphinx>=2,<3 -sphinx-rtd-theme==0.5.0 -sphinx-tabs==1.2.1 -sphinx-charts==0.0.4 -scipy~=1.5.2 -jsonschema~=3.2.0 +Sphinx +sphinx-rtd-theme +sphinx-tabs +sphinx-charts +jsonschema diff --git a/docs/source/_ext/sphinx_accordion/accordion.py b/docs/source/_ext/sphinx_accordion/accordion.py index cada02d..71d934c 100644 --- a/docs/source/_ext/sphinx_accordion/accordion.py +++ b/docs/source/_ext/sphinx_accordion/accordion.py @@ -246,7 +246,7 @@ def setup(app): if 'add_script_file' in dir(app): app.add_script_file(path) else: - app.add_javascript(path) + app.add_js_file(path) app.connect('html-page-context', update_context) app.connect('build-finished', copy_assets) diff --git a/setup.py b/setup.py index a7e9c39..f5fd9d7 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ setup( name="twined", - version="0.5.3", + version="0.5.4", py_modules=[], install_requires=["jsonschema ~= 4.4.0", "python-dotenv"], url="https://www.github.com/octue/twined", @@ -33,6 +33,7 @@ "Topic :: Software Development :: Libraries :: Python Modules", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Operating System :: OS Independent", ], python_requires=">=3.6", diff --git a/tests/test_children.py b/tests/test_children.py index 0631dbb..8dde688 100644 --- a/tests/test_children.py +++ b/tests/test_children.py @@ -5,24 +5,22 @@ class TestChildrenTwine(BaseTestCase): - """Tests related to the twine itself - ensuring that valid and invalid - `children` entries in a twine file work as expected - """ + """Tests ensuring that valid and invalid `children` entries in a twine file work as expected.""" def test_invalid_children_dict_not_array(self): - """Ensures InvalidTwine exceptions are raised when instantiating twines where `children` entry is incorrectly - specified as a dict, not an array + """Ensure that `InvalidTwine` exceptions are raised when instantiating twines where `children` entry is + incorrectly specified as a dict, not an array. """ with self.assertRaises(exceptions.InvalidTwine): Twine(source="""{"children": {}}""") def test_invalid_children_no_key(self): - """Ensures InvalidTwine exceptions are raised when instantiating twines where a child - is specified without the required `key` field + """Ensure that `InvalidTwine` exceptions are raised when instantiating twines where a child is specified without + the required `key` field. """ source = """ { - "children": [{"purpose": "The purpose.", "notes": "Here are some notes.", "filters": "tags:gis"}] + "children": [{"purpose": "The purpose.", "notes": "Here are some notes."}] } """ @@ -33,7 +31,7 @@ def test_valid_children(self): """Ensures that a twine with one child can be instantiated correctly.""" source = """ { - "children": [{"key": "gis", "purpose": "The purpose.", "notes": "Some notes.", "filters": "tags:gis"}] + "children": [{"key": "gis", "purpose": "The purpose.", "notes": "Some notes."}] } """ self.assertEqual(len(Twine(source=source).children), 1) @@ -49,7 +47,7 @@ class TestChildrenValidation(BaseTestCase): VALID_TWINE_WITH_CHILDREN = """ { - "children": [{"key": "gis", "purpose": "The purpose", "notes": "Some notes.", "filters": "tags:gis"}] + "children": [{"key": "gis", "purpose": "The purpose", "notes": "Some notes."}] } """ diff --git a/tox.ini b/tox.ini index 57f36f6..1a36ae9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py38} +envlist = {py38,py39} [testenv] setenv = diff --git a/twined/schema/__init__.py b/twined/schema/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/twined/schema/children_schema.json b/twined/schema/children_schema.json deleted file mode 100644 index 1da053d..0000000 --- a/twined/schema/children_schema.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "type": "array", - "items": { - "type": "object", - "properties": { - "key": { - "description": "A textual key identifying a group of child twins", - "type": "string" - }, - "id": { - "description": "The universally unique ID (UUID) of the running child twin", - "type": "string" - }, - "backend": { - "description": "The backend running the child.", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "GCP Pub/Sub", - "properties": { - "name": { - "description": "Type of backend (in this case, it can only be GCPPubSubBackend)", - "type": "string", - "pattern": "^(GCPPubSubBackend)$" - }, - "project_name": { - "description": "Name of the Google Cloud Platform (GCP) project the child exists in.", - "type": "string" - } - }, - "required": ["name", "project_name"] - } - ] - } - }, - "required": ["key", "id", "backend"] - } -} diff --git a/twined/schema/manifest_schema.json b/twined/schema/manifest_schema.json deleted file mode 100644 index abb9870..0000000 --- a/twined/schema/manifest_schema.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "$defs": { - "tags": { - "description": "Key-value tags associated with the object.", - "type": "object" - }, - "labels": { - "description": "Textual labels associated with the object", - "type": "array", - "items": { - "type": "string" - } - } - }, - "type": "object", - "properties": { - "id": { - "description": "ID of the manifest, typically a uuid", - "type": "string" - }, - "datasets": { - "type": "object", - "patternProperties": { - ".+": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "id": { - "description": "ID of the dataset, typically a uuid", - "type": "string" - }, - "name": { - "description": "Name of the dataset (the same as its key in the 'datasets' field).", - "type": "string" - }, - "tags": { - "$ref": "#/$defs/tags" - }, - "labels": { - "$ref": "#/$defs/labels" - }, - "files": { - "type": "array", - "items": { - "oneOf": [ - { - "type": "object", - "properties": { - "id": { - "description": "A file id", - "type": "string" - }, - "path": { - "description": "Path at which the file can be found", - "type": "string" - }, - "timestamp": { - "oneOf": [ - { - "description": "A posix based timestamp associated with the file. This may, but need not be, the created or modified time. ", - "type": "number" - }, - { - "description": "A posix based timestamp associated with the file. This may, but need not be, the created or modified time. ", - "type": "null" - } - ] - }, - "tags": { - "$ref": "#/$defs/tags" - }, - "labels": { - "$ref": "#/$defs/labels" - } - }, - "required": [ - "id", - "path", - "timestamp", - "tags", - "labels" - ] - }, - { - "type": "string" - } - ] - } - } - }, - "required": [ - "id", - "name", - "tags", - "labels", - "files" - ] - } - ] - } - } - } - }, - "required": ["id", "datasets"] -} diff --git a/twined/twine.py b/twined/twine.py index ae0236c..b703b4e 100644 --- a/twined/twine.py +++ b/twined/twine.py @@ -1,10 +1,18 @@ +import importlib.metadata import json as jsonlib import logging import os -import pkg_resources from dotenv import load_dotenv from jsonschema import ValidationError, validate as jsonschema_validate + +try: + # python < 3.9 + import importlib_resources +except ModuleNotFoundError: + # python >= 3.9 + import importlib.resources as importlib_resources + from . import exceptions from .utils import load_json, trim_suffix @@ -32,6 +40,10 @@ ) +CHILDREN_SCHEMA = "https://jsonschema.registry.octue.com/octue/children/0.1.0.json" +MANIFEST_SCHEMA = "https://jsonschema.registry.octue.com/octue/manifest/0.1.0.json" + + class Twine: """Twine class manages validation of inputs and outputs to/from a data service, based on spec in a 'twine' file. @@ -92,33 +104,32 @@ def _get_schema(self, strand): if strand == "twine": # The data is a twine. A twine *contains* schema, but we also need to verify that it matches a certain # schema itself. The twine schema is distributed with this packaged to ensure version consistency... - schema_path = "schema/twine_schema.json" + return jsonlib.loads( + importlib_resources.files("twined.schema").joinpath("twine_schema.json").read_text(encoding="utf-8") + ) - elif strand in CHILDREN_STRANDS: + if strand in CHILDREN_STRANDS: # The data is a list of children. The "children" strand of the twine describes matching criteria for # the children, not the schema of the "children" data, which is distributed with this package to ensure # version consistency... - schema_path = "schema/children_schema.json" + return {"$ref": CHILDREN_SCHEMA} - elif strand in MANIFEST_STRANDS: + if strand in MANIFEST_STRANDS: # The data is a manifest of files. The "*_manifest" strands of the twine describe matching criteria used to # filter files appropriate for consumption by the digital twin, not the schema of the manifest data, which # is distributed with this package to ensure version consistency... - schema_path = "schema/manifest_schema.json" - - else: - if strand not in SCHEMA_STRANDS: - raise exceptions.UnknownStrand(f"Unknown strand {strand}. Try one of {ALL_STRANDS}.") + return {"$ref": MANIFEST_SCHEMA} - # Get schema from twine.json file. - schema_key = strand + "_schema" + if strand not in SCHEMA_STRANDS: + raise exceptions.UnknownStrand(f"Unknown strand {strand}. Try one of {ALL_STRANDS}.") - try: - return getattr(self, schema_key) - except AttributeError: - raise exceptions.StrandNotFound(f"Cannot validate - no {schema_key} strand in the twine") + # Get schema from twine.json file. + schema_key = strand + "_schema" - return jsonlib.loads(pkg_resources.resource_string("twined", schema_path)) + try: + return getattr(self, schema_key) + except AttributeError: + raise exceptions.StrandNotFound(f"Cannot validate - no {schema_key} strand in the twine") def _validate_against_schema(self, strand, data): """Validate data against a schema, raises exceptions of type InvalidJson if not compliant. @@ -143,7 +154,7 @@ def _validate_against_schema(self, strand, data): def _validate_twine_version(self, twine_file_twined_version): """Validate that the installed version is consistent with an optional version specification in the twine file.""" - installed_twined_version = pkg_resources.get_distribution("twined").version + installed_twined_version = importlib.metadata.version("twined") logger.debug( "Twine versions... %s installed, %s specified in twine", installed_twined_version, twine_file_twined_version )