Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Validate that zowe.config.json file matches schema #192

Merged
merged 37 commits into from
Sep 27, 2023
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
883a0fc
moved the validation logic into ProfileManager
aadityasinha-dotcom Jun 26, 2023
e83fb65
added validation error warning
aadityasinha-dotcom Jun 26, 2023
ffc42e8
added jsonschema exceptions
aadityasinha-dotcom Jun 27, 2023
267f700
fixed indentation
aadityasinha-dotcom Jun 27, 2023
6465b67
raised error
aadityasinha-dotcom Jun 28, 2023
b93d818
fixed test
aadityasinha-dotcom Jun 28, 2023
69d983f
enhanced the validators.py and made some changes to validate schema
aadityasinha-dotcom Jun 29, 2023
f07ebec
fixed tests
aadityasinha-dotcom Jun 29, 2023
8f8d80c
fixed validator if there is not path, fixed raising exceptions for va…
aadityasinha-dotcom Jun 30, 2023
c2ae1ec
moved the validate_schema method to ConfigFile class
aadityasinha-dotcom Jul 6, 2023
6108ebf
changelog
aadityasinha-dotcom Jul 7, 2023
042a4ac
moved the validation logic to init_from_file so the validate_schema m…
aadityasinha-dotcom Jul 9, 2023
708da24
fixed test
aadityasinha-dotcom Jul 10, 2023
19ede33
changes
aadityasinha-dotcom Jul 10, 2023
dd0be07
simplified path_config_json
aadityasinha-dotcom Jul 12, 2023
5902381
added test for valide schema more test will be added soon
aadityasinha-dotcom Jul 17, 2023
8d65e9f
changed the opt_in flag to opt_out
aadityasinha-dotcom Jul 21, 2023
81c4b44
Merge branch 'main' into validate_schema
aadityasinha-dotcom Jul 30, 2023
8a94fdc
fix test
aadityasinha-dotcom Jul 31, 2023
600439a
added test for invalid schema
aadityasinha-dotcom Aug 1, 2023
7b2673b
Merge branch 'main' into validate_schema
aadityasinha-dotcom Aug 5, 2023
e01ffb0
refactored the code
aadityasinha-dotcom Aug 10, 2023
0030a10
added test for invalid schema and updated requirements.txt
aadityasinha-dotcom Aug 11, 2023
8edec11
added test for internet URI
aadityasinha-dotcom Aug 17, 2023
6f34785
did some changes, added unit test for internet URI
aadityasinha-dotcom Aug 21, 2023
cba960c
few changes added file protocol, fixed tests
aadityasinha-dotcom Aug 28, 2023
535ca88
Merge branch 'main' into validate_schema
aadityasinha-dotcom Aug 30, 2023
c0b9d9b
Merge branch 'main' into validate_schema
aadityasinha-dotcom Aug 31, 2023
6917e60
removed verify parameter
aadityasinha-dotcom Sep 4, 2023
fc5aa3a
removed config_type parameter
aadityasinha-dotcom Sep 13, 2023
26e997f
Merge branch 'main' into validate_schema
aadityasinha-dotcom Sep 13, 2023
1ba06a2
Merge branch 'main' into validate_schema
aadityasinha-dotcom Sep 14, 2023
561b370
fix tests
aadityasinha-dotcom Sep 19, 2023
f749d60
attempt to add the entire env/lib dir
zFernand0 Sep 27, 2023
e355a7c
cannot assume that env/lib is the reason why the merging didn't work
zFernand0 Sep 27, 2023
b824ac0
try to load jsonschema
zFernand0 Sep 27, 2023
a7e61a9
Merge branch 'main' into validate_schema
zFernand0 Sep 27, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/core/zowe/core_for_zowe_sdk/config_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ class ConfigFile:
4. Contents of the file.
4.1 Profiles
4.2 Defaults
4.3 Schema Property
5. Secure Properties associated with the file.
"""

Expand All @@ -77,6 +78,7 @@ class ConfigFile:
_location: Optional[str] = None
profiles: Optional[dict] = None
defaults: Optional[dict] = None
schema_property: Optional[dict] = None
secure_props: Optional[dict] = None
_missing_secure_props: list = field(default_factory=list)

Expand All @@ -100,6 +102,10 @@ def filepath(self) -> Optional[str]:
@property
def location(self) -> Optional[str]:
return self._location

@property
def schema_path(self) -> Optional[str]:
return self.schema_property

@location.setter
def location(self, dirname: str) -> None:
Expand All @@ -120,6 +126,7 @@ def init_from_file(self) -> None:
profile_jsonc = commentjson.load(fileobj)
aadityasinha-dotcom marked this conversation as resolved.
Show resolved Hide resolved

self.profiles = profile_jsonc.get("profiles", {})
self.schema_property = profile_jsonc.get("$schema", None)
aadityasinha-dotcom marked this conversation as resolved.
Show resolved Hide resolved
self.defaults = profile_jsonc.get("defaults", {})

# loading secure props is done in load_profile_properties
Expand Down
58 changes: 58 additions & 0 deletions src/core/zowe/core_for_zowe_sdk/profile_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@

import os.path
import warnings
import jsonschema
from typing import Optional

from .config_file import ConfigFile, Profile
from .validators import validate_config_json
from .custom_warnings import (
ConfigNotFoundWarning,
ProfileNotFoundWarning,
Expand Down Expand Up @@ -169,12 +171,59 @@ def get_profile(
)
finally:
return cfg_profile

@staticmethod
def validate_schema(
cfg: ConfigFile,
path_config_json: str,
opt_in: Optional[bool] = True,
) -> str:
"""
Get the $schema_property from the config and load the schema

Returns
-------
file_path to the $schema property
"""

path_schema_json = None
try:
path_schema_json = cfg.schema_path
if path_schema_json is None: # check if the $schema property is not defined
warnings.warn(
f"$schema property could not found"
)

# validate the $schema property
if path_schema_json and opt_in:
validate_config_json(path_config_json, path_schema_json)
aadityasinha-dotcom marked this conversation as resolved.
Show resolved Hide resolved
except jsonschema.exceptions.ValidationError as exc:
raise jsonschema.exceptions.ValidationError(
f"Instance was invalid under the provided $schema property, {exc}"
)
except jsonschema.exceptions.SchemaError as exc:
raise jsonschema.exception.SchemaError(
f"The provided schema is invalid, {exc}"
)
except jsonschema.exceptions.UndefinedTypeCheck as exc:
raise jsonschema.exceptions.UndefinedTypeCheck(
f"A type checker was asked to check a type it did not have registered, {exc}"
)
except jsonschema.exceptions.UnknownType as exc:
raise jsonschema.exceptions.UnknownType(
f"Unknown type is found in {path_schema_json}, exc"
)
except jsonschema.exceptions.FormatError as exc:
raise jsonschema.exceptions.FormatError(
f"Validating a format {path_config_json} failed for {path_schema_json}, {exc}"
)

def load(
self,
profile_name: Optional[str] = None,
profile_type: Optional[str] = None,
check_missing_props: bool = True,
opt_in: Optional[bool] = True,
) -> dict:
"""Load connection details from a team config profile.
Returns
Expand Down Expand Up @@ -209,6 +258,7 @@ def load(
"Global Config": self.global_config,
}
profile_props: dict = {}
schema_path = None

missing_secure_props = [] # track which secure props were not loaded

Expand All @@ -224,6 +274,14 @@ def load(
) # Define profile name that will be merged from other layers
profile_props = {**profile_loaded.data, **profile_props}

# Validating $schema property for all the layers
if cfg._location and config_type in ("Project Config", "Global Config"):
path_config_json = cfg._location + "/zowe.config.json"
self.validate_schema(cfg, path_config_json, opt_in)
elif cfg._location and config_type in ("Project User Config", "Global User Config"):
path_config_json = cfg._location + "/zowe.config.user.json"
self.validate_schema(cfg, path_config_json, opt_in)
aadityasinha-dotcom marked this conversation as resolved.
Show resolved Hide resolved

missing_secure_props.extend(profile_loaded.missing_secure_props)

if i == 1 and profile_props:
Expand Down
20 changes: 17 additions & 3 deletions src/core/zowe/core_for_zowe_sdk/validators.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@

import commentjson
from jsonschema import validate
import os
import requests


def validate_config_json(path_config_json: str, path_schema_json: str):
Expand All @@ -31,10 +33,22 @@ def validate_config_json(path_config_json: str, path_schema_json: str):
Provides details if config.json doesn't match schema.json, otherwise it returns None.
"""

# checks if the path_schema_json point to an internet URI and download the schema using the URI
if path_schema_json[:8] in ("https://", "http://"):
aadityasinha-dotcom marked this conversation as resolved.
Show resolved Hide resolved
response = requests.get(path_schema_json)
schema_data = response.content.decode("utf-8")
schema_json = commentjson.load(schema_data)

# checks if the path_schema_json is a file
elif os.path.isfile(path_schema_json):
with open(path_schema_json) as file:
schema_json = commentjson.load(file)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

would it be ok to add the file:// protocol here?

    elif os.path.isfile(path_schema_json) or path_schema_json.startswith("file://"):
        with open(path_schema_json.replace("file://", "")) as file:
            schema_json = commentjson.load(file)

I'm afraid that without it, the below $schema won't work:

    // "$schema": "./zowe.schema.json",
    "$schema": "file:///root/gh/zowe/zowe-client-python-sdk/zowe.schema.json",


# if there is no path_schema_json it will return None
else:
return None

with open(path_config_json) as file:
config_json = commentjson.load(file)
aadityasinha-dotcom marked this conversation as resolved.
Show resolved Hide resolved

with open(path_schema_json) as file:
schema_json = commentjson.load(file)

return validate(instance=config_json, schema=schema_json)