diff --git a/.vscode/settings.json b/.vscode/settings.json index 57c96298..5079d79f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,5 +9,13 @@ "python.testing.pytestArgs": ["tests"], "python.testing.pytestEnabled": true, "python.testing.unittestEnabled": false, - "rust-analyzer.linkedProjects": ["./src/secrets/Cargo.toml"], + "rust-analyzer.linkedProjects": [ + "./src/secrets/Cargo.toml" + ], + "python.analysis.extraPaths": [ + "./src/core", + "./src/zos_console", + "./src/zos_files", + "./src/zos_jobs" + ], } diff --git a/src/core/zowe/core_for_zowe_sdk/__init__.py b/src/core/zowe/core_for_zowe_sdk/__init__.py index 615b966f..58fc34a0 100644 --- a/src/core/zowe/core_for_zowe_sdk/__init__.py +++ b/src/core/zowe/core_for_zowe_sdk/__init__.py @@ -13,3 +13,18 @@ from .session import Session from .session_constants import * from .zosmf_profile import ZosmfProfile + +import logging +import os + +dirname = os.path.join(os.path.expanduser("~"), ".zowe/logs") + +if not os.path.isdir(dirname): + os.makedirs(dirname) + +logging.basicConfig( + filename=os.path.join(dirname, "python_sdk_logs.log"), + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + datefmt="%m/%d/%Y %I:%M:%S %p", +) diff --git a/src/core/zowe/core_for_zowe_sdk/config_file.py b/src/core/zowe/core_for_zowe_sdk/config_file.py index 581927ce..d873628c 100644 --- a/src/core/zowe/core_for_zowe_sdk/config_file.py +++ b/src/core/zowe/core_for_zowe_sdk/config_file.py @@ -21,6 +21,8 @@ import commentjson import requests +import logging + from .credential_manager import CredentialManager from .custom_warnings import ProfileNotFoundWarning, ProfileParsingWarning from .exceptions import ProfileNotFound @@ -71,6 +73,8 @@ class ConfigFile: jsonc: Optional[dict] = None _missing_secure_props: list = field(default_factory=list) + __logger = logging.getLogger(__name__) + @property def filename(self) -> str: if self.type == TEAM_CONFIG: @@ -101,11 +105,13 @@ def location(self, dirname: str) -> None: if os.path.isdir(dirname): self._location = dirname else: + self.__logger.error(f"given path {dirname} is not valid") raise FileNotFoundError(f"given path {dirname} is not valid") def init_from_file( self, validate_schema: Optional[bool] = True, + suppress_config_file_warnings: Optional[bool] = True, ) -> None: """ Initializes the class variable after @@ -118,7 +124,9 @@ def init_from_file( pass if self.filepath is None or not os.path.isfile(self.filepath): - warnings.warn(f"Config file does not exist at {self.filepath}") + if not suppress_config_file_warnings: + self.__logger.warning(f"Config file does not exist at {self.filepath}") + warnings.warn(f"Config file does not exist at {self.filepath}") return with open(self.filepath, encoding="UTF-8", mode="r") as fileobj: @@ -148,7 +156,8 @@ def validate_schema(self) -> None: path_schema_json = self.schema_path if path_schema_json is None: # check if the $schema property is not defined - warnings.warn(f"$schema property could not found") + self.__logger.warning(f"Could not find $schema property") + warnings.warn(f"Could not find $schema property") # validate the $schema property if path_schema_json: @@ -213,6 +222,7 @@ def get_profile( self.init_from_file(validate_schema) if profile_name is None and profile_type is None: + self.__logger.error(f"Failed to load profile '{profile_name}' because Could not find profile as both profile_name and profile_type is not set") raise ProfileNotFound( profile_name=profile_name, error_msg="Could not find profile as both profile_name and profile_type is not set.", @@ -250,7 +260,6 @@ def autodiscover_config_dir(self) -> None: break current_dir = os.path.dirname(current_dir) - raise FileNotFoundError(f"Could not find the file {self.filename}") def get_profilename_from_profiletype(self, profile_type: str) -> str: @@ -268,6 +277,7 @@ def get_profilename_from_profiletype(self, profile_type: str) -> str: try: profilename = self.defaults[profile_type] except KeyError: + self.__logger.warn(f"Given profile type '{profile_type}' has no default profilename") warnings.warn( f"Given profile type '{profile_type}' has no default profilename", ProfileParsingWarning, @@ -282,12 +292,14 @@ def get_profilename_from_profiletype(self, profile_type: str) -> str: if profile_type == temp_profile_type: return key except KeyError: + self.__logger.warning(f"Profile '{key}' has no type attribute") warnings.warn( f"Profile '{key}' has no type attribute", ProfileParsingWarning, ) # if no profile with matching type found, we raise an exception + self.__logger.error(f"No profile with matching profile_type '{profile_type}' found") raise ProfileNotFound( profile_name=profile_type, error_msg=f"No profile with matching profile_type '{profile_type}' found", @@ -334,6 +346,7 @@ def load_profile_properties(self, profile_name: str) -> dict: props = {**profile.get("properties", {}), **props} secure_fields.extend(profile.get("secure", [])) else: + self.__logger.warning(f"Profile {profile_name} not found") warnings.warn(f"Profile {profile_name} not found", ProfileNotFoundWarning) lst.pop() diff --git a/src/core/zowe/core_for_zowe_sdk/connection.py b/src/core/zowe/core_for_zowe_sdk/connection.py index fa97609c..2f764a99 100644 --- a/src/core/zowe/core_for_zowe_sdk/connection.py +++ b/src/core/zowe/core_for_zowe_sdk/connection.py @@ -9,7 +9,9 @@ Copyright Contributors to the Zowe Project. """ + from .exceptions import MissingConnectionArgs +import logging class ApiConnection: @@ -28,8 +30,11 @@ class ApiConnection: """ def __init__(self, host_url, user, password, ssl_verification=True): + logger = logging.getLogger(__name__) + """Construct an ApiConnection object.""" if not host_url or not user or not password: + logger.error("Missing connection argument") raise MissingConnectionArgs() self.host_url = host_url diff --git a/src/core/zowe/core_for_zowe_sdk/credential_manager.py b/src/core/zowe/core_for_zowe_sdk/credential_manager.py index 3459619c..b640d9b1 100644 --- a/src/core/zowe/core_for_zowe_sdk/credential_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/credential_manager.py @@ -9,12 +9,15 @@ Copyright Contributors to the Zowe Project. """ + import base64 import sys from typing import Optional import commentjson +import logging + from .constants import constants from .exceptions import SecureProfileLoadFailed @@ -27,6 +30,7 @@ class CredentialManager: secure_props = {} + __logger = logging.getLogger(__name__) @staticmethod def load_secure_props() -> None: @@ -49,6 +53,7 @@ def load_secure_props() -> None: return except Exception as exc: + CredentialManager.__logger.error(f"Fail to load secure profile {constants["ZoweServiceName"]}") raise SecureProfileLoadFailed(constants["ZoweServiceName"], error_msg=str(exc)) from exc secure_config: str @@ -75,7 +80,9 @@ def save_secure_props() -> None: if sys.platform == "win32": # Delete the existing credential CredentialManager._delete_credential(constants["ZoweServiceName"], constants["ZoweAccountName"]) - CredentialManager._set_credential(constants["ZoweServiceName"], constants["ZoweAccountName"], encoded_credential) + CredentialManager._set_credential( + constants["ZoweServiceName"], constants["ZoweAccountName"], encoded_credential + ) @staticmethod def _get_credential(service_name: str, account_name: str) -> Optional[str]: diff --git a/src/core/zowe/core_for_zowe_sdk/profile_manager.py b/src/core/zowe/core_for_zowe_sdk/profile_manager.py index 7c0c78d1..20a34a64 100644 --- a/src/core/zowe/core_for_zowe_sdk/profile_manager.py +++ b/src/core/zowe/core_for_zowe_sdk/profile_manager.py @@ -15,6 +15,7 @@ import warnings from copy import deepcopy from typing import Optional +import logging import jsonschema from deepmerge import always_merger @@ -57,10 +58,13 @@ def __init__(self, appname: str = "zowe", show_warnings: bool = True): self.project_config = ConfigFile(type=TEAM_CONFIG, name=appname) self.project_user_config = ConfigFile(type=USER_CONFIG, name=appname) + self.__logger = logging.getLogger(__name__) + self.global_config = ConfigFile(type=TEAM_CONFIG, name=GLOBAL_CONFIG_NAME) try: self.global_config.location = GLOBAL_CONFIG_LOCATION except Exception: + self.__logger.warning("Could not find Global Config Directory") warnings.warn( "Could not find Global Config Directory, please provide one.", ConfigNotFoundWarning, @@ -70,6 +74,7 @@ def __init__(self, appname: str = "zowe", show_warnings: bool = True): try: self.global_user_config.location = GLOBAL_CONFIG_LOCATION except Exception: + self.__logger.warning("Could not find Global User Config Directory") warnings.warn( "Could not find Global User Config Directory, please provide one.", ConfigNotFoundWarning, @@ -218,6 +223,7 @@ def load( check_missing_props: bool = True, validate_schema: Optional[bool] = True, override_with_env: Optional[bool] = False, + suppress_config_file_warnings: Optional[bool] = True, ) -> dict: """Load connection details from a team config profile. Returns @@ -236,7 +242,9 @@ def load( If `profile_type` is not base, then we will load properties from both `profile_type` and base profiles and merge them together. """ + if profile_name is None and profile_type is None: + self.__logger.error(f"Failed to load profile as both profile_name and profile_type are not set") raise ProfileNotFound( profile_name=profile_name, error_msg="Could not find profile as both profile_name and profile_type is not set.", @@ -254,12 +262,13 @@ def load( cfg_name = None cfg_schema = None cfg_schema_dir = None - + for cfg_layer in (self.project_user_config, self.project_config, self.global_user_config, self.global_config): if cfg_layer.profiles is None: try: - cfg_layer.init_from_file(validate_schema) + cfg_layer.init_from_file(validate_schema, suppress_config_file_warnings) except SecureProfileLoadFailed: + self.__logger.warning(f"Could not load secure properties for {cfg_layer.filepath}") warnings.warn( f"Could not load secure properties for {cfg_layer.filepath}", SecurePropsNotFoundWarning, @@ -314,6 +323,7 @@ def load( missing_props.add(item) if len(missing_props) > 0: + self.__logger.error(f"Failed to load secure values: {missing_props}") raise SecureValuesNotFound(values=missing_props) warnings.resetwarnings() @@ -366,6 +376,7 @@ def get_highest_priority_layer(self, json_path: str) -> Optional[ConfigFile]: highest_layer = layer if highest_layer is None: + self.__logger.error(f"Could not find a valid layer for {json_path}") raise FileNotFoundError(f"Could not find a valid layer for {json_path}") return highest_layer diff --git a/src/core/zowe/core_for_zowe_sdk/request_handler.py b/src/core/zowe/core_for_zowe_sdk/request_handler.py index 062bfd02..7f3ac1c6 100644 --- a/src/core/zowe/core_for_zowe_sdk/request_handler.py +++ b/src/core/zowe/core_for_zowe_sdk/request_handler.py @@ -12,6 +12,7 @@ import requests import urllib3 +import logging from .exceptions import InvalidRequestMethod, RequestFailed, UnexpectedStatus @@ -40,6 +41,7 @@ def __init__(self, session_arguments): self.session_arguments = session_arguments self.valid_methods = ["GET", "POST", "PUT", "DELETE"] self.__handle_ssl_warnings() + self.__logger = logging.getLogger(__name__) def __handle_ssl_warnings(self): """Turn off warnings if the SSL verification argument if off.""" @@ -104,6 +106,7 @@ def __validate_method(self): If the input request method is not supported """ if self.method not in self.valid_methods: + self.__logger.error(f"Invalid HTTP method input {self.method}") raise InvalidRequestMethod(self.method) def __send_request(self, stream=False): @@ -126,12 +129,14 @@ def __validate_response(self): # Automatically checks if status code is between 200 and 400 if self.response: if self.response.status_code not in self.expected_code: + self.__logger.error(f"The status code from z/OSMF was: {self.expected_code}\nExpected: {self.response.status_code}\nRequest output:{self.response.text}") raise UnexpectedStatus(self.expected_code, self.response.status_code, self.response.text) else: output_str = str(self.response.request.url) output_str += "\n" + str(self.response.request.headers) output_str += "\n" + str(self.response.request.body) output_str += "\n" + str(self.response.text) + self.__logger.error(f"HTTP Request has failed with status code {self.response.status_code}. \n {output_str}") raise RequestFailed(self.response.status_code, output_str) def __normalize_response(self): diff --git a/src/core/zowe/core_for_zowe_sdk/sdk_api.py b/src/core/zowe/core_for_zowe_sdk/sdk_api.py index 146f29aa..65751ce1 100644 --- a/src/core/zowe/core_for_zowe_sdk/sdk_api.py +++ b/src/core/zowe/core_for_zowe_sdk/sdk_api.py @@ -11,6 +11,7 @@ """ import urllib +import logging from . import session_constants from .exceptions import UnsupportedAuthType @@ -28,6 +29,8 @@ def __init__(self, profile, default_url): session = Session(profile) self.session: ISession = session.load() + self.logger = logging.getLogger(__name__) + self.default_service_url = default_url self.default_headers = { "Content-Type": "application/json", @@ -53,6 +56,7 @@ def __init__(self, profile, default_url): elif self.session.type == session_constants.AUTH_TYPE_TOKEN: self.default_headers["Cookie"] = f"{self.session.tokenType}={self.session.tokenValue}" else: + self.logger.error("Unsupported authorization type") raise UnsupportedAuthType(self.session.type) def _create_custom_request_arguments(self): diff --git a/src/core/zowe/core_for_zowe_sdk/session.py b/src/core/zowe/core_for_zowe_sdk/session.py index 54647f38..fe2ac533 100644 --- a/src/core/zowe/core_for_zowe_sdk/session.py +++ b/src/core/zowe/core_for_zowe_sdk/session.py @@ -15,6 +15,7 @@ from . import session_constants +import logging @dataclass class ISession: @@ -42,9 +43,12 @@ class Session: def __init__(self, props: dict) -> None: # set host and port + self.__logger = logging.getLogger(__name__) + if props.get("host") is not None: self.session: ISession = ISession(host=props.get("host")) else: + self.__logger.error("Host not supplied") raise Exception("Host must be supplied") # determine authentication type @@ -61,6 +65,7 @@ def __init__(self, props: dict) -> None: self.session.tokenValue = props.get("tokenValue") self.session.type = session_constants.AUTH_TYPE_BEARER else: + self.__logger.error("Authentication method not supplied") raise Exception("An authentication method must be supplied") # set additional parameters diff --git a/src/core/zowe/core_for_zowe_sdk/zosmf_profile.py b/src/core/zowe/core_for_zowe_sdk/zosmf_profile.py index 5ab3ea41..b877fdea 100644 --- a/src/core/zowe/core_for_zowe_sdk/zosmf_profile.py +++ b/src/core/zowe/core_for_zowe_sdk/zosmf_profile.py @@ -16,6 +16,8 @@ import yaml +import logging + from .connection import ApiConnection from .constants import constants from .exceptions import SecureProfileLoadFailed @@ -53,6 +55,7 @@ def __init__(self, profile_name): The name of the Zowe z/OSMF profile """ self.profile_name = profile_name + self.__logger = logging.getLogger(__name__) @property def profiles_dir(self): @@ -107,12 +110,14 @@ def __get_secure_value(self, name): def __load_secure_credentials(self): """Load secure credentials for a z/OSMF profile.""" if not HAS_KEYRING: + self.__logger.error(f"{self.profile_name} keyring module not installed") raise SecureProfileLoadFailed(self.profile_name, "Keyring module not installed") try: zosmf_user = self.__get_secure_value("user") zosmf_password = self.__get_secure_value("password") except Exception as e: + self.__logger.error(f"Failed to load secure profile '{self.profile_name}' because '{e}'") raise SecureProfileLoadFailed(self.profile_name, e) else: return (zosmf_user, zosmf_password) diff --git a/src/zos_files/zowe/zos_files_for_zowe_sdk/files.py b/src/zos_files/zowe/zos_files_for_zowe_sdk/files.py index efe9d539..f45ad57e 100644 --- a/src/zos_files/zowe/zos_files_for_zowe_sdk/files.py +++ b/src/zos_files/zowe/zos_files_for_zowe_sdk/files.py @@ -10,7 +10,6 @@ Copyright Contributors to the Zowe Project. """ - import os from zowe.core_for_zowe_sdk import SdkApi @@ -232,6 +231,7 @@ def copy_dataset_or_member( if enq in ("SHR", "SHRW", "EXCLU"): data["enq"] = enq else: + self.logger.error("Invalid value for enq.") raise ValueError("Invalid value for enq.") if volser: data["from-dataset"]["volser"] = volser @@ -270,6 +270,7 @@ def create_data_set(self, dataset_name, options={}): if options.get("like") is None: if options.get("primary") is None or options.get("lrecl") is None: + self.logger.error("If 'like' is not specified, you must specify 'primary' or 'lrecl'.") raise ValueError("If 'like' is not specified, you must specify 'primary' or 'lrecl'.") for opt in ( @@ -292,44 +293,51 @@ def create_data_set(self, dataset_name, options={}): ): if opt == "dsorg": if options.get(opt) is not None and options[opt] not in ("PO", "PS"): + self.logger.error(f"{opt} is not 'PO' or 'PS'.") raise KeyError - if opt == "alcunit": + elif opt == "alcunit": if options.get(opt) is None: options[opt] = "TRK" else: if options[opt] not in ("CYL", "TRK"): + self.logger.error(f"{opt} is not 'CYL' or 'TRK'.") raise KeyError - if opt == "primary": + elif opt == "primary": if options.get(opt) is not None: if options["primary"] > 16777215: + self.logger.error("Specified value exceeds limit.") raise ValueError - if opt == "secondary": + elif opt == "secondary": if options.get("primary") is not None: if options.get(opt) is None: options["secondary"] = int(options["primary"] / 10) if options["secondary"] > 16777215: + self.logger.error("Specified value exceeds limit.") raise ValueError - if opt == "dirblk": + elif opt == "dirblk": if options.get(opt) is not None: if options.get("dsorg") == "PS": if options["dirblk"] != 0: + self.logger.error("Can't allocate directory blocks for files.") raise ValueError elif options.get("dsorg") == "PO": if options["dirblk"] == 0: + self.logger.error("Can't allocate empty directory blocks.") raise ValueError - if opt == "recfm": + elif opt == "recfm": if options.get(opt) is None: options[opt] = "F" else: if options[opt] not in ("F", "FB", "V", "VB", "U", "FBA", "FBM", "VBA", "VBM"): + self.logger.error("Invalid record format.") raise KeyError - if opt == "blksize": + elif opt == "blksize": if options.get(opt) is None and options.get("lrecl") is not None: options[opt] = options["lrecl"] @@ -356,6 +364,7 @@ def create_default_data_set(self, dataset_name: str, default_type: str): """ if default_type not in ("partitioned", "sequential", "classic", "c", "binary"): + self.logger.error("Invalid type for default data set.") raise ValueError("Invalid type for default data set.") custom_args = self._create_custom_request_arguments() @@ -537,6 +546,7 @@ def upload_file_to_dsn(self, input_file, dataset_name, encoding=_ZOWE_FILES_DEFA with open(input_file, "rb") as in_file: response_json = self.write_to_dsn(dataset_name, in_file) else: + self.logger.error(f"File {input_file} not found.") raise FileNotFound(input_file) def write_to_uss(self, filepath_name, data, encoding=_ZOWE_FILES_DEFAULT_ENCODING): @@ -559,6 +569,7 @@ def upload_file_to_uss(self, input_file, filepath_name, encoding=_ZOWE_FILES_DEF with open(input_file, "r", encoding="utf-8") as in_file: response_json = self.write_to_uss(filepath_name, in_file) else: + self.logger.error(f"File {input_file} not found.") raise FileNotFound(input_file) def get_file_content_streamed(self, file_path, binary=False): @@ -610,10 +621,12 @@ def create_zFS_file_system(self, file_system_name, options={}): for key, value in options.items(): if key == "perms": if value < 0 or value > 777: + self.logger.error("Invalid Permissions Option.") raise exceptions.InvalidPermsOption(value) if key == "cylsPri" or key == "cylsSec": if value > constants.zos_file_constants["MaxAllocationQuantity"]: + self.logger.error("Maximum allocation quantity exceeded.") raise exceptions.MaxAllocationQuantityExceeded custom_args = self._create_custom_request_arguments() @@ -843,6 +856,7 @@ def rename_dataset_member(self, dataset_name: str, before_member_name: str, afte if enq in ("SHRW", "EXCLU"): data["enq"] = enq.strip() else: + self.logger.error("Invalid value for enq.") raise ValueError("Invalid value for enq.") custom_args = self._create_custom_request_arguments() diff --git a/src/zos_jobs/zowe/zos_jobs_for_zowe_sdk/jobs.py b/src/zos_jobs/zowe/zos_jobs_for_zowe_sdk/jobs.py index b9e13fca..ae4319e1 100644 --- a/src/zos_jobs/zowe/zos_jobs_for_zowe_sdk/jobs.py +++ b/src/zos_jobs/zowe/zos_jobs_for_zowe_sdk/jobs.py @@ -9,6 +9,7 @@ Copyright Contributors to the Zowe Project. """ + import os from zowe.core_for_zowe_sdk import SdkApi @@ -75,6 +76,7 @@ def cancel_job(self, jobname: str, jobid: str, modify_version="2.0"): A JSON object containing the result of the request execution """ if modify_version not in ("1.0", "2.0"): + self.logger.error('Modify version not accepted; Must be "1.0" or "2.0"') raise ValueError('Accepted values for modify_version: "1.0" or "2.0"') custom_args = self._create_custom_request_arguments() @@ -104,6 +106,7 @@ def delete_job(self, jobname, jobid, modify_version="2.0"): A JSON object containing the result of the request execution """ if modify_version not in ("1.0", "2.0"): + self.logger.error('Modify version not accepted; Must be "1.0" or "2.0"') raise ValueError('Accepted values for modify_version: "1.0" or "2.0"') custom_args = self._create_custom_request_arguments() @@ -145,6 +148,7 @@ def change_job_class(self, jobname: str, jobid: str, class_name: str, modify_ver A JSON object containing the result of the request execution """ if modify_version not in ("1.0", "2.0"): + self.logger.error('Accepted values for modify_version: "1.0" or "2.0"') raise ValueError('Accepted values for modify_version: "1.0" or "2.0"') response_json = self._issue_job_request({"class": class_name}, jobname, jobid, modify_version) @@ -168,6 +172,7 @@ def hold_job(self, jobname: str, jobid: str, modify_version="2.0"): A JSON object containing the result of the request execution """ if modify_version not in ("1.0", "2.0"): + self.logger.error('Accepted values for modify_version: "1.0" or "2.0"') raise ValueError('Accepted values for modify_version: "1.0" or "2.0"') response_json = self._issue_job_request({"request": "hold"}, jobname, jobid, modify_version) @@ -191,6 +196,7 @@ def release_job(self, jobname: str, jobid: str, modify_version="2.0"): A JSON object containing the result of the request execution """ if modify_version not in ("1.0", "2.0"): + self.logger.error('Modify version not accepted; Must be "1.0" or "2.0"') raise ValueError('Accepted values for modify_version: "1.0" or "2.0"') response_json = self._issue_job_request({"request": "release"}, jobname, jobid, modify_version) @@ -271,6 +277,7 @@ def submit_from_local_file(self, jcl_path): file_content = jcl_file.read() return self.submit_plaintext(file_content) else: + self.logger.error("Provided argument is not a file path {}".format(jcl_path)) raise FileNotFoundError("Provided argument is not a file path {}".format(jcl_path)) def submit_plaintext(self, jcl):