From b55846a398620df78b195bd3f9ab225dd126a62f Mon Sep 17 00:00:00 2001 From: Uladzislau Date: Mon, 11 Sep 2023 11:48:15 +0200 Subject: [PATCH 1/5] GH-211 Fixed issue for datasets and jobs with special characters in URL Signed-off-by: Uladzislau --- CHANGELOG.md | 5 ++- src/core/zowe/core_for_zowe_sdk/sdk_api.py | 12 ++++++ .../zowe/zos_files_for_zowe_sdk/files.py | 42 +++++++++---------- .../zowe/zos_jobs_for_zowe_sdk/jobs.py | 14 +++---- tests/unit/test_zos_files.py | 16 ++++--- tests/unit/test_zos_jobs.py | 7 +++- tests/unit/test_zowe_core.py | 17 ++++++++ 7 files changed, 76 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1982a99..dc3ec78c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,5 +4,8 @@ All notable changes to the Zowe Client Python SDK will be documented in this fil ## Recent Changes +- Bugfix: Fixed issue for datasets and jobs with special characters in URL [#211] (https://github.com/zowe/zowe-client-python-sdk/issues/211) + + - Feature: Added a CredentialManager class to securely retrieve values from credentials and manage multiple credential entries on Windows [#134](https://github.com/zowe/zowe-client-python-sdk/issues/134) -- Feature: Added method to load profile properties from environment variables \ No newline at end of file +- Feature: Added method to load profile properties from environment variables diff --git a/src/core/zowe/core_for_zowe_sdk/sdk_api.py b/src/core/zowe/core_for_zowe_sdk/sdk_api.py index 7234f3fb..486c2e68 100644 --- a/src/core/zowe/core_for_zowe_sdk/sdk_api.py +++ b/src/core/zowe/core_for_zowe_sdk/sdk_api.py @@ -10,6 +10,7 @@ Copyright Contributors to the Zowe Project. """ +import urllib from .exceptions import UnsupportedAuthType from .request_handler import RequestHandler from .session import Session, ISession @@ -60,3 +61,14 @@ def _create_custom_request_arguments(self): dictionary creation """ return self.request_arguments.copy() + + def _encode_uri_component(self, str_to_adjust): + """Adjust string to be correct in a URL + + Returns + ------- + adjusted_str + A string with special characters, acceptable for a URL + """ + + return urllib.parse.quote(str_to_adjust, safe="!~*'()") if str_to_adjust is not None else None diff --git a/src/zos_files/zowe/zos_files_for_zowe_sdk/files.py b/src/zos_files/zowe/zos_files_for_zowe_sdk/files.py index 4ca01ebe..9a5c3e54 100644 --- a/src/zos_files/zowe/zos_files_for_zowe_sdk/files.py +++ b/src/zos_files/zowe/zos_files_for_zowe_sdk/files.py @@ -117,7 +117,7 @@ def list_dsn(self, name_pattern, return_attributes= False): A JSON with a list of dataset names (and attributes if specified) matching the given pattern. """ custom_args = self._create_custom_request_arguments() - custom_args["params"] = {"dslevel": name_pattern} + custom_args["params"] = {"dslevel": self._encode_uri_component(name_pattern)} custom_args["url"] = "{}ds".format(self.request_endpoint) @@ -149,7 +149,7 @@ def list_dsn_members(self, dataset_name, member_pattern=None, for k,v in additional_parms.items(): url = "{}{}{}={}".format(url,separator,k,v) separator = '&' - custom_args['url'] = url + custom_args['url'] = self._encode_uri_component(url) custom_args["headers"]["X-IBM-Max-Items"] = "{}".format(limit) custom_args["headers"]["X-IBM-Attributes"] = attributes response_json = self.request_handler.perform_request("GET", custom_args) @@ -190,7 +190,7 @@ def copy_uss_to_dataset(self, from_filename, to_dataset_name, to_member_name=Non path_to_member = f"{to_dataset_name}({to_member_name})" if to_member_name else to_dataset_name custom_args = self._create_custom_request_arguments() custom_args['json'] = data - custom_args["url"] = "{}ds/{}".format(self.request_endpoint, path_to_member) + custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(path_to_member)) response_json = self.request_handler.perform_request("PUT", custom_args, expected_code=[200]) return response_json @@ -224,11 +224,11 @@ def copy_dataset_or_member(self,from_dataset_name,to_dataset_name,from_member_na data={ "request":"copy", - "from-dataset":{ - "dsn":from_dataset_name.strip(), + "from-dataset":{ + "dsn":from_dataset_name.strip(), "member":from_member_name - }, - "replace":replace + }, + "replace":replace } @@ -245,7 +245,7 @@ def copy_dataset_or_member(self,from_dataset_name,to_dataset_name,from_member_na custom_args = self._create_custom_request_arguments() custom_args['json'] = data - custom_args["url"] = "{}ds/{}".format(self.request_endpoint, path_to_member) + custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(path_to_member)) response_json = self.request_handler.perform_request("PUT", custom_args, expected_code=[200]) return response_json @@ -258,7 +258,7 @@ def get_dsn_content(self, dataset_name): A JSON with the contents of a given dataset """ custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}ds/{}".format(self.request_endpoint, dataset_name) + custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(dataset_name)) response_json = self.request_handler.perform_request("GET", custom_args) return response_json @@ -327,7 +327,7 @@ def create_data_set(self, dataset_name, options = {}): options[opt] = options["lrecl"] custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}ds/{}".format(self.request_endpoint, dataset_name) + custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(dataset_name)) custom_args["json"] = options response_json = self.request_handler.perform_request("POST", custom_args, expected_code = [201]) return response_json @@ -403,7 +403,7 @@ def create_default_data_set(self, dataset_name: str, default_type: str): "dirblk": 25 } - custom_args["url"] = "{}ds/{}".format(self.request_endpoint, dataset_name) + custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(dataset_name)) response_json = self.request_handler.perform_request("POST", custom_args, expected_code=[201]) return response_json @@ -438,7 +438,7 @@ def get_dsn_content_streamed(self, dataset_name): A raw socket response """ custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}ds/{}".format(self.request_endpoint, dataset_name) + custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(dataset_name)) raw_response = self.request_handler.perform_streamed_request("GET", custom_args) return raw_response @@ -457,7 +457,7 @@ def get_dsn_binary_content(self, dataset_name, with_prefixes=False): The contents of the dataset with no transformation """ custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}ds/{}".format(self.request_endpoint, dataset_name) + custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(dataset_name)) custom_args["headers"]["Accept"] = "application/octet-stream" if with_prefixes: custom_args["headers"]["X-IBM-Data-Type"] = 'record' @@ -481,7 +481,7 @@ def get_dsn_binary_content_streamed(self, dataset_name, with_prefixes=False): The raw socket response """ custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}ds/{}".format(self.request_endpoint, dataset_name) + custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(dataset_name)) custom_args["headers"]["Accept"] = "application/octet-stream" if with_prefixes: custom_args["headers"]["X-IBM-Data-Type"] = 'record' @@ -499,7 +499,7 @@ def write_to_dsn(self, dataset_name, data, encoding=_ZOWE_FILES_DEFAULT_ENCODING A JSON containing the result of the operation """ custom_args = self._create_custom_request_arguments() - custom_args["url"] = "{}ds/{}".format(self.request_endpoint, dataset_name) + custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(dataset_name)) custom_args["data"] = data custom_args['headers']['Content-Type'] = 'text/plain; charset={}'.format(encoding) response_json = self.request_handler.perform_request( @@ -573,7 +573,7 @@ def delete_data_set(self, dataset_name, volume=None, member_name=None): url = "{}ds/{}".format(self.request_endpoint, dataset_name) if volume is not None: url = "{}ds/-{}/{}".format(self.request_endpoint, volume, dataset_name) - custom_args["url"] = url + custom_args["url"] = self._encode_uri_component(url) response_json = self.request_handler.perform_request( "DELETE", custom_args, expected_code=[200, 202, 204]) return response_json @@ -700,7 +700,7 @@ def recall_migrated_dataset(self, dataset_name: str, wait=False): custom_args = self._create_custom_request_arguments() custom_args["json"] = data - custom_args["url"] = "{}ds/{}".format(self.request_endpoint, dataset_name) + custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(dataset_name)) response_json = self.request_handler.perform_request("PUT", custom_args, expected_code=[200]) return response_json @@ -733,7 +733,7 @@ def delete_migrated_data_set(self, dataset_name: str, purge=False, wait=False): custom_args = self._create_custom_request_arguments() custom_args["json"] = data - custom_args["url"] = "{}ds/{}".format(self.request_endpoint, dataset_name) + custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(dataset_name)) response_json = self.request_handler.perform_request("PUT", custom_args, expected_code=[200]) return response_json @@ -762,7 +762,7 @@ def migrate_data_set(self, dataset_name: str, wait=False): custom_args = self._create_custom_request_arguments() custom_args["json"] = data - custom_args["url"] = "{}ds/{}".format(self.request_endpoint, dataset_name) + custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(dataset_name)) response_json = self.request_handler.perform_request("PUT", custom_args, expected_code=[200]) return response_json @@ -793,7 +793,7 @@ def rename_dataset(self, before_dataset_name: str, after_dataset_name: str): custom_args = self._create_custom_request_arguments() custom_args["json"] = data - custom_args["url"] = "{}ds/{}".format(self.request_endpoint, after_dataset_name.strip()) + custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(after_dataset_name).strip()) response_json = self.request_handler.perform_request("PUT", custom_args, expected_code=[200]) return response_json @@ -839,7 +839,7 @@ def rename_dataset_member(self, dataset_name: str, before_member_name: str, afte custom_args = self._create_custom_request_arguments() custom_args['json'] = data - custom_args["url"] = "{}ds/{}".format(self.request_endpoint, path_to_member) + custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(path_to_member)) response_json = self.request_handler.perform_request("PUT", custom_args, expected_code=[200]) return response_json diff --git a/src/zos_jobs/zowe/zos_jobs_for_zowe_sdk/jobs.py b/src/zos_jobs/zowe/zos_jobs_for_zowe_sdk/jobs.py index 785a7228..0d508569 100644 --- a/src/zos_jobs/zowe/zos_jobs_for_zowe_sdk/jobs.py +++ b/src/zos_jobs/zowe/zos_jobs_for_zowe_sdk/jobs.py @@ -51,7 +51,7 @@ def get_job_status(self, jobname, jobid): """ custom_args = self._create_custom_request_arguments() job_url = "{}/{}".format(jobname, jobid) - request_url = "{}{}".format(self.request_endpoint, job_url) + request_url = "{}{}".format(self.request_endpoint, self._encode_uri_component(job_url)) custom_args["url"] = request_url response_json = self.request_handler.perform_request("GET", custom_args) return response_json @@ -78,7 +78,7 @@ def cancel_job(self, jobname: str, jobid: str, modify_version="2.0"): custom_args = self._create_custom_request_arguments() job_url = "{}/{}".format(jobname, jobid) - request_url = "{}{}".format(self.request_endpoint, job_url) + request_url = "{}{}".format(self.request_endpoint, self._encode_uri_component(job_url)) custom_args["url"] = request_url custom_args["json"] = { "request": "cancel", @@ -110,7 +110,7 @@ def delete_job(self, jobname, jobid, modify_version="2.0"): custom_args = self._create_custom_request_arguments() job_url = "{}/{}".format(jobname, jobid) - request_url = "{}{}".format(self.request_endpoint, job_url) + request_url = "{}{}".format(self.request_endpoint, self._encode_uri_component(job_url)) custom_args["url"] = request_url custom_args["headers"]["X-IBM-Job-Modify-Version"] = modify_version @@ -121,7 +121,7 @@ def _issue_job_request(self, req: dict, jobname: str, jobid: str, modify_version custom_args = self._create_custom_request_arguments() job_url = "{}/{}".format(jobname, jobid) - request_url = "{}{}".format(self.request_endpoint, job_url) + request_url = "{}{}".format(self.request_endpoint, self._encode_uri_component(job_url)) custom_args["url"] = request_url custom_args["json"] = { **req, @@ -317,7 +317,7 @@ def get_spool_files(self, correlator): """ custom_args = self._create_custom_request_arguments() job_url = "{}/files".format(correlator) - request_url = "{}{}".format(self.request_endpoint, job_url) + request_url = "{}{}".format(self.request_endpoint, self._encode_uri_component(job_url)) custom_args["url"] = request_url response_json = self.request_handler.perform_request("GET", custom_args) return response_json @@ -336,7 +336,7 @@ def get_jcl_text(self, correlator): """ custom_args = self._create_custom_request_arguments() job_url = "{}/files/JCL/records".format(correlator) - request_url = "{}{}".format(self.request_endpoint, job_url) + request_url = "{}{}".format(self.request_endpoint, self._encode_uri_component(job_url)) custom_args["url"] = request_url response_json = self.request_handler.perform_request("GET", custom_args) return response_json @@ -360,7 +360,7 @@ def get_spool_file_contents(self, correlator, id): """ custom_args = self._create_custom_request_arguments() job_url = "{}/files/{}/records".format(correlator, id) - request_url = "{}{}".format(self.request_endpoint, job_url) + request_url = "{}{}".format(self.request_endpoint, self._encode_uri_component(job_url)) custom_args["url"] = request_url response_json = self.request_handler.perform_request("GET", custom_args) return response_json diff --git a/tests/unit/test_zos_files.py b/tests/unit/test_zos_files.py index 1a147a2b..85d4027d 100644 --- a/tests/unit/test_zos_files.py +++ b/tests/unit/test_zos_files.py @@ -1,4 +1,5 @@ """Unit tests for the Zowe Python SDK z/OS Files package.""" +import re from unittest import TestCase, mock from zowe.zos_files_for_zowe_sdk import Files, exceptions @@ -74,7 +75,7 @@ def test_unmount_zFS_file_system(self, mock_send_request): @mock.patch('requests.Session.send') def test_list_dsn(self, mock_send_request): - """Test creating a zfs sends a request""" + """Test list DSN sends request""" mock_send_request.return_value = mock.Mock(headers={"Content-Type": "application/json"}, status_code=200) test_values = [ @@ -311,10 +312,10 @@ def test_rename_dataset_member_raises_exception(self): def test_rename_dataset_member_parametrized(self): """Test renaming a dataset member with different values""" test_values = [ - (('DSN', "MBROLD", "MBRNEW", "EXCLU"), True), - (('DSN', "MBROLD", "MBRNEW", "SHRW"), True), + (('DSN', "MBROLD$", "MBRNEW$", "EXCLU"), True), + (('DSN', "MBROLD#", "MBRNE#", "SHRW"), True), (('DSN', "MBROLD", "MBRNEW", "INVALID"), False), - (('DATA.SET.NAME', 'MEMBEROLD', 'MEMBERNEW'), True), + (('DATA.SET.@NAME', 'MEMBEROLD', 'MEMBERNEW'), True), (('DS.NAME', "MONAME", "MNNAME"), True), ] @@ -337,8 +338,11 @@ def test_rename_dataset_member_parametrized(self): files_test_profile.rename_dataset_member(*test_case[0]) custom_args = files_test_profile._create_custom_request_arguments() custom_args["json"] = data - custom_args["url"] = "https://mock-url.com:443/zosmf/restfiles/ds/{}({})".format( - test_case[0][0], test_case[0][2]) + ds_path = "{}({})".format(test_case[0][0], test_case[0][2]) + ds_path_adjusted = files_test_profile._encode_uri_component(ds_path) + self.assertNotRegex(ds_path_adjusted, r'[\$\@\#]') + self.assertRegex(ds_path_adjusted, r'[\(' + re.escape(test_case[0][2]) + r'\)]') + custom_args["url"] = "https://mock-url.com:443/zosmf/restfiles/ds/{}".format(ds_path_adjusted) files_test_profile.request_handler.perform_request.assert_called_once_with("PUT", custom_args, expected_code=[200]) else: diff --git a/tests/unit/test_zos_jobs.py b/tests/unit/test_zos_jobs.py index e64b12cb..3b683d36 100644 --- a/tests/unit/test_zos_jobs.py +++ b/tests/unit/test_zos_jobs.py @@ -81,7 +81,7 @@ def test_modified_version_error(self, mock_send_request): def test_cancel_job_modify_version_parameterized(self): """Test cancelling a job with different values sends the expected request""" test_values = [ - (("TESTJOB", "JOB00010", "1.0"), True), + (("TESTJOB", "JOB$0010", "1.0"), True), (("TESTJOBN", "JOB00011", "2.0"), True), (("TESTJOB", "JOB00012", "2"), False), (("TESTJOBN", "JOB00113", "3.0"), False), @@ -100,7 +100,10 @@ def test_cancel_job_modify_version_parameterized(self): "request": "cancel", "version": test_case[0][2], } - custom_args["url"] = "https://mock-url.com:443/zosmf/restjobs/jobs/{}/{}".format(test_case[0][0], test_case[0][1]) + job_url = "{}/{}".format(test_case[0][0], test_case[0][1]) + job_url_adjusted = jobs_test_object._encode_uri_component(job_url) + self.assertNotRegex(job_url_adjusted, r'\$') + custom_args["url"] = "https://mock-url.com:443/zosmf/restjobs/jobs/{}".format(job_url_adjusted) jobs_test_object.request_handler.perform_request.assert_called_once_with("PUT", custom_args, expected_code=[202, 200]) else: with self.assertRaises(ValueError) as e_info: diff --git a/tests/unit/test_zowe_core.py b/tests/unit/test_zowe_core.py index d63067ae..392f2ac7 100644 --- a/tests/unit/test_zowe_core.py +++ b/tests/unit/test_zowe_core.py @@ -125,6 +125,23 @@ def test_should_handle_token_auth(self): self.token_props["tokenType"] + "=" + self.token_props["tokenValue"], ) + def test_encode_uri_component(self): + """Test string is being adjusted to the correct URL parameter""" + + sdk_api = SdkApi(self.basic_props, self.default_url) + + actual_not_empty = sdk_api._encode_uri_component('MY.STRING@.TEST#.$HERE(MBR#NAME)') + expected_not_empty = 'MY.STRING%40.TEST%23.%24HERE(MBR%23NAME)' + self.assertEqual(actual_not_empty, expected_not_empty) + + actual_wildcard = sdk_api._encode_uri_component('GET.#DS.*') + expected_wildcard = 'GET.%23DS.*' + self.assertEqual(actual_wildcard, expected_wildcard) + + actual_none = sdk_api._encode_uri_component(None) + expected_none = None + self.assertEqual(actual_none, expected_none) + class TestRequestHandlerClass(unittest.TestCase): """RequestHandler class unit tests.""" From 76e6cb8937d12d85105faa46461cf3e388e895bb Mon Sep 17 00:00:00 2001 From: Maira Usman Date: Wed, 13 Sep 2023 22:16:26 +0500 Subject: [PATCH 2/5] added "black" to requirements.txt Signed-off-by: Maira Usman --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index dde23817..977fb9a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,4 +32,5 @@ zipp==3.4.0 -e ./src/zos_files -e ./src/zos_jobs -e ./src/zos_tso --e ./src/zosmf \ No newline at end of file +-e ./src/zosmf +black From 4d3e8603a9082f8ae01d77e29a801467752e3f19 Mon Sep 17 00:00:00 2001 From: Maira Usman Date: Wed, 13 Sep 2023 22:42:38 +0500 Subject: [PATCH 3/5] added black in requirements.txt Signed-off-by: Maira Usman --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 977fb9a2..be3afc15 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,6 +27,7 @@ Unidecode==1.2.0 urllib3==1.26.5 wheel zipp==3.4.0 +black -e ./src/core -e ./src/zos_console -e ./src/zos_files From f745f72319274b773fbf13c3c78471a0e4fc7ca8 Mon Sep 17 00:00:00 2001 From: Maira Usman Date: Wed, 13 Sep 2023 23:04:42 +0500 Subject: [PATCH 4/5] added "black " in requirements.txt Signed-off-by: Maira Usman --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index be3afc15..c8b28b18 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,4 +34,3 @@ black -e ./src/zos_jobs -e ./src/zos_tso -e ./src/zosmf -black From c664bc08aceb551a43a8efd0f592ef9172912138 Mon Sep 17 00:00:00 2001 From: Maira Usman Date: Wed, 13 Sep 2023 23:09:24 +0500 Subject: [PATCH 5/5] update vesrion for typing-extension in requirements.txt Signed-off-by: Maira Usman --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c8b28b18..b904d1a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ PyYAML==6.0.1 requests==2.31.0 six==1.15.0 snowballstemmer==2.1.0 -typing-extensions==3.7.4.3 +typing-extensions==4.0.1 Unidecode==1.2.0 urllib3==1.26.5 wheel