Fix upload file to USS and integration tests

Signed-off-by: Timothy Johnson <[email protected]>

.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "ms-python.black-formatter",
+        "ms-python.isort",
+        "ms-python.vscode-pylance"
+    ]
+}

.vscode/settings.json

@@ -1,13 +1,13 @@
 {
-    "editor.formatOnSave": true,
-    "python.formatting.provider": "black",
-    "rust-analyzer.linkedProjects": ["./src/secrets/Cargo.toml"],
     "[python]": {
         "editor.codeActionsOnSave": {
-            "source.organizeImports": true
-        }
+            "source.organizeImports": "explicit"
+        },
+        "editor.defaultFormatter": "ms-python.black-formatter",
+        "editor.formatOnSave": true
     },
     "python.testing.pytestArgs": ["tests"],
+    "python.testing.pytestEnabled": true,
     "python.testing.unittestEnabled": false,
-    "python.testing.pytestEnabled": true
+    "rust-analyzer.linkedProjects": ["./src/secrets/Cargo.toml"],
 }

CHANGELOG.md

@@ -2,6 +2,19 @@
 
 All notable changes to the Zowe Client Python SDK will be documented in this file.
 
+## Recent Changes
+
+### Bug Fixes
+
+- Fixed `Files.download_dsn` and `Files.download_binary_dsn` failing to write contents to disk [#179](https://github.com/zowe/zowe-client-python-sdk/issues/179)
+- Fixed `Files.delete_data_set` and `Files.list_dsn_members` so they encode URLs correctly
+- Fixed loading environment variables when there is no schema file in current directory
+
+### Enhancements
+
+- Added method `Files.download_uss` to download USS files to disk
+- Added support to `Tso` class for loading TSO profile properties
+
 ## `1.0.0-dev13`
 
 ### Bug Fixes

src/core/zowe/core_for_zowe_sdk/config_file.py

@@ -152,11 +152,9 @@ def validate_schema(self) -> None:
 
         # validate the $schema property
         if path_schema_json:
-            validate_config_json(self.jsonc, path_schema_json, cwd = self.location)
+            validate_config_json(self.jsonc, path_schema_json, cwd=self.location)
 
-    def schema_list(
-        self,
-    ) -> list:
+    def schema_list(self, cwd=None) -> list:
         """
         Loads the schema properties
         in a sorted order according to the priority
@@ -180,7 +178,7 @@ def schema_list(
                 schema_json = json.load(f)
 
         elif not os.path.isabs(schema):
-            schema = os.path.join(self.location, schema)
+            schema = os.path.join(self.location or cwd, schema)
             with open(schema) as f:
                 schema_json = json.load(f)
         else:

src/core/zowe/core_for_zowe_sdk/profile_manager.py

@@ -114,9 +114,7 @@ def config_filepath(self) -> Optional[str]:
         return self.project_config.filepath
 
     @staticmethod
-    def get_env(
-        cfg: ConfigFile,
-    ) -> dict:
+    def get_env(cfg: ConfigFile, cwd=None) -> dict:
         """
         Maps the env variables to the profile properties
 
@@ -127,7 +125,7 @@ def get_env(
             Containing profile properties from env variables (prop: value)
         """
 
-        props = cfg.schema_list()
+        props = cfg.schema_list(cwd)
         if props == []:
             return {}
 
@@ -255,6 +253,7 @@ def load(
         profiles_merged: dict = {}
         cfg_name = None
         cfg_schema = None
+        cfg_schema_dir = None
 
         for cfg_layer in (self.project_user_config, self.project_config, self.global_user_config, self.global_config):
             if cfg_layer.profiles is None:
@@ -272,6 +271,7 @@ def load(
                 cfg_name = cfg_layer.name
             if not cfg_schema and cfg_layer.schema_property:
                 cfg_schema = cfg_layer.schema_property
+                cfg_schema_dir = cfg_layer._location
 
         usrProject = self.project_user_config.profiles or {}
         project = self.project_config.profiles or {}
@@ -299,7 +299,7 @@ def load(
             missing_secure_props.extend(profile_loaded.missing_secure_props)
 
         if override_with_env:
-            env_var = {**self.get_env(cfg)}
+            env_var = {**self.get_env(cfg, cfg_schema_dir)}
 
         if profile_type != BASE_PROFILE:
             profile_props = {

src/zos_files/zowe/zos_files_for_zowe_sdk/files.py

@@ -12,7 +12,6 @@
 
 
 import os
-import shutil
 
 from zowe.core_for_zowe_sdk import SdkApi
 from zowe.core_for_zowe_sdk.exceptions import FileNotFound
@@ -140,12 +139,8 @@ def list_dsn_members(self, dataset_name, member_pattern=None, member_start=None,
             additional_parms["start"] = member_start
         if member_pattern is not None:
             additional_parms["pattern"] = member_pattern
-        url = "{}ds/{}/member".format(self.request_endpoint, dataset_name)
-        separator = "?"
-        for k, v in additional_parms.items():
-            url = "{}{}{}={}".format(url, separator, k, v)
-            separator = "&"
-        custom_args["url"] = self._encode_uri_component(url)
+        custom_args["params"] = additional_parms
+        custom_args["url"] = "{}ds/{}/member".format(self.request_endpoint, self._encode_uri_component(dataset_name))
         custom_args["headers"]["X-IBM-Max-Items"] = "{}".format(limit)
         custom_args["headers"]["X-IBM-Attributes"] = attributes
         response_json = self.request_handler.perform_request("GET", custom_args)
@@ -443,13 +438,13 @@ def get_dsn_content_streamed(self, dataset_name):
 
         Returns
         -------
-        raw
-            A raw socket response
+        response
+            A response object from the requests library
         """
         custom_args = self._create_custom_request_arguments()
         custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(dataset_name))
-        raw_response = self.request_handler.perform_streamed_request("GET", custom_args)
-        return raw_response
+        response = self.request_handler.perform_streamed_request("GET", custom_args)
+        return response
 
     def get_dsn_binary_content(self, dataset_name, with_prefixes=False):
         """
@@ -462,8 +457,8 @@ def get_dsn_binary_content(self, dataset_name, with_prefixes=False):
                                  default: False
         Returns
         -------
-        bytes
-            The contents of the dataset with no transformation
+        response
+            A response object from the requests library
         """
         custom_args = self._create_custom_request_arguments()
         custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(dataset_name))
@@ -472,8 +467,8 @@ def get_dsn_binary_content(self, dataset_name, with_prefixes=False):
             custom_args["headers"]["X-IBM-Data-Type"] = "record"
         else:
             custom_args["headers"]["X-IBM-Data-Type"] = "binary"
-        content = self.request_handler.perform_request("GET", custom_args)
-        return content
+        response = self.request_handler.perform_request("GET", custom_args)
+        return response
 
     def get_dsn_binary_content_streamed(self, dataset_name, with_prefixes=False):
         """
@@ -486,8 +481,8 @@ def get_dsn_binary_content_streamed(self, dataset_name, with_prefixes=False):
                                  default: False
         Returns
         -------
-        raw
-            The raw socket response
+        response
+            A response object from the requests library
         """
         custom_args = self._create_custom_request_arguments()
         custom_args["url"] = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(dataset_name))
@@ -496,8 +491,8 @@ def get_dsn_binary_content_streamed(self, dataset_name, with_prefixes=False):
             custom_args["headers"]["X-IBM-Data-Type"] = "record"
         else:
             custom_args["headers"]["X-IBM-Data-Type"] = "binary"
-        content = self.request_handler.perform_streamed_request("GET", custom_args)
-        return content
+        response = self.request_handler.perform_streamed_request("GET", custom_args)
+        return response
 
     def write_to_dsn(self, dataset_name, data, encoding=_ZOWE_FILES_DEFAULT_ENCODING):
         """Write content to an existing dataset.
@@ -516,9 +511,10 @@ def write_to_dsn(self, dataset_name, data, encoding=_ZOWE_FILES_DEFAULT_ENCODING
 
     def download_dsn(self, dataset_name, output_file):
         """Retrieve the contents of a dataset and saves it to a given file."""
-        raw_response = self.get_dsn_content_streamed(dataset_name)
+        response = self.get_dsn_content_streamed(dataset_name)
         with open(output_file, "w", encoding="utf-8") as f:
-            shutil.copyfileobj(raw_response, f)
+            for chunk in response.iter_content(chunk_size=4096, decode_unicode=True):
+                f.write(chunk)
 
     def download_binary_dsn(self, dataset_name, output_file, with_prefixes=False):
         """Retrieve the contents of a binary dataset and saves it to a given file.
@@ -529,15 +525,11 @@ def download_binary_dsn(self, dataset_name, output_file, with_prefixes=False):
         output_file:str - Name of the local file to create
         with_prefixes:boolean - If true, include a four big endian bytes record length prefix.
                                 The default is False
-
-        Returns
-        -------
-        bytes
-            Binary content of the dataset.
         """
-        content = self.get_dsn_binary_content_streamed(dataset_name, with_prefixes=with_prefixes)
+        response = self.get_dsn_binary_content_streamed(dataset_name, with_prefixes=with_prefixes)
         with open(output_file, "wb") as f:
-            shutil.copyfileobj(content, f)
+            for chunk in response.iter_content(chunk_size=4096):
+                f.write(chunk)
 
     def upload_file_to_dsn(self, input_file, dataset_name, encoding=_ZOWE_FILES_DEFAULT_ENCODING):
         """Upload contents of a given file and uploads it to a dataset."""
@@ -564,21 +556,42 @@ def write_to_uss(self, filepath_name, data, encoding=_ZOWE_FILES_DEFAULT_ENCODIN
     def upload_file_to_uss(self, input_file, filepath_name, encoding=_ZOWE_FILES_DEFAULT_ENCODING):
         """Upload contents of a given file and uploads it to UNIX file"""
         if os.path.isfile(input_file):
-            in_file = open(input_file, "r", encoding="utf-8")
-            file_contents = in_file.read()
-            response_json = self.write_to_uss(filepath_name, file_contents)
+            with open(input_file, "r", encoding="utf-8") as in_file:
+                response_json = self.write_to_uss(filepath_name, in_file)
         else:
             raise FileNotFound(input_file)
 
+    def get_file_content_streamed(self, file_path, binary=False):
+        """Retrieve the contents of a given USS file streamed.
+
+        Returns
+        -------
+        response
+            A response object from the requests library
+        """
+        custom_args = self._create_custom_request_arguments()
+        custom_args["url"] = "{}fs/{}".format(self.request_endpoint, self._encode_uri_component(file_path.lstrip("/")))
+        if binary:
+            custom_args["headers"]["X-IBM-Data-Type"] = "binary"
+        response = self.request_handler.perform_streamed_request("GET", custom_args)
+        return response
+
+    def download_uss(self, file_path, output_file, binary=False):
+        """Retrieve the contents of a USS file and saves it to a local file."""
+        response = self.get_file_content_streamed(file_path, binary)
+        with open(output_file, "wb" if binary else "w", encoding="utf-8") as f:
+            for chunk in response.iter_content(chunk_size=4096, decode_unicode=not binary):
+                f.write(chunk)
+
     def delete_data_set(self, dataset_name, volume=None, member_name=None):
         """Deletes a sequential or partitioned data."""
         custom_args = self._create_custom_request_arguments()
         if member_name is not None:
             dataset_name = f"{dataset_name}({member_name})"
-        url = "{}ds/{}".format(self.request_endpoint, dataset_name)
+        url = "{}ds/{}".format(self.request_endpoint, self._encode_uri_component(dataset_name))
         if volume is not None:
-            url = "{}ds/-{}/{}".format(self.request_endpoint, volume, dataset_name)
-        custom_args["url"] = self._encode_uri_component(url)
+            url = "{}ds/-{}/{}".format(self.request_endpoint, volume, self._encode_uri_component(dataset_name))
+        custom_args["url"] = url
         response_json = self.request_handler.perform_request("DELETE", custom_args, expected_code=[200, 202, 204])
         return response_json
 

src/zos_jobs/zowe/zos_jobs_for_zowe_sdk/jobs.py

@@ -267,9 +267,8 @@ def submit_from_local_file(self, jcl_path):
             A JSON containing the result of the request execution
         """
         if os.path.isfile(jcl_path):
-            jcl_file = open(jcl_path, "r", encoding="utf-8")
-            file_content = jcl_file.read()
-            jcl_file.close()
+            with open(jcl_path, "r", encoding="utf-8") as jcl_file:
+                file_content = jcl_file.read()
             return self.submit_plaintext(file_content)
         else:
             raise FileNotFoundError("Provided argument is not a file path {}".format(jcl_path))
@@ -388,32 +387,30 @@ def get_job_output_as_files(self, status, output_dir):
             A JSON containing the result of the request execution
         """
 
-        _job_name = status["jobname"]
-        _job_id = status["jobid"]
-        _job_correlator = status["job-correlator"]
-
-        _output_dir = os.path.join(output_dir, _job_name, _job_id)
-        os.makedirs(_output_dir, exist_ok=True)
-        _output_file = os.path.join(output_dir, _job_name, _job_id, "jcl.txt")
-        _data_spool_file = self.get_jcl_text(_job_correlator)
-        _dataset_content = _data_spool_file["response"]
-        _out_file = open(_output_file, "w", encoding="utf-8")
-        _out_file.write(_dataset_content)
-        _out_file.close()
-
-        _spool = self.get_spool_files(_job_correlator)
-        for _spool_file in _spool:
-            _stepname = _spool_file["stepname"]
-            _ddname = _spool_file["ddname"]
-            _spoolfile_id = _spool_file["id"]
-            _output_dir = os.path.join(output_dir, _job_name, _job_id, _stepname)
-            os.makedirs(_output_dir, exist_ok=True)
-
-            _output_file = os.path.join(output_dir, _job_name, _job_id, _stepname, _ddname)
-            _data_spool_file = self.get_spool_file_contents(_job_correlator, _spoolfile_id)
-            _dataset_content = _data_spool_file["response"]
-            _out_file = open(_output_file, "w", encoding="utf-8")
-            _out_file.write(_dataset_content)
-            _out_file.close()
+        job_name = status["jobname"]
+        job_id = status["jobid"]
+        job_correlator = status["job-correlator"]
+
+        output_dir = os.path.join(output_dir, job_name, job_id)
+        os.makedirs(output_dir, exist_ok=True)
+        output_file = os.path.join(output_dir, job_name, job_id, "jcl.txt")
+        data_spool_file = self.get_jcl_text(job_correlator)
+        dataset_content = data_spool_file["response"]
+        with open(output_file, "w", encoding="utf-8") as out_file:
+            out_file.write(dataset_content)
+
+        spool = self.get_spool_files(job_correlator)
+        for spool_file in spool:
+            stepname = spool_file["stepname"]
+            ddname = spool_file["ddname"]
+            spoolfile_id = spool_file["id"]
+            output_dir = os.path.join(output_dir, job_name, job_id, stepname)
+            os.makedirs(output_dir, exist_ok=True)
+
+            output_file = os.path.join(output_dir, job_name, job_id, stepname, ddname)
+            data_spool_file = self.get_spool_file_contents(job_correlator, spoolfile_id)
+            dataset_content = data_spool_file["response"]
+            with open(output_file, "w", encoding="utf-8") as out_file:
+                out_file.write(dataset_content)
 
         return