Merge branch 'master' into attach-proxy-options

packages/cli/CHANGELOG.md

@@ -1,9 +1,11 @@
 # Change Log
 All notable changes to the Zowe CLI package will be documented in this file.
 
-## `8.9.0`
+## Recent Changes
+-Enhancement: The `zowe zos-files copy data-set` command now copies members from a source partitioned data set to an existing target partitioned data set.[#2386](https://github.com/zowe/zowe-cli/pull/2386)
 
--Enhancement: Added new command zowe zos-files download all-members-matching, (zowe files dl amm), to download members matching specified pattern(s). The success message for the Download.allMembers API was changed from originally "Data set downloaded successfully" to "Member(s) downloaded successfully." The change also alters the commandResponse when using the --rfj flag. [#2359](https://github.com/zowe/zowe-cli/pull/2359)
+## `8.9.0`
+- Enhancement: Added new command zowe zos-files download all-members-matching, (zowe files dl amm), to download members matching specified pattern(s). The success message for the Download.allMembers API was changed from originally "Data set downloaded successfully" to "Member(s) downloaded successfully." The change also alters the commandResponse when using the --rfj flag. [#2359](https://github.com/zowe/zowe-cli/pull/2359)
 
 ## `8.8.0`
 

packages/cli/src/zosfiles/-strings-/en.ts

@@ -188,8 +188,8 @@ export default {
         DESCRIPTION: "Copy a data set.",
         ACTIONS: {
             DATA_SET: {
-                SUMMARY: "Copy a data set to another data set",
-                DESCRIPTION: "Copy a data set to another data set.",
+                SUMMARY: "Copy a data set/partitioned data set to another data set/partitioned data set",
+                DESCRIPTION: "Copy a data set/partitioned data set to another data set/partitioned data set.",
                 POSITIONALS: {
                     FROMDSNAME: "The name of the data set that you want to copy from",
                     TODSNAME: "The name of the data set that you want to copy to (data set must be preallocated)"
@@ -202,7 +202,8 @@ export default {
                     EX2: "Copy the data set member named 'USER.FROM.SET(MEM1)' to the data set member named 'USER.TO.SET(MEM2)'",
                     EX3: "Copy the data set named 'USER.FROM.SET' to the data set member named 'USER.TO.SET(MEM2)'",
                     EX4: "Copy the data set member named 'USER.FROM.SET(MEM1)' to the data set named 'USER.TO.SET'",
-                    EX5: "Copy the data set named 'USER.FROM.SET' to the data set named 'USER.TO.SET' and replace like-named members"
+                    EX5: "Copy the data set named 'USER.FROM.SET' to the data set named 'USER.TO.SET' and replace like-named members",
+                    EX6: "Copy the partitioned data set named 'TEST.PDS1' to the partitioned data set named 'TEST.PDS2'"
                 }
             },
             DATA_SET_CROSS_LPAR: {

packages/zosfiles/CHANGELOG.md

@@ -2,13 +2,15 @@
 
 All notable changes to the Zowe z/OS files SDK package will be documented in this file.
 
+## Recent Changes
+- Enhancement: The `Copy.dataset` method now recognizes partitioned data sets and can copy members of a source PDS into an existing target PDS. [#2386](https://github.com/zowe/zowe-cli/pull/2386)
+
 ## `8.9.1`
 
 - BugFix: Corrected the `apiResponse` response value from `streamToDataSet()`,`streamToUss()`,`bufferToUss()` and `bufferToDataSet()` on the Upload SDK. [#2381](https://github.com/zowe/zowe-cli/pull/2381)
 - BugFix: Corrected the `Upload.BufferToUssFile()` SDK function to properly tag uploaded files. [#2378](https://github.com/zowe/zowe-cli/pull/2378)
 
 ## `8.9.0`
-
 - Enhancement: Added a `List.membersMatchingPattern` method to download all members that match a specific pattern.[#2359](https://github.com/zowe/zowe-cli/pull/2359)
 
 ## `8.8.4`

packages/zosfiles/__tests__/__system__/methods/copy/Copy.system.test.ts

@@ -10,14 +10,18 @@
 */
 
 import { Create, Upload, Delete, CreateDataSetTypeEnum, Copy, ZosFilesMessages, Get, IDataSet,
-    ICrossLparCopyDatasetOptions, IGetOptions, IZosFilesResponse } from "../../../../src";
-import { Imperative, Session } from "@zowe/imperative";
+    ICrossLparCopyDatasetOptions, IGetOptions, IZosFilesResponse,
+    ZosFilesUtils} from "../../../../src";
+import { Imperative, IO, Session } from "@zowe/imperative";
 import { inspect } from "util";
 import { TestEnvironment } from "../../../../../../__tests__/__src__/environment/TestEnvironment";
 import { ITestPropertiesSchema } from "../../../../../../__tests__/__src__/properties/ITestPropertiesSchema";
 import { join } from "path";
 import { readFileSync } from "fs";
 import { ITestEnvironment } from "../../../../../../__tests__/__src__/environment/ITestEnvironment";
+import { tmpdir } from "os";
+import path = require("path");
+import * as fs from "fs";
 
 let REAL_SESSION: Session;
 let REAL_TARGET_SESSION: Session;
@@ -98,6 +102,60 @@ describe("Copy", () => {
                     expect(contents1.toString()).toEqual(contents2.toString());
                 });
             });
+            describe("Partioned > Partioned", () => {
+                let downloadDir: string;
+                beforeEach(async () => {
+                    try {
+                        downloadDir = path.join(tmpdir(), fromDataSetName);
+                        fs.mkdirSync(downloadDir, { recursive: true });
+                        const mockFile = path.join(downloadDir, "mockFile.txt");
+                        fs.writeFileSync(mockFile, "test file content");
+
+                        const uploadFileList: string[] = ZosFilesUtils.getFileListFromPath(downloadDir);
+                        const stream = IO.createReadStream(uploadFileList[0]);
+                        await Create.dataSet(REAL_SESSION, CreateDataSetTypeEnum.DATA_SET_SEQUENTIAL, fromDataSetName);
+                        await Create.dataSet(REAL_SESSION, CreateDataSetTypeEnum.DATA_SET_SEQUENTIAL, toDataSetName);
+                        await Upload.streamToDataSet(REAL_SESSION, stream, fromDataSetName);
+                    } catch (err) {
+                        Imperative.console.info(`Error: ${inspect(err)}`);
+                    }
+                });
+                it("Should copy a partitioned data set", async () => {
+                    let error;
+                    let response;
+                    let contents1;
+                    let contents2;
+
+                    try {
+                        response = await Copy.dataSet(
+                            REAL_SESSION,
+                            {dsn: toDataSetName},
+                            {"from-dataset": {
+                                dsn:fromDataSetName
+                            }}
+                        );
+                        contents1 = await Get.dataSet(REAL_SESSION, fromDataSetName);
+                        contents2 = await Get.dataSet(REAL_SESSION, toDataSetName);
+                        Imperative.console.info(`Response: ${inspect(response)}`);
+                    } catch (err) {
+                        error = err;
+                        Imperative.console.info(`Error: ${inspect(err)}`);
+                    }
+
+                    expect(error).toBeFalsy();
+
+                    expect(response).toBeTruthy();
+                    expect(response.success).toBe(true);
+                    expect(response.commandResponse).toContain(ZosFilesMessages.datasetCopiedSuccessfully.message);
+
+                    expect(contents1).toBeTruthy();
+                    expect(contents2).toBeTruthy();
+                    expect(contents1.toString()).toEqual(contents2.toString());
+                });
+                afterEach(() => {
+                    fs.rmSync(downloadDir, { recursive: true, force: true });
+                });
+            });
             describe("Member > Member", () => {
                 beforeEach(async () => {
                     try {

packages/zosfiles/__tests__/__system__/methods/upload/Upload.system.test.ts

@@ -326,7 +326,11 @@ describe("Upload Data Set", () => {
 
                 expect(error).toBeFalsy();
 
-                expect(uploadResponse.apiResponse).toMatchObject({"success": true, "from": "Buffer<>","to": dsname+"(TEST)"});
+                expect(uploadResponse.apiResponse).toMatchObject({
+                    success: true,
+                    from: "<Buffer 61 62 63 64 65 66 67 68 69 6a...>",
+                    to: dsname + "(TEST)",
+                });
                 expect(Buffer.from(getResponse.toString().trim())).toEqual(data);
             });
 
@@ -349,7 +353,7 @@ describe("Upload Data Set", () => {
 
                 expect(error).toBeFalsy();
 
-                expect(uploadResponse.apiResponse).toMatchObject({"success": true, "from": "Stream<>","to": dsname+"(TEST)"});
+                expect(uploadResponse.apiResponse).toMatchObject({"success": true, "from": "[Readable]","to": dsname+"(TEST)"});
                 expect(getResponse.toString().trim()).toEqual(testdata);
             });
 
@@ -793,7 +797,7 @@ describe("Upload USS file", () => {
 
             expect(error).toBeFalsy();
 
-            expect(uploadResponse.apiResponse).toMatchObject({"success": true, "from": "Buffer<>","to": ussname});
+            expect(uploadResponse.apiResponse).toMatchObject({"success": true, "from": "<Buffer 61 62 63 64 65 66 67 68 69 6a...>","to": ussname});
             expect(getResponse).toEqual(Buffer.from(data.toString()));
         });
 
@@ -815,7 +819,7 @@ describe("Upload USS file", () => {
 
             expect(error).toBeFalsy();
 
-            expect(uploadResponse.apiResponse).toMatchObject({"success": true, "from": "Stream<>","to": ussname});
+            expect(uploadResponse.apiResponse).toMatchObject({"success": true, "from": "[Readable]","to": ussname});
             expect(getResponse).toEqual(Buffer.from(testdata));
         });
 

packages/zosfiles/__tests__/__unit__/methods/copy/Copy.unit.test.ts

@@ -9,12 +9,12 @@
 *
 */
 
-import { Session, ImperativeError } from "@zowe/imperative";
+import { Session, ImperativeError, IO } from "@zowe/imperative";
 import { posix } from "path";
-
+import * as fs from "fs";
 import { error } from "console";
 
-import { Copy, Create, Get, List, Upload, ZosFilesConstants, ZosFilesMessages, IZosFilesResponse } from "../../../../src";
+import { Copy, Create, Get, List, Upload, ZosFilesConstants, ZosFilesMessages, IZosFilesResponse, Download, ZosFilesUtils } from "../../../../src";
 import { ZosmfHeaders, ZosmfRestClient } from "@zowe/core-for-zowe-sdk";
 
 describe("Copy", () => {
@@ -33,14 +33,18 @@ describe("Copy", () => {
         const fromMemberName = "mem1";
         const toDataSetName = "USER.DATA.TO";
         const toMemberName = "mem2";
+        let isPDSSpy: jest.SpyInstance;
 
         beforeEach(() => {
             copyExpectStringSpy.mockClear();
             copyExpectStringSpy.mockImplementation(async () => {
                 return "";
             });
+            isPDSSpy = jest.spyOn(Copy as any, "isPDS").mockResolvedValue(false);
+        });
+        afterAll(() => {
+            isPDSSpy.mockRestore();
         });
-
         describe("Success Scenarios", () => {
             describe("Sequential > Sequential", () => {
                 it("should send a request", async () => {
@@ -438,6 +442,40 @@ describe("Copy", () => {
                     expect(lastArgumentOfCall).toHaveProperty("replace", false);
                 });
             });
+            describe("Partitioned > Partitioned", () => {
+                let copyPDSSpy = jest.spyOn(Copy, "copyPDS");
+                beforeEach(() => {
+                    copyPDSSpy.mockClear();
+                    copyPDSSpy = jest.spyOn(Copy, "copyPDS").mockResolvedValue({
+                        success:true,
+                        commandResponse: ZosFilesMessages.datasetCopiedSuccessfully.message,
+                    });
+                    isPDSSpy = jest.spyOn(Copy as any, "isPDS").mockResolvedValue(true);
+                });
+                afterAll(() => {
+                    copyPDSSpy.mockRestore();
+                    isPDSSpy.mockRestore();
+                });
+                it("should call copyPDS to copy members of source PDS to target PDS", async () => {
+                    const response = await Copy.dataSet(
+                        dummySession,
+                        {dsn: toDataSetName},
+                        {"from-dataset": {
+                            dsn:fromDataSetName
+                        }}
+                    );
+                    expect(isPDSSpy).toHaveBeenNthCalledWith(1, dummySession, fromDataSetName);
+                    expect(isPDSSpy).toHaveBeenNthCalledWith(2, dummySession, toDataSetName);
+
+                    expect(copyPDSSpy).toHaveBeenCalledTimes(1);
+                    expect(copyPDSSpy).toHaveBeenCalledWith(dummySession, fromDataSetName, toDataSetName);
+
+                    expect(response).toEqual({
+                        success: true,
+                        commandResponse: ZosFilesMessages.datasetCopiedSuccessfully.message
+                    });
+                });
+            });
         });
         describe("Failure Scenarios", () => {
             it("should fail if the zOSMF REST client fails", async () => {
@@ -515,6 +553,107 @@ describe("Copy", () => {
         });
     });
 
+    describe("Copy Partitioned Data Set", () => {
+        const listAllMembersSpy   = jest.spyOn(List, "allMembers");
+        const downloadAllMembersSpy = jest.spyOn(Download, "allMembers");
+        const uploadSpy = jest.spyOn(Upload, "streamToDataSet");
+        const fileListPathSpy = jest.spyOn(ZosFilesUtils, "getFileListFromPath");
+        const generateMemName = jest.spyOn(ZosFilesUtils, "generateMemberName");
+        const fromDataSetName = "USER.DATA.FROM";
+        const toDataSetName = "USER.DATA.TO";
+        const readStream = jest.spyOn(IO, "createReadStream");
+        const rmSync = jest.spyOn(fs, "rmSync");
+        const listDatasetSpy = jest.spyOn(List, "dataSet");
+
+        const dsPO = {
+            dsname: fromDataSetName,
+            dsorg: "PO",
+        };
+        const dsPS = {
+            dsname: fromDataSetName,
+            dsorg: "PS",
+        };
+
+        it("should detect PDS datasets correctly during copy", async () => {
+            let caughtError;
+            let response;
+            listDatasetSpy.mockImplementation(async (): Promise<any>  => {
+                return {
+                    apiResponse: {
+                        items: [dsPO]
+                    }
+                };
+            });
+            try {
+                response = await Copy.isPDS(dummySession, dsPO.dsname);
+            }
+            catch(e) {
+                caughtError = e;
+            }
+            expect(response).toEqual(true);
+            expect(listDatasetSpy).toHaveBeenCalledWith(dummySession, dsPO.dsname, { attributes: true });
+        });
+
+        it("should return false if the data set is not partitioned", async () => {
+            let response;
+            let caughtError;
+            listDatasetSpy.mockImplementation(async (): Promise<any>  => {
+                return {
+                    apiResponse: {
+                        items: [dsPS]
+                    }
+                };
+            });
+            try {
+                response = await Copy.isPDS(dummySession, dsPS.dsname);
+            }
+            catch(e) {
+                caughtError = e;
+            }
+            expect(response).toEqual(false);
+            expect(listDatasetSpy).toHaveBeenCalledWith(dummySession, dsPS.dsname, { attributes: true });
+        });
+
+        it("should successfully copy members from source to target PDS", async () => {
+            let caughtError;
+            let response;
+            const sourceResponse = {
+                apiResponse: {
+                    items: [
+                        { member: "mem1" },
+                        { member: "mem2" },
+                    ]
+                }
+            };
+            const fileList = ["mem1", "mem2"];
+
+            listAllMembersSpy.mockImplementation(async (): Promise<any>  => sourceResponse);
+            downloadAllMembersSpy.mockImplementation(async (): Promise<any> => undefined);
+            fileListPathSpy.mockReturnValue(fileList);
+            generateMemName.mockReturnValue("mem1");
+            readStream.mockReturnValue("test" as any);
+            uploadSpy.mockResolvedValue(undefined);
+            rmSync.mockImplementation(jest.fn());
+
+
+            try{
+                response = await Copy.copyPDS(dummySession, fromDataSetName, toDataSetName);
+            }
+            catch(e) {
+                caughtError = e;
+            }
+            expect(listAllMembersSpy).toHaveBeenCalledWith(dummySession, fromDataSetName);
+            expect(downloadAllMembersSpy).toHaveBeenCalledWith(dummySession, fromDataSetName, expect.any(Object));
+            expect(fileListPathSpy).toHaveBeenCalled();
+            expect(uploadSpy).toHaveBeenCalledTimes(fileList.length);
+            expect(rmSync).toHaveBeenCalled();
+            expect(response).toEqual({
+                success: true,
+                commandResponse: ZosFilesMessages.datasetCopiedSuccessfully.message,
+            });
+        });
+    });
+
     describe("Data Set Cross LPAR", () => {
         const getDatasetSpy    = jest.spyOn(Get, "dataSet");
         const listDatasetSpy   = jest.spyOn(List, "dataSet");

packages/zosfiles/__tests__/__unit__/methods/upload/Upload.unit.test.ts

@@ -22,7 +22,6 @@ import { IUploadOptions } from "../../../../src/methods/upload/doc/IUploadOption
 import { Upload } from "../../../../src/methods/upload/Upload";
 import { List } from "../../../../src/methods/list/List";
 import { Utilities } from "../../../../src/methods/utilities/Utilities";
-import { inspect } from "util";
 import { ZosFilesUtils } from "../../../../src/utils/ZosFilesUtils";
 import { stripNewLines } from "../../../../../../__tests__/__src__/TestUtils";
 import { Create } from "../../../../src/methods/create";