Copying a PDS into existing PDS w/ no members

packages/zosfiles/CHANGELOG.md

@@ -3,6 +3,7 @@
 All notable changes to the Zowe z/OS files SDK package will be documented in this file.
 
 ## Recent Changes
+-Enhancement: `Copy.dataset` method now recognizes Partioned data sets and can copy members of source PDS into an existing target PDS. [#2386](https://github.com/zowe/zowe-cli/pull/2386)
 
 - Enhancement: Added a `List.membersMatchingPattern` method to download all members that match a specific pattern.[#2359](https://github.com/zowe/zowe-cli/pull/2359)
 

packages/zosfiles/__tests__/__system__/methods/copy/Copy.system.test.ts

@@ -10,14 +10,18 @@
 */
 
 import { Create, Upload, Delete, CreateDataSetTypeEnum, Copy, ZosFilesMessages, Get, IDataSet,
-    ICrossLparCopyDatasetOptions, IGetOptions, IZosFilesResponse } from "../../../../src";
-import { Imperative, Session } from "@zowe/imperative";
+    ICrossLparCopyDatasetOptions, IGetOptions, IZosFilesResponse, 
+    ZosFilesUtils} from "../../../../src";
+import { Imperative, IO, Session } from "@zowe/imperative";
 import { inspect } from "util";
 import { TestEnvironment } from "../../../../../../__tests__/__src__/environment/TestEnvironment";
 import { ITestPropertiesSchema } from "../../../../../../__tests__/__src__/properties/ITestPropertiesSchema";
 import { join } from "path";
 import { readFileSync } from "fs";
 import { ITestEnvironment } from "../../../../../../__tests__/__src__/environment/ITestEnvironment";
+import { tmpdir } from "os";
+import path = require("path");
+import * as fs from "fs";
 
 let REAL_SESSION: Session;
 let REAL_TARGET_SESSION: Session;
@@ -98,6 +102,56 @@
                     expect(contents1.toString()).toEqual(contents2.toString());
                 });
             });
+            describe("Partioned > Partioned", () => {
+                beforeEach(async () => {
+                    try {
+                        const downloadDir = path.join(tmpdir(), fromDataSetName);
+                        fs.mkdirSync(downloadDir, { recursive: true });
+                        const mockFile = path.join(downloadDir, "mockFile.txt");
+                        fs.writeFileSync(mockFile, "test file content");
+
+                        const uploadFileList: string[] = ZosFilesUtils.getFileListFromPath(downloadDir);
+                        const stream = IO.createReadStream(uploadFileList[0]);
+                        await Create.dataSet(REAL_SESSION, CreateDataSetTypeEnum.DATA_SET_SEQUENTIAL, fromDataSetName);
+                        await Create.dataSet(REAL_SESSION, CreateDataSetTypeEnum.DATA_SET_SEQUENTIAL, toDataSetName);
+                        await Upload.streamToDataSet(REAL_SESSION, stream, fromDataSetName);
+                    } catch (err) {
+                        Imperative.console.info(`Error: ${inspect(err)}`);
+                    }
+                });
+                it("Should copy a partitioned data set", async () => {
+                    let error;
+                    let response;
+                    let contents1;
+                    let contents2;
+
+                    try {
+                        response = await Copy.dataSet(
+                            REAL_SESSION,
+                            {dsn: toDataSetName},
+                            {"from-dataset": {
+                                dsn:fromDataSetName
+                            }}
+                        );
+                        contents1 = await Get.dataSet(REAL_SESSION, fromDataSetName);
+                        contents2 = await Get.dataSet(REAL_SESSION, toDataSetName);
+                        Imperative.console.info(`Response: ${inspect(response)}`);
+                    } catch (err) {
+                        error = err;
+                        Imperative.console.info(`Error: ${inspect(err)}`);
+                    }
+
+                    expect(error).toBeFalsy();
+
+                    expect(response).toBeTruthy();
+                    expect(response.success).toBe(true);
+                    expect(response.commandResponse).toContain(ZosFilesMessages.datasetCopiedSuccessfully.message);
+
+                    expect(contents1).toBeTruthy();
+                    expect(contents2).toBeTruthy();
+                    expect(contents1.toString()).toEqual(contents2.toString());
+                });
+            });
             describe("Member > Member", () => {
                 beforeEach(async () => {
                     try {

packages/zosfiles/__tests__/__unit__/methods/copy/Copy.unit.test.ts

@@ -11,11 +11,13 @@
 
 import { Session, ImperativeError } from "@zowe/imperative";
 import { posix } from "path";
-
+import * as fs from "fs";
 import { error } from "console";
 
-import { Copy, Create, Get, List, Upload, ZosFilesConstants, ZosFilesMessages, IZosFilesResponse } from "../../../../src";
+import { Copy, Create, Get, List, Upload, ZosFilesConstants, ZosFilesMessages, IZosFilesResponse, Download, ZosFilesUtils } from "../../../../src";
 import { ZosmfHeaders, ZosmfRestClient } from "@zowe/core-for-zowe-sdk";
+import { tmpdir } from "os";
+import path = require("path");
 
 describe("Copy", () => {
     const dummySession = new Session({
@@ -29,6 +31,8 @@
 
     describe("Data Set", () => {
         const copyExpectStringSpy = jest.spyOn(ZosmfRestClient, "putExpectString");
+        let copyPDSSpy = jest.spyOn(Copy, "copyPDS");
+        let isPDSSpy: jest.SpyInstance;
         const fromDataSetName = "USER.DATA.FROM";
         const fromMemberName = "mem1";
         const toDataSetName = "USER.DATA.TO";
@@ -39,6 +43,12 @@
             copyExpectStringSpy.mockImplementation(async () => {
                 return "";
             });
+            copyPDSSpy.mockClear();
+            copyPDSSpy = jest.spyOn(Copy, "copyPDS").mockResolvedValue({
+                success:true,
+                commandResponse: ZosFilesMessages.datasetCopiedSuccessfully.message,
+            });
+            isPDSSpy = jest.spyOn(Copy as any, "isPDS").mockResolvedValue(true);
         });
 
         describe("Success Scenarios", () => {
@@ -71,7 +81,7 @@
                         success: true,
                         commandResponse: ZosFilesMessages.datasetCopiedSuccessfully.message
                     });
                     expect(copyExpectStringSpy).toHaveBeenCalledTimes(1);
                     expect(copyExpectStringSpy).toHaveBeenLastCalledWith(
                         dummySession,
                         expectedEndpoint,
@@ -109,7 +119,7 @@
                         success: true,
                         commandResponse: ZosFilesMessages.datasetCopiedSuccessfully.message
                     });
                     expect(copyExpectStringSpy).toHaveBeenCalledTimes(1);
                     expect(copyExpectStringSpy).toHaveBeenLastCalledWith(
                         dummySession,
                         expectedEndpoint,
@@ -148,7 +158,7 @@
                         success: true,
                         commandResponse: ZosFilesMessages.datasetCopiedSuccessfully.message
                     });
                     expect(copyExpectStringSpy).toHaveBeenCalledTimes(1);
                     expect(copyExpectStringSpy).toHaveBeenLastCalledWith(
                         dummySession,
                         expectedEndpoint,
@@ -187,7 +197,7 @@
                         success: true,
                         commandResponse: ZosFilesMessages.datasetCopiedSuccessfully.message
                     });
                     expect(copyExpectStringSpy).toHaveBeenCalledTimes(1);
                     expect(copyExpectStringSpy).toHaveBeenLastCalledWith(
                         dummySession,
                         expectedEndpoint,
@@ -225,7 +235,7 @@
                         success: true,
                         commandResponse: ZosFilesMessages.datasetCopiedSuccessfully.message
                     });
                     expect(copyExpectStringSpy).toHaveBeenCalledTimes(1);
                     expect(copyExpectStringSpy).toHaveBeenLastCalledWith(
                         dummySession,
                         expectedEndpoint,
@@ -263,7 +273,7 @@
                         success: true,
                         commandResponse: ZosFilesMessages.datasetCopiedSuccessfully.message
                     });
                     expect(copyExpectStringSpy).toHaveBeenCalledTimes(1);
                     expect(copyExpectStringSpy).toHaveBeenLastCalledWith(
                         dummySession,
                         expectedEndpoint,
@@ -302,7 +312,7 @@
                         success: true,
                         commandResponse: ZosFilesMessages.datasetCopiedSuccessfully.message
                     });
                     expect(copyExpectStringSpy).toHaveBeenCalledTimes(1);
                     expect(copyExpectStringSpy).toHaveBeenLastCalledWith(
                         dummySession,
                         expectedEndpoint,
@@ -341,7 +351,7 @@
                         success: true,
                         commandResponse: ZosFilesMessages.datasetCopiedSuccessfully.message
                     });
                     expect(copyExpectStringSpy).toHaveBeenCalledTimes(1);
                     expect(copyExpectStringSpy).toHaveBeenLastCalledWith(
                         dummySession,
                         expectedEndpoint,
@@ -358,7 +368,7 @@
                         { "from-dataset": { dsn: fromDataSetName } }
                     );
 
                     expect(copyExpectStringSpy).toHaveBeenCalledTimes(1);
                     const argumentsOfCall = copyExpectStringSpy.mock.calls[0];
                     const lastArgumentOfCall = argumentsOfCall[argumentsOfCall.length - 1];
                     expect(lastArgumentOfCall).not.toHaveProperty("enq");
@@ -373,7 +383,7 @@
                         }
                     );
 
                     expect(copyExpectStringSpy).toHaveBeenCalledTimes(1);
                     const argumentsOfCall = copyExpectStringSpy.mock.calls[0];
                     const lastArgumentOfCall = argumentsOfCall[argumentsOfCall.length - 1];
                     expect(lastArgumentOfCall).toHaveProperty("enq", "SHR");
@@ -438,6 +448,28 @@
                     expect(lastArgumentOfCall).toHaveProperty("replace", false);
                 });
             });
+            describe("Partitioned > Partitioned", () => {
+                it("should call copyPDS to copy members of source PDS to target PDS", async () => {
+                    const response = await Copy.dataSet(
+                        dummySession,
+                        {dsn: toDataSetName},
+                        {"from-dataset": {
+                            dsn:fromDataSetName
+                        }}
+                    );
+                    expect(isPDSSpy).toHaveBeenCalledTimes(2);
+                    expect(isPDSSpy).toHaveBeenNthCalledWith(1, dummySession, fromDataSetName);
+                    expect(isPDSSpy).toHaveBeenNthCalledWith(2, dummySession, toDataSetName);
+
+                    expect(copyPDSSpy).toHaveBeenCalledTimes(1);
+                    expect(copyPDSSpy).toHaveBeenCalledWith(dummySession, fromDataSetName, toDataSetName);
+
+                    expect(response).toEqual({
+                        success: true,
+                        commandResponse: ZosFilesMessages.datasetCopiedSuccessfully.message
+                    });
+                });
+            });
         });
         describe("Failure Scenarios", () => {
             it("should fail if the zOSMF REST client fails", async () => {
@@ -515,6 +547,41 @@
         });
     });
 
+    // describe("Copy Partitioned Data Set", () => {
+    //     const listAllMembersSpy   = jest.spyOn(List, "allMembers");
+    //     const downloadAllMembersSpy = jest.spyOn(Download, "allMembers");
+    //     const uploadSpy = jest.spyOn(Upload, "streamToDataSet");
+    //     const fileListPathSpy = jest.spyOn(ZosFilesUtils, "getFileListFromPath");
+    //     const fromDataSetName = "USER.DATA.FROM";
+    //     const toDataSetName = "USER.DATA.TO";
+    //     it("should successfully copy members from source to target PDS", async () => {
+    //         listAllMembersSpy.mockImplementation(async (): Promise<any> => ({
+    //             apiResponse: {
+    //                 items: [
+    //                     {member: "mem1"},
+    //                     {member: "mem2"}
+    //                 ]
+    //             }
+    //         }));
+    //         downloadAllMembersSpy.mockImplementation(async (): Promise<any> => undefined);
+
+    //         uploadSpy.mockImplementation(async (): Promise<any> => undefined);
+
+    //         const response = await Copy.copyPDS(dummySession, fromDataSetName, toDataSetName);
+    //         // const downloadDir = path.join(tmpdir(), fromDataSetName);
+    //         expect(listAllMembersSpy).toHaveBeenCalledWith(dummySession, fromDataSetName);
+    //         expect(downloadAllMembersSpy).toHaveBeenCalled();
+    //         // expect(fileListPathSpy).toHaveBeenCalledWith(path.join(tmpdir(), fromDataSetName));
+    //         expect(uploadSpy).toHaveBeenCalledTimes(2);
+
+    //         // expect(fs.rmSync).toHaveBeenCalled();
+    //         expect(response).toEqual({
+    //             success: true,
+    //             commandResponse: ZosFilesMessages.datasetCopiedSuccessfully.message,
+    //         });
+    //     });
+    // });
+
     describe("Data Set Cross LPAR", () => {
         const getDatasetSpy    = jest.spyOn(Get, "dataSet");
         const listDatasetSpy   = jest.spyOn(List, "dataSet");

packages/zosfiles/src/methods/copy/Copy.ts

@@ -9,10 +9,10 @@
 *
 */
 
-import { AbstractSession, ImperativeError, ImperativeExpect, ITaskWithStatus, Logger, Headers,
-    IHeaderContent, TaskStage } from "@zowe/imperative";
+import { AbstractSession, ImperativeError, ImperativeExpect, ITaskWithStatus,
+    Logger, Headers, IHeaderContent, TaskStage, IO} from "@zowe/imperative";
 import { posix } from "path";
-
+import * as fs from "fs";
 import { Create, CreateDataSetTypeEnum, ICreateDataSetOptions } from "../create";
 import { Get } from "../get";
 import { Upload } from "../upload";
@@ -26,6 +26,10 @@
 import { IDataSet } from "../../doc/IDataSet";
 import { ICopyDatasetOptions } from "./doc/ICopyDatasetOptions";
 import { ICrossLparCopyDatasetOptions } from "./doc/ICrossLparCopyDatasetOptions";
+import { Download } from "../download";
+import { ZosFilesUtils } from "../../utils/ZosFilesUtils";
+import { tmpdir } from "os";
+import path = require("path");
 /**
  * This class holds helper functions that are used to copy the contents of datasets through the
  * z/OSMF APIs.
@@ -53,6 +57,12 @@
         ImperativeExpect.toBeDefinedAndNonBlank(options["from-dataset"].dsn, "fromDataSetName");
         ImperativeExpect.toBeDefinedAndNonBlank(toDataSetName, "toDataSetName");
 
+        const sourceIsPds = await Copy.isPDS(session, options["from-dataset"].dsn);
+        const targetIsPds = await Copy.isPDS(session, toDataSetName);
+
+        if(sourceIsPds && targetIsPds) {
+            return await Copy.copyPDS(session, options["from-dataset"].dsn, toDataSetName);
+        }
         const endpoint: string = posix.join(
             ZosFilesConstants.RESOURCE,
             ZosFilesConstants.RES_DS_FILES,
@@ -93,6 +103,76 @@
         }
     }
 
+    /**
+     * Private function that checks if a dataset is type PDS
+    **/
+    private static async isPDS(
+        session: AbstractSession,
+        dataSetName: string
+    ): Promise<boolean> {
+        try {
+            const response = await List.dataSet(session, dataSetName, {attributes: true});
+            const dsntp = response.apiResponse.items[0].dsntp;
+            const dsorg = response.apiResponse.items[0].dsorg;
+            return dsntp === "PDS" && dsorg === "PO";
+        }
+        catch(error) {
+            Logger.getAppLogger().error(error);
+            throw error;
+        }
+    }
+
+    /**
+     * Copy the members of a Partitioned dataset into another Partitioned dataset
+     *
+     * @param {AbstractSession}   session        - z/OSMF connection info
+     * @param {IDataSet}          toDataSet      - The data set to copy to
+     * @param {IDataSetOptions}   options        - Options
+     *
+     * @returns {Promise<IZosFilesResponse>} A response indicating the status of the copying
+     *
+     * @throws {ImperativeError} Data set name must be specified as a non-empty string
+     * @throws {Error} When the {@link ZosmfRestClient} throws an error
+     *
+     * @see https://www.ibm.com/support/knowledgecenter/en/SSLTBW_2.1.0/com.ibm.zos.v2r1.izua700/IZUHPINFO_API_PutDataSetMemberUtilities.htm
+     */
+
+    public static async copyPDS (
+        session: AbstractSession,
+        fromPds: string,
+        toPds: string
+    ): Promise<IZosFilesResponse> {
+        try {
+            const sourceResponse = await List.allMembers(session, fromPds);
+            const sourceMemberList: Array<{ member: string }> = sourceResponse.apiResponse.items;
+
+            if(sourceMemberList.length == 0) {
+                return {
+                    success: false,
+                    commandResponse: `Source dataset (${fromPds}) - ` + ZosFilesMessages.noMembersFound.message
+                };
+            }
+
+            const downloadDir = path.join(tmpdir(), fromPds);
+            await Download.allMembers(session, fromPds, {directory:downloadDir});
+            const uploadFileList: string[] = ZosFilesUtils.getFileListFromPath(downloadDir);
+
+            for (const file of uploadFileList) {
+                const uploadingDsn = `${toPds}(${ZosFilesUtils.generateMemberName(file)})`;
+                const uploadStream = IO.createReadStream(file);
+                await Upload.streamToDataSet(session, uploadStream, uploadingDsn);
+            }
+            fs.rmSync(downloadDir, {recursive: true});
+            return {
+                success:true,
+                commandResponse: ZosFilesMessages.datasetCopiedSuccessfully.message
+            };
+        }
+        catch (error) {
+            Logger.getAppLogger().error(error);
+            throw error;
+        }
+    }
 
     /**
      * Copy the contents of a dataset from one LPAR to another LPAR