Copying a PDS into existing PDS w/ no members

packages/zosfiles/CHANGELOG.md

@@ -3,6 +3,7 @@
 All notable changes to the Zowe z/OS files SDK package will be documented in this file.
 
 ## Recent Changes
+-Enhancement: `Copy.dataset` method now recognizes Partioned data sets and can copy members of source PDS into an existing target PDS. [#2386](https://github.com/zowe/zowe-cli/pull/2386)
 
 - Enhancement: Added a `List.membersMatchingPattern` method to download all members that match a specific pattern.[#2359](https://github.com/zowe/zowe-cli/pull/2359)
 

packages/zosfiles/__tests__/__system__/methods/copy/Copy.system.test.ts

@@ -10,14 +10,18 @@
 */
 
 import { Create, Upload, Delete, CreateDataSetTypeEnum, Copy, ZosFilesMessages, Get, IDataSet,
-    ICrossLparCopyDatasetOptions, IGetOptions, IZosFilesResponse } from "../../../../src";
-import { Imperative, Session } from "@zowe/imperative";
+    ICrossLparCopyDatasetOptions, IGetOptions, IZosFilesResponse,
+    ZosFilesUtils} from "../../../../src";
+import { Imperative, IO, Session } from "@zowe/imperative";
 import { inspect } from "util";
 import { TestEnvironment } from "../../../../../../__tests__/__src__/environment/TestEnvironment";
 import { ITestPropertiesSchema } from "../../../../../../__tests__/__src__/properties/ITestPropertiesSchema";
 import { join } from "path";
 import { readFileSync } from "fs";
 import { ITestEnvironment } from "../../../../../../__tests__/__src__/environment/ITestEnvironment";
+import { tmpdir } from "os";
+import path = require("path");
+import * as fs from "fs";
 
 let REAL_SESSION: Session;
 let REAL_TARGET_SESSION: Session;
@@ -98,6 +102,60 @@ describe("Copy", () => {
                     expect(contents1.toString()).toEqual(contents2.toString());
                 });
             });
+            describe("Partioned > Partioned", () => {
+                let downloadDir: string;
+                beforeEach(async () => {
+                    try {
+                        downloadDir = path.join(tmpdir(), fromDataSetName);
+                        fs.mkdirSync(downloadDir, { recursive: true });
+                        const mockFile = path.join(downloadDir, "mockFile.txt");
+                        fs.writeFileSync(mockFile, "test file content");
+
+                        const uploadFileList: string[] = ZosFilesUtils.getFileListFromPath(downloadDir);
+                        const stream = IO.createReadStream(uploadFileList[0]);
+                        await Create.dataSet(REAL_SESSION, CreateDataSetTypeEnum.DATA_SET_SEQUENTIAL, fromDataSetName);
+                        await Create.dataSet(REAL_SESSION, CreateDataSetTypeEnum.DATA_SET_SEQUENTIAL, toDataSetName);
+                        await Upload.streamToDataSet(REAL_SESSION, stream, fromDataSetName);
+                    } catch (err) {
+                        Imperative.console.info(`Error: ${inspect(err)}`);
+                    }
+                });
+                it("Should copy a partitioned data set", async () => {
+                    let error;
+                    let response;
+                    let contents1;
+                    let contents2;
+
+                    try {
+                        response = await Copy.dataSet(
+                            REAL_SESSION,
+                            {dsn: toDataSetName},
+                            {"from-dataset": {
+                                dsn:fromDataSetName
+                            }}
+                        );
+                        contents1 = await Get.dataSet(REAL_SESSION, fromDataSetName);
+                        contents2 = await Get.dataSet(REAL_SESSION, toDataSetName);
+                        Imperative.console.info(`Response: ${inspect(response)}`);
+                    } catch (err) {
+                        error = err;
+                        Imperative.console.info(`Error: ${inspect(err)}`);
+                    }
+
+                    expect(error).toBeFalsy();
+
+                    expect(response).toBeTruthy();
+                    expect(response.success).toBe(true);
+                    expect(response.commandResponse).toContain(ZosFilesMessages.datasetCopiedSuccessfully.message);
+
+                    expect(contents1).toBeTruthy();
+                    expect(contents2).toBeTruthy();
+                    expect(contents1.toString()).toEqual(contents2.toString());
+                });
+                afterEach(() => {
+                    fs.rmSync(downloadDir, { recursive: true, force: true });
+                });
+            });
             describe("Member > Member", () => {
                 beforeEach(async () => {
                     try {

packages/zosfiles/__tests__/__unit__/methods/copy/Copy.unit.test.ts

@@ -9,12 +9,12 @@
 *
 */
 
-import { Session, ImperativeError } from "@zowe/imperative";
+import { Session, ImperativeError, IO } from "@zowe/imperative";
 import { posix } from "path";
-
+import * as fs from "fs";
 import { error } from "console";
 
-import { Copy, Create, Get, List, Upload, ZosFilesConstants, ZosFilesMessages, IZosFilesResponse } from "../../../../src";
+import { Copy, Create, Get, List, Upload, ZosFilesConstants, ZosFilesMessages, IZosFilesResponse, Download, ZosFilesUtils } from "../../../../src";
 import { ZosmfHeaders, ZosmfRestClient } from "@zowe/core-for-zowe-sdk";
 
 describe("Copy", () => {
@@ -33,14 +33,18 @@
         const fromMemberName = "mem1";
         const toDataSetName = "USER.DATA.TO";
         const toMemberName = "mem2";
+        let isPDSSpy: jest.SpyInstance;
 
         beforeEach(() => {
             copyExpectStringSpy.mockClear();
             copyExpectStringSpy.mockImplementation(async () => {
                 return "";
             });
+            isPDSSpy = jest.spyOn(Copy as any, "isPDS").mockResolvedValue(false);
+        });
+        afterAll(() => {
+            isPDSSpy.mockRestore();
         });
-
         describe("Success Scenarios", () => {
             describe("Sequential > Sequential", () => {
                 it("should send a request", async () => {
@@ -438,6 +442,40 @@
                     expect(lastArgumentOfCall).toHaveProperty("replace", false);
                 });
             });
+            describe("Partitioned > Partitioned", () => {
+                let copyPDSSpy = jest.spyOn(Copy, "copyPDS");
+                beforeEach(() => {
+                    copyPDSSpy.mockClear();
+                    copyPDSSpy = jest.spyOn(Copy, "copyPDS").mockResolvedValue({
+                        success:true,
+                        commandResponse: ZosFilesMessages.datasetCopiedSuccessfully.message,
+                    });
+                    isPDSSpy = jest.spyOn(Copy as any, "isPDS").mockResolvedValue(true);
+                });
+                afterAll(() => {
+                    copyPDSSpy.mockRestore();
+                    isPDSSpy.mockRestore();
+                });
+                it("should call copyPDS to copy members of source PDS to target PDS", async () => {
+                    const response = await Copy.dataSet(
+                        dummySession,
+                        {dsn: toDataSetName},
+                        {"from-dataset": {
+                            dsn:fromDataSetName
+                        }}
+                    );
+                    expect(isPDSSpy).toHaveBeenNthCalledWith(1, dummySession, fromDataSetName);
+                    expect(isPDSSpy).toHaveBeenNthCalledWith(2, dummySession, toDataSetName);
+
+                    expect(copyPDSSpy).toHaveBeenCalledTimes(1);
+                    expect(copyPDSSpy).toHaveBeenCalledWith(dummySession, fromDataSetName, toDataSetName);
+
+                    expect(response).toEqual({
+                        success: true,
+                        commandResponse: ZosFilesMessages.datasetCopiedSuccessfully.message
+                    });
+                });
+            });
         });
         describe("Failure Scenarios", () => {
             it("should fail if the zOSMF REST client fails", async () => {
@@ -515,6 +553,107 @@
         });
     });
 
+    describe("Copy Partitioned Data Set", () => {
+        const listAllMembersSpy   = jest.spyOn(List, "allMembers");
+        const downloadAllMembersSpy = jest.spyOn(Download, "allMembers");
+        const uploadSpy = jest.spyOn(Upload, "streamToDataSet");
+        const fileListPathSpy = jest.spyOn(ZosFilesUtils, "getFileListFromPath");
+        const generateMemName = jest.spyOn(ZosFilesUtils, "generateMemberName");
+        const fromDataSetName = "USER.DATA.FROM";
+        const toDataSetName = "USER.DATA.TO";
+        const readStream = jest.spyOn(IO, "createReadStream");
+        const rmSync = jest.spyOn(fs, "rmSync");
+        const listDatasetSpy = jest.spyOn(List, "dataSet");
+
+        const dsPO = {
+            dsname: fromDataSetName,
+            dsorg: "PO",
+        };
+        const dsPS = {
+            dsname: fromDataSetName,
+            dsorg: "PS",
+        };
+
+        it("should detect PDS datasets correctly during copy", async () => {
+            let caughtError;
+            let response;
+            listDatasetSpy.mockImplementation(async (): Promise<any>  => {
+                return {
+                    apiResponse: {
+                        items: [dsPO]
+                    }
+                };
+            });
+            try {
+                response = await Copy.isPDS(dummySession, dsPO.dsname);
+            }
+            catch(e) {
+                caughtError = e;
+            }
+            expect(response).toEqual(true);
+            expect(listDatasetSpy).toHaveBeenCalledWith(dummySession, dsPO.dsname, { attributes: true });
+        });
+
+        it("should return false if the data set is not partitioned", async () => {
+            let response;
+            let caughtError;
+            listDatasetSpy.mockImplementation(async (): Promise<any>  => {
+                return {
+                    apiResponse: {
+                        items: [dsPS]
+                    }
+                };
+            });
+            try {
+                response = await Copy.isPDS(dummySession, dsPS.dsname);
+            }
+            catch(e) {
+                caughtError = e;
+            }
+            expect(response).toEqual(false);
+            expect(listDatasetSpy).toHaveBeenCalledWith(dummySession, dsPS.dsname, { attributes: true });
+        });
+
+        it("should successfully copy members from source to target PDS", async () => {
+            let caughtError;
+            let response;
+            const sourceResponse = {
+                apiResponse: {
+                    items: [
+                        { member: "mem1" },
+                        { member: "mem2" },
+                    ]
+                }
+            };
+            const fileList = ["mem1", "mem2"];
+
+            listAllMembersSpy.mockImplementation(async (): Promise<any>  => sourceResponse);
+            downloadAllMembersSpy.mockImplementation(async (): Promise<any> => undefined);
+            fileListPathSpy.mockReturnValue(fileList);
+            generateMemName.mockReturnValue("mem1");
+            readStream.mockReturnValue("test" as any);
+            uploadSpy.mockResolvedValue(undefined);
+            rmSync.mockImplementation(jest.fn());
+
+
+            try{
+                response = await Copy.copyPDS(dummySession, fromDataSetName, toDataSetName);
+            }
+            catch(e) {
+                caughtError = e;
+            }
+            expect(listAllMembersSpy).toHaveBeenCalledWith(dummySession, fromDataSetName);
+            expect(downloadAllMembersSpy).toHaveBeenCalledWith(dummySession, fromDataSetName, expect.any(Object));
+            expect(fileListPathSpy).toHaveBeenCalled();
+            expect(uploadSpy).toHaveBeenCalledTimes(fileList.length);
+            expect(rmSync).toHaveBeenCalled();
+            expect(response).toEqual({
+                success: true,
+                commandResponse: ZosFilesMessages.datasetCopiedSuccessfully.message,
+            });
+        });
+    });
+
     describe("Data Set Cross LPAR", () => {
         const getDatasetSpy    = jest.spyOn(Get, "dataSet");
         const listDatasetSpy   = jest.spyOn(List, "dataSet");

packages/zosfiles/src/methods/copy/Copy.ts

@@ -9,10 +9,10 @@
 *
 */
 
-import { AbstractSession, ImperativeError, ImperativeExpect, ITaskWithStatus, Logger, Headers,
-    IHeaderContent, TaskStage } from "@zowe/imperative";
+import { AbstractSession, ImperativeError, ImperativeExpect, ITaskWithStatus,
+    Logger, Headers, IHeaderContent, TaskStage, IO} from "@zowe/imperative";
 import { posix } from "path";
-
+import * as fs from "fs";
 import { Create, CreateDataSetTypeEnum, ICreateDataSetOptions } from "../create";
 import { Get } from "../get";
 import { Upload } from "../upload";
@@ -26,6 +26,10 @@
 import { IDataSet } from "../../doc/IDataSet";
 import { ICopyDatasetOptions } from "./doc/ICopyDatasetOptions";
 import { ICrossLparCopyDatasetOptions } from "./doc/ICrossLparCopyDatasetOptions";
+import { Download } from "../download";
+import { ZosFilesUtils } from "../../utils/ZosFilesUtils";
+import { tmpdir } from "os";
+import path = require("path");
 /**
  * This class holds helper functions that are used to copy the contents of datasets through the
  * z/OSMF APIs.
@@ -53,6 +57,12 @@
         ImperativeExpect.toBeDefinedAndNonBlank(options["from-dataset"].dsn, "fromDataSetName");
         ImperativeExpect.toBeDefinedAndNonBlank(toDataSetName, "toDataSetName");
 
+        const sourceIsPds = await this.isPDS(session, options["from-dataset"].dsn);
+        const targetIsPds = await this.isPDS(session, toDataSetName);
+        if (sourceIsPds && targetIsPds) {
+            return await this.copyPDS(session, options["from-dataset"].dsn, toDataSetName);
+        }
+
         const endpoint: string = posix.join(
             ZosFilesConstants.RESOURCE,
             ZosFilesConstants.RES_DS_FILES,
@@ -93,6 +103,75 @@
         }
     }
 
+    /**
+     * Private function that checks if a dataset is type PDS
+    **/
+    public static async isPDS(
+        session: AbstractSession,
+        dataSetName: string
+    ): Promise<boolean> {
+        try {
+            const response = await List.dataSet(session, dataSetName, {attributes: true});
+            const dsorg = response.apiResponse.items[0].dsorg;
+            return dsorg === "POE" || dsorg === "PO";
+        }
+        catch(error) {
+            Logger.getAppLogger().error(error);
+            throw error;
+        }
+    }
+
+    /**
+     * Copy the members of a Partitioned dataset into another Partitioned dataset
+     *
+     * @param {AbstractSession}   session        - z/OSMF connection info
+     * @param {IDataSet}          toDataSet      - The data set to copy to
+     * @param {IDataSetOptions}   options        - Options
+     *
+     * @returns {Promise<IZosFilesResponse>} A response indicating the status of the copying
+     *
+     * @throws {ImperativeError} Data set name must be specified as a non-empty string
+     * @throws {Error} When the {@link ZosmfRestClient} throws an error
+     *
+     * @see https://www.ibm.com/support/knowledgecenter/en/SSLTBW_2.1.0/com.ibm.zos.v2r1.izua700/IZUHPINFO_API_PutDataSetMemberUtilities.htm
+     */
+
+    public static async copyPDS (
+        session: AbstractSession,
+        fromPds: string,
+        toPds: string
+    ): Promise<IZosFilesResponse> {
+        try {
+            const sourceResponse = await List.allMembers(session, fromPds);
+            const sourceMemberList: Array<{ member: string }> = sourceResponse.apiResponse.items;
+
+            if(sourceMemberList.length == 0) {
+                return {
+                    success: false,
+                    commandResponse: `Source dataset (${fromPds}) - ` + ZosFilesMessages.noMembersFound.message
+                };
+            }
+
+            const downloadDir = path.join(tmpdir(), fromPds);
+            await Download.allMembers(session, fromPds, {directory:downloadDir});
+            const uploadFileList: string[] = ZosFilesUtils.getFileListFromPath(downloadDir);
+
+            for (const file of uploadFileList) {
+                const uploadingDsn = `${toPds}(${ZosFilesUtils.generateMemberName(file)})`;
+                const uploadStream = IO.createReadStream(file);
+                await Upload.streamToDataSet(session, uploadStream, uploadingDsn);
+            }
+            fs.rmSync(downloadDir, {recursive: true});
+            return {
+                success:true,
+                commandResponse: ZosFilesMessages.datasetCopiedSuccessfully.message
+            };
+        }
+        catch (error) {
+            Logger.getAppLogger().error(error);
+            throw error;
+        }
+    }
 
     /**
      * Copy the contents of a dataset from one LPAR to another LPAR