Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HPCC4J-567 Adapt WsFileIOClient to latest ESP behavior #703

Merged
merged 1 commit into from
Apr 4, 2024
Merged

HPCC4J-567 Adapt WsFileIOClient to latest ESP behavior #703

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 97 additions & 4 deletions wsclient/src/main/java/org/hpccsystems/ws/client/HPCCWsFileIOClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,12 +232,14 @@ public boolean ping() throws Exception
}

/**
* Creates the HPCC file.
* @deprecated Due to change in server behavior
* Use boolean createHPCCFile(String fileName, String targetLandingZone, boolean overwritefile, String lzAddress) instead
* Creates an HPCC file.
*
* @param fileName
* - The target HPCC file name
* @param targetLandingZone
* - The "netaddress" of the target landing, can be localhost, should be fetched from landingzones in filesprayclient
* - The LZ name, no longer the netaddress of the LZ. should be fetched from landingzones in filesprayclient
* @param overwritefile
* - If the file exists, should it be overwritten?
* @return true, if successful
Expand All @@ -247,17 +249,47 @@ public boolean ping() throws Exception
* the array of esp exception wrapper
*/
public boolean createHPCCFile(String fileName, String targetLandingZone, boolean overwritefile) throws Exception, ArrayOfEspExceptionWrapper
{
return createHPCCFile(fileName, targetLandingZone, overwritefile, null);
}

/**
* Creates an HPCC file.
*
* @param fileName
* - The target HPCC file name
* @param targetLandingZone
* - The LZ name, no longer the netaddress of the LZ. should be fetched from landingzones in filesprayclient
* @param overwritefile
* - If the file exists, should it be overwritten?
* @param lzAddress
* - The Landing zone address
* @return true, if successful
* @throws java.lang.Exception
* - Caller should handle exception in case of errors
* @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper
* the array of esp exception wrapper
*/
public boolean createHPCCFile(String fileName, String targetLandingZone, boolean overwritefile, String lzAddress) throws Exception, ArrayOfEspExceptionWrapper
{
boolean success = false;
log.debug("Attempting to create HPCC File: " + fileName);

if (targetLandingZone == null || targetLandingZone.isEmpty())
throw new Exception("HPCCWsFileIOClient::createHPCCFile: targetLandingZone required!");

if (fileName == null || fileName.isEmpty())
throw new Exception("HPCCWsFileIOClient::createHPCCFile: fileName required!");

verifyStub(); // Throws exception if stub failed

CreateFileRequest request = new CreateFileRequest();

request.setDestDropZone(targetLandingZone);
request.setDestRelativePath(fileName);
request.setOverwrite(overwritefile);
if (lzAddress != null && !lzAddress.isEmpty())
request.setDestNetAddress(lzAddress);

CreateFileResponse resp = null;
try
Expand Down Expand Up @@ -298,13 +330,15 @@ public boolean createHPCCFile(String fileName, String targetLandingZone, boolean
* - At what offset should this be written - Specify 0 if necessary
* @param uploadchunksize
* - Chunksize to upload the data
* @param lzAddress
* - The Landing zone address
* @return true, if successful
* @throws java.lang.Exception
* the exception
* @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper
* the array of esp exception wrapper
*/
public boolean writeHPCCFileData(byte[] data, String fileName, String targetLandingZone, boolean append, long offset, int uploadchunksize)
public boolean writeHPCCFileData(byte[] data, String fileName, String targetLandingZone, boolean append, long offset, int uploadchunksize, String lzAddress)
throws Exception, ArrayOfEspExceptionWrapper
{
boolean success = true;
Expand All @@ -318,6 +352,8 @@ public boolean writeHPCCFileData(byte[] data, String fileName, String targetLand
request.setDestDropZone(targetLandingZone);
request.setDestRelativePath(fileName);
request.setOffset(offset);
if (lzAddress != null && !lzAddress.isEmpty())
request.setDestNetAddress(lzAddress);

int dataindex = 0;
int limit = uploadchunksize <= 0 ? defaultUploadChunkSize : uploadchunksize;
Expand Down Expand Up @@ -368,6 +404,35 @@ public boolean writeHPCCFileData(byte[] data, String fileName, String targetLand
return success;
}

/**
* @deprecated Due to change in server behavior
* Use boolean writeHPCCFileData(byte[] data, String fileName, String targetLandingZone, boolean append, long offset, int uploadchunksize, String lzAddress) instead
* Write HPCC file data.
*
* @param data
* - The data to write
* @param fileName
* - The target HPCC file to write to
* @param targetLandingZone
* - The "netaddress" of the target landing, can be localhost, should be fetched from landingzones in filesprayclient
* @param append
* - Should this data be appended?
* @param offset
* - At what offset should this be written - Specify 0 if necessary
* @param uploadchunksize
* - Chunksize to upload the data
* @return true, if successful
* @throws java.lang.Exception
* the exception
* @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper
* the array of esp exception wrapper
*/
public boolean writeHPCCFileData(byte[] data, String fileName, String targetLandingZone, boolean append, long offset, int uploadchunksize)
throws Exception, ArrayOfEspExceptionWrapper
{
return writeHPCCFileData(data, fileName, targetLandingZone, append, offset, uploadchunksize, null);
}

/**
* Read file data.
*
Expand All @@ -379,19 +444,23 @@ public boolean writeHPCCFileData(byte[] data, String fileName, String targetLand
* the datasize
* @param offset
* the offset
* @param dropzoneAddress
* the dropzone address (not needed in containerized mode)
* @return the string
* @throws java.lang.Exception
* the exception
* @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper
* the array of esp exception wrapper
*/
public String readFileData(String dropzone, String fileName, long datasize, long offset) throws Exception, ArrayOfEspExceptionWrapper
public String readFileData(String dropzone, String fileName, long datasize, long offset, String dropzoneAddress) throws Exception, ArrayOfEspExceptionWrapper
{
ReadFileDataRequest readFileDataRequest = new ReadFileDataRequest();
readFileDataRequest.setDestDropZone(dropzone);
readFileDataRequest.setDestRelativePath(fileName);
readFileDataRequest.setDataSize(datasize);
readFileDataRequest.setOffset(offset);
if (dropzoneAddress != null && !dropzoneAddress.isEmpty())
readFileDataRequest.setDestNetAddress(dropzoneAddress);

ReadFileDataResponse resp = null;
try
Expand Down Expand Up @@ -426,4 +495,28 @@ public String readFileData(String dropzone, String fileName, long datasize, long

return data;
}

/**
* @deprecated Due to change in server behavior
* Use String readFileData(String dropzone, String fileName, long datasize, long offset, String dropzoneAddress) instead
* Read file data.
*
* @param dropzone
* the dropzone
* @param fileName
* the file name
* @param datasize
* the datasize
* @param offset
* the offset
* @return the string
* @throws java.lang.Exception
* the exception
* @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper
* the array of esp exception wrapper
*/
public String readFileData(String dropzone, String fileName, long datasize, long offset) throws Exception, ArrayOfEspExceptionWrapper
{
return readFileData(dropzone, fileName, datasize, offset, null);
}
}
60 changes: 47 additions & 13 deletions wsclient/src/test/java/org/hpccsystems/ws/client/WSFileIOClientTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,24 @@ HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
public class WSFileIOClientTest extends BaseRemoteTest
{
private final static HPCCWsFileIOClient client = wsclient.getWsFileIOClient();

private static boolean isContainerized = false;
private final static String testfilename = System.getProperty("lztestfile", "myfilename.txt");
private final static String targetLZ = System.getProperty("lzname", "localhost");
private final static String targetLZ = System.getProperty("lzname", "mydropzone"); //targetLZ accepted the address "localhost" once upon a time.
private final static String targetLZPath = System.getProperty("lzpath", "/var/lib/HPCCSystems/mydropzone");
private final static String HPCC_30117 = System.getProperty("HPCC30117", "fixed");
private final static String targetLZAddress = System.getProperty("lzaddress", ".");

static
{
try
{
if (client.isTargetHPCCContainerized())
isContainerized = true;
}
catch (Exception e)
{
System.out.println("Could not determine if target service is containerized, default: 'false'");
}

if (System.getProperty("lztestfile") == null)
System.out.println("lztestfile not provided - defaulting to myfilename.txt");

Expand All @@ -60,20 +70,18 @@ public class WSFileIOClientTest extends BaseRemoteTest
if (System.getProperty("lzpath") == null)
System.out.println("lzpath not provided - defaulting to /var/lib/HPCCSystems/mydropzone");

if (System.getProperty("HPCC30117") == null)
System.out.println("HPCC30117 status not provided - defaulting to fixed");
else
System.out.println("HPCC30117 status: '" + HPCC_30117 + "'");
if (System.getProperty("lzaddress") == null)
System.out.println("lzaddress not provided - defaulting to '.'");
}

@Test
public void copyFile() throws Exception
{
String lzfile=System.currentTimeMillis() + "_csvtest.csv";
String hpccfilename="temp::" + lzfile;
client.createHPCCFile(lzfile, targetLZ, true);
client.createHPCCFile(lzfile, targetLZ, true, isContainerized ? null : targetLZAddress);
byte[] data = "Product,SKU,Color\r\nBike,1234,Blue\r\nCar,2345,Red\r\n".getBytes();
client.writeHPCCFileData(data, lzfile, targetLZ, true, 0, 20);
client.writeHPCCFileData(data, lzfile, targetLZ, true, 0, 20, isContainerized ? null : targetLZAddress);
try
{
System.out.println("Starting file spray.");
Expand Down Expand Up @@ -152,23 +160,49 @@ public void copyFile() throws Exception
@Test
public void AcreateHPCCFile() throws Exception, ArrayOfEspExceptionWrapper
{
System.out.println("Creating file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' on HPCC: '" + super.connString +"'");
Assert.assertTrue(client.createHPCCFile(testfilename, targetLZ, true));
if (isContainerized)
{
System.out.println("Creating file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' on HPCC: '" + super.connString +"'");
System.out.println("Target HPCC is containerized, not providing targetLZAddress");
Assert.assertTrue(client.createHPCCFile(testfilename, targetLZ, true, null));
}
else
{
System.out.println("Creating file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' targetLZaddress: '" + targetLZAddress + "' on HPCC: '" + super.connString +"'");
System.out.println("Target HPCC is NOT containerized, providing targetLZAddress");
Assert.assertTrue(client.createHPCCFile(testfilename, targetLZ, true, targetLZAddress));
}
}

@Test
public void BwriteHPCCFile() throws Exception, ArrayOfEspExceptionWrapper
{
System.out.println("Writing data to file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' on HPCC: '" + super.connString +"'");
byte[] data = "HELLO MY DARLING, HELLO MY DEAR!1234567890ABCDEFGHIJKLMNOPQRSTUVXYZ".getBytes();
Assert.assertTrue(client.writeHPCCFileData(data, testfilename, targetLZ, true, 0, 20));
if (isContainerized)
{
Assert.assertTrue(client.writeHPCCFileData(data, testfilename, targetLZ, true, 0, 20, null));
}
else
{
Assert.assertTrue(client.writeHPCCFileData(data, testfilename, targetLZ, true, 0, 20, targetLZAddress));
}
}

@Test
public void CreadHPCCFile() throws Exception, ArrayOfEspExceptionWrapper
{
System.out.println("reading data from file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' on HPCC: '" + super.connString +"'");
byte[] data = "HELLO MY DARLING, HELLO MY DEAR!1234567890ABCDEFGHIJKLMNOPQRSTUVXYZ".getBytes();
String response = client.readFileData(targetLZ, testfilename, data.length, 0);
String response = null;
if (isContainerized)
{
response = client.readFileData(targetLZ, testfilename, data.length, 0, null);
}
else
{
response = client.readFileData(targetLZ, testfilename, data.length, 0, targetLZAddress);
}
Assert.assertNotNull(response);
Assert.assertArrayEquals(data, response.getBytes());
}
Expand Down
Loading