From f88e6ac39f5a2c23d1266952fe659dc67ad716ba Mon Sep 17 00:00:00 2001 From: Rodrigo Pastrana Date: Tue, 4 Jun 2024 18:01:35 -0400 Subject: [PATCH] HPCC-586 Code review - Removes dependancy on otlp exporter - Renames project name variable - Adds manual http span on Utils.Connection.sendHTTPRequest - Injects traceparent http header on outgoing Utils.Connection.sendHTTPRequest - Only attempts to autoconfig remotetest if otel.java.global-autoconfig enabled - Catches autoconfiguration exeptions - Adds root span around platform tester Signed-off-by: Rodrigo Pastrana --- pom.xml | 54 ++- .../ws/client/BaseHPCCWsClient.java | 4 +- .../ws/client/utils/Connection.java | 80 +++- .../hpccsystems/ws/client/BaseRemoteTest.java | 29 +- .../client/platform/test/PlatformTester.java | 389 ++++++++++-------- 5 files changed, 335 insertions(+), 221 deletions(-) diff --git a/pom.xml b/pom.xml index de795adf0..86df44679 100644 --- a/pom.xml +++ b/pom.xml @@ -113,35 +113,31 @@ - io.opentelemetry - opentelemetry-api - - - io.opentelemetry - opentelemetry-sdk - - - io.opentelemetry - opentelemetry-exporter-logging - - - io.opentelemetry - opentelemetry-sdk-extension-autoconfigure - - - io.opentelemetry - opentelemetry-sdk-extension-autoconfigure-spi - - - io.opentelemetry - opentelemetry-exporter-otlp - - - - io.opentelemetry.semconv - opentelemetry-semconv - 1.25.0-alpha - + io.opentelemetry + opentelemetry-api + + + io.opentelemetry + opentelemetry-sdk + + + io.opentelemetry + opentelemetry-exporter-logging + + + io.opentelemetry + opentelemetry-sdk-extension-autoconfigure + + + io.opentelemetry + opentelemetry-sdk-extension-autoconfigure-spi + + + + io.opentelemetry.semconv + opentelemetry-semconv + 1.25.0-alpha + junit junit diff --git a/wsclient/src/main/java/org/hpccsystems/ws/client/BaseHPCCWsClient.java b/wsclient/src/main/java/org/hpccsystems/ws/client/BaseHPCCWsClient.java index be69ba1d3..8d74a636f 100644 --- a/wsclient/src/main/java/org/hpccsystems/ws/client/BaseHPCCWsClient.java +++ b/wsclient/src/main/java/org/hpccsystems/ws/client/BaseHPCCWsClient.java @@ -63,7 +63,7 @@ */ public abstract class BaseHPCCWsClient extends DataSingleton { - public static final String INSTRUMENTED_LIBRARY_NAME = "WsClient"; + public static final String PROJECT_NAME = "WsClient"; private static OpenTelemetry globalOTel = null; /** Constant log */ protected static final Logger log = LogManager.getLogger(BaseHPCCWsClient.class); @@ -253,7 +253,7 @@ public Tracer getWsClientTracer() if (globalOTel == null) initOTel(); - return globalOTel.getTracer(INSTRUMENTED_LIBRARY_NAME); + return globalOTel.getTracer(PROJECT_NAME); } /** * All instances of HPCCWsXYZClient should utilize this init function diff --git a/wsclient/src/main/java/org/hpccsystems/ws/client/utils/Connection.java b/wsclient/src/main/java/org/hpccsystems/ws/client/utils/Connection.java index a28da183d..830c86910 100644 --- a/wsclient/src/main/java/org/hpccsystems/ws/client/utils/Connection.java +++ b/wsclient/src/main/java/org/hpccsystems/ws/client/utils/Connection.java @@ -12,9 +12,23 @@ import java.util.Base64; import java.util.Base64.Decoder; import java.util.Base64.Encoder; +import java.util.HashMap; +import java.util.Map; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.hpccsystems.ws.client.BaseHPCCWsClient; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.opentelemetry.context.propagation.TextMapSetter; +import io.opentelemetry.semconv.HttpAttributes; +import io.opentelemetry.semconv.ServerAttributes; /** * Represents and structures connection information. @@ -1028,6 +1042,14 @@ public String sendHTTPRequest(String uri, String method) throws Exception URL url = new URL (getBaseUrl() + (uri != null && uri.startsWith("/") ? "" : "/") + uri); + Span sendHTTPReqSpan = GlobalOpenTelemetry.get().getTracer(BaseHPCCWsClient.PROJECT_NAME) + .spanBuilder(method.toUpperCase() + " " + url.toExternalForm()) + .setAttribute(ServerAttributes.SERVER_ADDRESS, getHost()) + .setAttribute(ServerAttributes.SERVER_PORT, Long.getLong(getPort())) + .setAttribute(HttpAttributes.HTTP_REQUEST_METHOD, HttpAttributes.HttpRequestMethodValues.GET) + .setSpanKind(SpanKind.CLIENT) + .startSpan(); + HttpURLConnection httpURLConnection = (HttpURLConnection) url.openConnection(); //throws IOException Connection.log.info("Sending HTTP " + method + "Request to:" + url.toString()); @@ -1035,32 +1057,56 @@ public String sendHTTPRequest(String uri, String method) throws Exception if (hasCredentials()) { httpURLConnection.setRequestProperty("Authorization", getBasicAuthString()); + sendHTTPReqSpan.setAttribute("hasCredentials", true); } - - httpURLConnection.setRequestMethod(method); //throws ProtocolException - - int responseCode = httpURLConnection.getResponseCode(); //throws IOException - - Connection.log.info("HTTP Response code: " + responseCode); - - if (responseCode == HttpURLConnection.HTTP_OK) //success + else { - BufferedReader in = new BufferedReader(new InputStreamReader(httpURLConnection.getInputStream())); //throws IOException - String inputLine; - StringBuffer response = new StringBuffer(); + sendHTTPReqSpan.setAttribute("hasCredentials", false); + } - while ((inputLine = in.readLine()) != null) // throws IOException + try (Scope scope = sendHTTPReqSpan.makeCurrent()) + { + Span currentSpan = Span.current(); + if (currentSpan != null && currentSpan.getSpanContext().isValid()) { - response.append(inputLine); + //overkill? we could just construct traceparent manually + Map carrier = new HashMap<>(); + TextMapSetter> setter = Map::put; + W3CTraceContextPropagator.getInstance().inject(Context.current(), carrier, setter); + String traceparent = carrier.getOrDefault("traceparent", "00-" + currentSpan.getSpanContext().getTraceId() + "-" + currentSpan.getSpanContext().getSpanId() + "-00"); + + httpURLConnection.setRequestProperty("traceparent", traceparent); } + httpURLConnection.setRequestMethod(method); //throws ProtocolException - in.close(); //throws IOException + int responseCode = httpURLConnection.getResponseCode(); //throws IOException + sendHTTPReqSpan.setAttribute("http.response.status_code", responseCode); + Connection.log.info("HTTP Response code: " + responseCode); - return response.toString(); + if (responseCode == HttpURLConnection.HTTP_OK) //success + { + BufferedReader in = new BufferedReader(new InputStreamReader(httpURLConnection.getInputStream())); //throws IOException + String inputLine; + StringBuffer response = new StringBuffer(); + + while ((inputLine = in.readLine()) != null) // throws IOException + { + response.append(inputLine); + } + + in.close(); //throws IOException + sendHTTPReqSpan.setStatus(StatusCode.OK); + return response.toString(); + } + else + { + sendHTTPReqSpan.setStatus(StatusCode.ERROR); + throw new IOException("HTTP request failed! Code (" + responseCode + ") " + httpURLConnection.getResponseMessage() ); + } } - else + finally { - throw new IOException("HTTP request failed! Code (" + responseCode + ") " + httpURLConnection.getResponseMessage() ); + sendHTTPReqSpan.end(); } } } diff --git a/wsclient/src/test/java/org/hpccsystems/ws/client/BaseRemoteTest.java b/wsclient/src/test/java/org/hpccsystems/ws/client/BaseRemoteTest.java index 36ef4c4f5..ffb5e856f 100644 --- a/wsclient/src/test/java/org/hpccsystems/ws/client/BaseRemoteTest.java +++ b/wsclient/src/test/java/org/hpccsystems/ws/client/BaseRemoteTest.java @@ -40,9 +40,11 @@ HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®. import org.junit.BeforeClass; import org.junit.experimental.categories.Category; +import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.TracerProvider; import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; @Category(org.hpccsystems.commons.annotations.RemoteTests.class) @@ -70,7 +72,7 @@ public abstract class BaseRemoteTest protected final static int testThreadCount = Integer.parseInt(System.getProperty("testthreadcount", "10")); public static final String DEFAULTHPCCFILENAME = "benchmark::all_types::200kb"; - protected static OpenTelemetry globalOTel = AutoConfiguredOpenTelemetrySdk.initialize().getOpenTelemetrySdk(); + protected static OpenTelemetry globalOTel = null; /* * Code to generate superfile with default file as subfile @@ -120,6 +122,31 @@ public static SpanBuilder getRemoteTestTraceBuilder(String spanName) public static void initialize() throws Exception { + if (Boolean.getBoolean("otel.java.global-autoconfigure.enabled")) + { + System.out.println("OpenTelemetry autoconfiguration enabled:"); + System.out.println(" otel.traces.exporter sys property: "+System.getProperty("otel.traces.exporter")); + System.out.println(" OTEL_TRACES_EXPORTER Env var: " + System.getenv("OTEL_TRACES_EXPORTER")); + System.out.println(" OTEL_TRACES_SAMPLER Env var: " + System.getenv("OTEL_TRACES_SAMPLER")); + System.out.println(" otel.traces.sampler sys property: "+System.getProperty("otel.traces.sampler")); + System.out.println(" otel.logs.exporter: "+ System.getProperty("otel.logs.exporter")); + System.out.println(" OTEL_LOGS_EXPORTER Env var: " + System.getenv("OTEL_LOGS_EXPORTER")); + System.out.println(" otel.metrics.exporter: "+ System.getProperty("otel.metrics.exporter")); + System.out.println(" OTEL_METRICS_EXPORTER Env var: " + System.getenv("OTEL_METRICS_EXPORTER")); + try + { + globalOTel = AutoConfiguredOpenTelemetrySdk.initialize().getOpenTelemetrySdk(); + } + catch (Exception e) + { + System.out.println("OpenTelemetry autoconfiguration failed: " + e.getMessage()); + } + } + else + { + globalOTel = GlobalOpenTelemetry.get(); + } + // This allows testing against locally created self signed certs to work. // In production certs will need to be created valid hostnames javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier( diff --git a/wsclient/src/test/java/org/hpccsystems/ws/client/platform/test/PlatformTester.java b/wsclient/src/test/java/org/hpccsystems/ws/client/platform/test/PlatformTester.java index d71e776bd..ea15beda8 100644 --- a/wsclient/src/test/java/org/hpccsystems/ws/client/platform/test/PlatformTester.java +++ b/wsclient/src/test/java/org/hpccsystems/ws/client/platform/test/PlatformTester.java @@ -12,6 +12,7 @@ import org.hpccsystems.ws.client.HPCCWsDFUClient; import org.hpccsystems.ws.client.HPCCWsSMCClient; import org.hpccsystems.ws.client.HPCCWsSQLClient; +import org.hpccsystems.ws.client.HPCCWsWorkUnitsClient; import org.hpccsystems.ws.client.extended.HPCCWsAttributesClient; import org.hpccsystems.ws.client.gen.axis2.wsdfu.v1_39.SecAccessType; import org.hpccsystems.ws.client.platform.PhysicalFile; @@ -31,6 +32,12 @@ import org.junit.experimental.categories.Category; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Scope; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; + @Category(org.hpccsystems.commons.annotations.RemoteTests.class) public class PlatformTester { @@ -175,221 +182,260 @@ else if (currentParam.matches(WSSQLPORTPATTERN)) try { - Platform platform = Platform.get(prot, hpccServer, port, user, pass); - - HPCCWsSMCClient wssmc = platform.getWsClient().getWsSMCClient(); - - Version targetVersion = new Version(wssmc.getHPCCBuild()); - - Version v = platform.getVersion(); - System.out.println(v.toString()); - - HPCCWsClient client1 = platform.checkOutHPCCWsClient(); - HPCCWsClient client2 = platform.checkOutHPCCWsClient(); - HPCCWsClient client3 = platform.checkOutHPCCWsClient(); - HPCCWsClient client4 = platform.checkOutHPCCWsClient(); - if (platform.validateHPCCWsClient(client1)) + OpenTelemetry globalOTel = null; + if (Boolean.getBoolean("otel.java.global-autoconfigure.enabled")) { - HPCCWsAttributesClient wsAttributesClient1 = client1.getWsAttributesClient(); + System.out.println("OpenTelemetry autoconfiguration enabled:"); + System.out.println(" otel.traces.exporter sys property: "+System.getProperty("otel.traces.exporter")); + System.out.println(" OTEL_TRACES_EXPORTER Env var: " + System.getenv("OTEL_TRACES_EXPORTER")); + System.out.println(" OTEL_TRACES_SAMPLER Env var: " + System.getenv("OTEL_TRACES_SAMPLER")); + System.out.println(" otel.traces.sampler sys property: "+System.getProperty("otel.traces.sampler")); + System.out.println(" otel.logs.exporter: "+ System.getProperty("otel.logs.exporter")); + System.out.println(" OTEL_LOGS_EXPORTER Env var: " + System.getenv("OTEL_LOGS_EXPORTER")); + System.out.println(" otel.metrics.exporter: "+ System.getProperty("otel.metrics.exporter")); + System.out.println(" OTEL_METRICS_EXPORTER Env var: " + System.getenv("OTEL_METRICS_EXPORTER")); + try + { + globalOTel = AutoConfiguredOpenTelemetrySdk.initialize().getOpenTelemetrySdk(); + } + catch (Exception e) + { + System.out.println("OpenTelemetry autoconfiguration failed: " + e.getMessage()); + } } - - if (platform.validateHPCCWsClient(client2)) + else { - HPCCWsAttributesClient wsAttributesClient2 = client2.getWsAttributesClient(); - platform.expireHPCCWsClient(client2); - if (!platform.validateHPCCWsClient(client2)) - System.out.println("not validated"); - else - wsAttributesClient2 = client2.getWsAttributesClient(); + globalOTel = GlobalOpenTelemetry.get(); } - HPCCWsAttributesClient wsAttributesClient3 = client3.getWsAttributesClient(); - HPCCWsAttributesClient wsAttributesClient4 = client4.getWsAttributesClient(); + Span rootSpan = globalOTel.getTracer("PlatformTester").spanBuilder("rootspan").startSpan(); + try (Scope scope = rootSpan.makeCurrent()) + { + Platform platform = Platform.get(prot, hpccServer, port, user, pass); + + HPCCWsSMCClient wssmc = platform.getWsClient().getWsSMCClient(); - platform.checkInHPCCWsClient(client2); - platform.checkInHPCCWsClient(client3); - platform.checkInHPCCWsClient(client4); + Version targetVersion = new Version(wssmc.getHPCCBuild()); - org.hpccsystems.ws.client.platform.DropZone[] dropzones = platform.getDropZones(); - for(int i = 0; i < dropzones.length; i++) - { - System.out.println("Dropzone Name: " + dropzones[i].getName()); - System.out.println("Dropzone Directory: " + dropzones[i].getDirectory()); - System.out.println("Dropzone Machines: "); - PhysicalMachine [] dzmachines = dropzones[i].getMachines(); - for (PhysicalMachine physicalmachine : dzmachines) + Version v = platform.getVersion(); + System.out.println(v.toString()); + + HPCCWsClient client1 = platform.checkOutHPCCWsClient(); + HPCCWsClient client2 = platform.checkOutHPCCWsClient(); + HPCCWsClient client3 = platform.checkOutHPCCWsClient(); + HPCCWsClient client4 = platform.checkOutHPCCWsClient(); + if (platform.validateHPCCWsClient(client1)) + { + HPCCWsAttributesClient wsAttributesClient1 = client1.getWsAttributesClient(); + } + + if (platform.validateHPCCWsClient(client2)) + { + HPCCWsAttributesClient wsAttributesClient2 = client2.getWsAttributesClient(); + platform.expireHPCCWsClient(client2); + if (!platform.validateHPCCWsClient(client2)) + System.out.println("not validated"); + else + wsAttributesClient2 = client2.getWsAttributesClient(); + } + + HPCCWsAttributesClient wsAttributesClient3 = client3.getWsAttributesClient(); + HPCCWsAttributesClient wsAttributesClient4 = client4.getWsAttributesClient(); + + platform.checkInHPCCWsClient(client2); + platform.checkInHPCCWsClient(client3); + platform.checkInHPCCWsClient(client4); + + org.hpccsystems.ws.client.platform.DropZone[] dropzones = platform.getDropZones(); + for(int i = 0; i < dropzones.length; i++) { - System.out.println("\tName: " + physicalmachine.getName()); - System.out.println("\tConfigured Address: " + physicalmachine.getConfigNetaddress()); - System.out.println("\tActual Address: " + physicalmachine.getNetaddress()); - System.out.println("\tOS: " + physicalmachine.getOSName()); - boolean isWin = physicalmachine.getOSCode() == HPCCEnvOSCode.MachineOsW2K; - System.out.println("\tFiles: "); - - PhysicalFile[] physicalFiles = physicalmachine.getFiles(); - for (PhysicalFile physicalFile : physicalFiles) + System.out.println("Dropzone Name: " + dropzones[i].getName()); + System.out.println("Dropzone Directory: " + dropzones[i].getDirectory()); + System.out.println("Dropzone Machines: "); + PhysicalMachine [] dzmachines = dropzones[i].getMachines(); + for (PhysicalMachine physicalmachine : dzmachines) { - String name = physicalFile.getName() + (physicalFile.getIsDir() ? (!isWin ? "/" : "\\") : ""); - System.out.format( "\t\t%-30s %15s %15s\n", name, physicalFile.getIsDir() ? "" : physicalFile.getFilesize() , physicalFile.getModifiedtime()); + System.out.println("\tName: " + physicalmachine.getName()); + System.out.println("\tConfigured Address: " + physicalmachine.getConfigNetaddress()); + System.out.println("\tActual Address: " + physicalmachine.getNetaddress()); + System.out.println("\tOS: " + physicalmachine.getOSName()); + boolean isWin = physicalmachine.getOSCode() == HPCCEnvOSCode.MachineOsW2K; + System.out.println("\tFiles: "); + + PhysicalFile[] physicalFiles = physicalmachine.getFiles(); + for (PhysicalFile physicalFile : physicalFiles) + { + String name = physicalFile.getName() + (physicalFile.getIsDir() ? (!isWin ? "/" : "\\") : ""); + System.out.format( "\t\t%-30s %15s %15s\n", name, physicalFile.getIsDir() ? "" : physicalFile.getFilesize() , physicalFile.getModifiedtime()); + } } } - } - HPCCFileSprayClient fsc = client1.getFileSprayClient(); - List dzLocal = fsc.fetchLocalDropZones(); - if (dzLocal != null && dzLocal.size() > 0) - { - System.out.println("fetchLocalDropZones test ..."); - for(int i = 0; i < dzLocal.size(); i++) + HPCCFileSprayClient fsc = client1.getFileSprayClient(); + List dzLocal = fsc.fetchLocalDropZones(); + if (dzLocal != null && dzLocal.size() > 0) { - DropZoneWrapper thisDZ = dzLocal.get(i); - boolean islinux = thisDZ.getLinux().equals("false") ? false : true; - - System.out.println("DropZone[" + i + "]"); - System.out.println("\tName: " + thisDZ.getName()); - System.out.println("\tPath: " + thisDZ.getPath()); - System.out.println("\tNetAddress: " + thisDZ.getNetAddress()); - System.out.println("\tComputer: " + thisDZ.getComputer()); - System.out.println("\tIsLinux: " + thisDZ.getLinux()); - - List pfs = fsc.listFiles(dzLocal.get(i).getNetAddress(), dzLocal.get(i).getPath(), null); - System.out.println("\tFile Listing:"); - if (pfs != null && pfs.size()> 0) + System.out.println("fetchLocalDropZones test ..."); + for(int i = 0; i < dzLocal.size(); i++) { - for(int fileindex = 0; fileindex < pfs.size(); fileindex++) + DropZoneWrapper thisDZ = dzLocal.get(i); + boolean islinux = thisDZ.getLinux().equals("false") ? false : true; + + System.out.println("DropZone[" + i + "]"); + System.out.println("\tName: " + thisDZ.getName()); + System.out.println("\tPath: " + thisDZ.getPath()); + System.out.println("\tNetAddress: " + thisDZ.getNetAddress()); + System.out.println("\tComputer: " + thisDZ.getComputer()); + System.out.println("\tIsLinux: " + thisDZ.getLinux()); + + List pfs = fsc.listFiles(dzLocal.get(i).getNetAddress(), dzLocal.get(i).getPath(), null); + System.out.println("\tFile Listing:"); + if (pfs != null && pfs.size()> 0) { - String name = pfs.get(fileindex).getName() + (pfs.get(fileindex).getIsDir() ? (islinux ? "/" : "\\") : ""); - System.out.format( "\t\t%-30s %15s %15s\n", name, pfs.get(fileindex).getIsDir() ? "" : pfs.get(fileindex).getFilesize() , pfs.get(fileindex).getModifiedtime()); + for(int fileindex = 0; fileindex < pfs.size(); fileindex++) + { + String name = pfs.get(fileindex).getName() + (pfs.get(fileindex).getIsDir() ? (islinux ? "/" : "\\") : ""); + System.out.format( "\t\t%-30s %15s %15s\n", name, pfs.get(fileindex).getIsDir() ? "" : pfs.get(fileindex).getFilesize() , pfs.get(fileindex).getModifiedtime()); + } } } } - } - List dzByAddress = fsc.fetchDropZones(hpccServer); - if (dzByAddress != null && dzByAddress.size() > 0) - { - System.out.println("fetchDropZones by address test ..."); - for (int i = 0; i < dzByAddress.size(); i++) + List dzByAddress = fsc.fetchDropZones(hpccServer); + if (dzByAddress != null && dzByAddress.size() > 0) { - DropZoneWrapper thisDZ = dzByAddress.get(i); - boolean islinux = thisDZ.getLinux().equals("false") ? false : true; - - System.out.println("DropZone[" + i + "]"); - System.out.println("\tName: " + thisDZ.getName()); - System.out.println("\tNetAddress: " + thisDZ.getNetAddress()); - System.out.println("\tPath: " + thisDZ.getPath()); - System.out.println("\tComputer: " + thisDZ.getComputer()); - System.out.println("\tIsLinux: " + thisDZ.getLinux()); - - List pfs = fsc.listFiles(thisDZ.getNetAddress(), thisDZ.getPath(), null); - System.out.println("\tFile Listing:"); - if (pfs != null && pfs.size() > 0) + System.out.println("fetchDropZones by address test ..."); + for (int i = 0; i < dzByAddress.size(); i++) { - for (int fileindex = 0; fileindex < pfs.size(); fileindex++) { - PhysicalFileStructWrapper thisfile = pfs.get(fileindex); - String name = thisfile.getName() + (thisfile.getIsDir() ? (islinux ? "/" : "\\") : ""); - System.out.format("\t\t%-30s %15s %15s\n", name, thisfile.getIsDir() ? "" : thisfile.getFilesize(), thisfile.getModifiedtime()); + DropZoneWrapper thisDZ = dzByAddress.get(i); + boolean islinux = thisDZ.getLinux().equals("false") ? false : true; + + System.out.println("DropZone[" + i + "]"); + System.out.println("\tName: " + thisDZ.getName()); + System.out.println("\tNetAddress: " + thisDZ.getNetAddress()); + System.out.println("\tPath: " + thisDZ.getPath()); + System.out.println("\tComputer: " + thisDZ.getComputer()); + System.out.println("\tIsLinux: " + thisDZ.getLinux()); + + List pfs = fsc.listFiles(thisDZ.getNetAddress(), thisDZ.getPath(), null); + System.out.println("\tFile Listing:"); + if (pfs != null && pfs.size() > 0) + { + for (int fileindex = 0; fileindex < pfs.size(); fileindex++) { + PhysicalFileStructWrapper thisfile = pfs.get(fileindex); + String name = thisfile.getName() + (thisfile.getIsDir() ? (islinux ? "/" : "\\") : ""); + System.out.format("\t\t%-30s %15s %15s\n", name, thisfile.getIsDir() ? "" : thisfile.getFilesize(), thisfile.getModifiedtime()); + } } } } - } - List pfs = fsc.listFiles(dzByAddress.get(0).getNetAddress(), dzByAddress.get(0).getPath(), null); + List pfs = fsc.listFiles(dzByAddress.get(0).getNetAddress(), dzByAddress.get(0).getPath(), null); - // Test file download - System.out.println("Download test ..."); - String fileName = null; - for (int i = 0; pfs != null && i < pfs.size(); i++) - { - if (pfs.get(i).getIsDir() == false - && pfs.get(i).getFilesize() < 4 * 1024 * 1024) // Only download small files for the test + // Test file download + System.out.println("Download test ..."); + String fileName = null; + for (int i = 0; pfs != null && i < pfs.size(); i++) { - fileName = pfs.get(i).getName(); - break; + if (pfs.get(i).getIsDir() == false + && pfs.get(i).getFilesize() < 4 * 1024 * 1024) // Only download small files for the test + { + fileName = pfs.get(i).getName(); + break; + } } - } - if (fileName != null) - { - System.out.println("Attempting to download: " + fileName + " from DropZone"); - String outputFile = System.getProperty("java.io.tmpdir") + File.separator + fileName; - System.out.println("Output File: " + outputFile); + if (fileName != null) + { + System.out.println("Attempting to download: " + fileName + " from DropZone"); + String outputFile = System.getProperty("java.io.tmpdir") + File.separator + fileName; + System.out.println("Output File: " + outputFile); - File tmpFile = new File(outputFile); + File tmpFile = new File(outputFile); - long bytesTransferred = fsc.downloadFile(tmpFile,dzByAddress.get(0),fileName); - if (bytesTransferred <= 0) - { - System.out.println("Download failed."); + long bytesTransferred = fsc.downloadFile(tmpFile,dzByAddress.get(0),fileName); + if (bytesTransferred <= 0) + { + System.out.println("Download failed."); + } + else + { + System.out.println("File Download Test: Bytes transferred: " + bytesTransferred); + } } else { - System.out.println("File Download Test: Bytes transferred: " + bytesTransferred); + System.out.println("Skipping file download test. No small files found in DropZone."); } - } - else - { - System.out.println("Skipping file download test. No small files found in DropZone."); - } - String tmpPeopleFile = System.getProperty("java.io.tmpdir") + File.separator + "people"; - writeFile( tmpPeopleFile, Persons.data, false); + String tmpPeopleFile = System.getProperty("java.io.tmpdir") + File.separator + "people"; + writeFile( tmpPeopleFile, Persons.data, false); - String tmpAccountFile = System.getProperty("java.io.tmpdir") + File.separator + "account"; - writeFile( tmpAccountFile, Accounts.data, false); + String tmpAccountFile = System.getProperty("java.io.tmpdir") + File.separator + "account"; + writeFile( tmpAccountFile, Accounts.data, false); - int pport = HPCCWsAttributesClient.getServiceWSDLPort(); - HPCCWsClient connector = platform.checkOutHPCCWsClient(); - connector.setVerbosemode(true); - System.out.println("wsdfu ver: " + connector.getwsDFUClientClientVer()); - HPCCWsDFUClient wsDFUClient = connector.getWsDFUClient(); - if (v.getMajor() == 7 && v.getMinor() == 0) - { - System.out.println("Attempting file access on HPCC 7.0.x cluster..."); - DFUFileAccessInfoWrapper a = wsDFUClient.getFileAccess(SecAccessType.Read, "benchmark::integer::2mb", "thor_160", 120, "random", true, true, true); - } - else if (v.getMajor() == 7 && v.getMinor() > 0) - { - System.out.println("Attempting file access on HPCC 7.0.x cluster..."); - wsDFUClient.getFileAccess("benchmark::integer::2mb", "thor_160", 120, "random"); - } - platform.checkInHPCCWsClient(connector); + int pport = HPCCWsAttributesClient.getServiceWSDLPort(); + HPCCWsClient connector = platform.checkOutHPCCWsClient(); + connector.setVerbosemode(true); + System.out.println("wswu ver: " + connector.getWsWorkunitsClientVer()); + HPCCWsWorkUnitsClient wsWUClient = connector.getWsWorkunitsClient(); + wsWUClient.ping(); - connector = platform.checkOutHPCCWsClient(); - System.out.println("wsfileio ver: " + connector.getWsFileIOClientVer()); - System.out.println("wssmc ver: " + connector.getWsSMCClientClientVer()); - System.out.println("wspackageprocess ver: " + connector.getHPCCWsPackageProcessClientVer()); + System.out.println("wsdfu ver: " + connector.getwsDFUClientClientVer()); - List newgetFileDataColumns = wsDFUClient.getFileMetaData(".::kw_test_sup", null); + HPCCWsDFUClient wsDFUClient = connector.getWsDFUClient(); + if (v.getMajor() == 7 && v.getMinor() == 0) + { + System.out.println("Attempting file access on HPCC 7.0.x cluster..."); + DFUFileAccessInfoWrapper a = wsDFUClient.getFileAccess(SecAccessType.Read, "benchmark::integer::2mb", "thor_160", 120, "random", true, true, true); + } + else if (v.getMajor() == 7 && v.getMinor() > 0) + { + System.out.println("Attempting file access on HPCC 7.0.x cluster..."); + wsDFUClient.getFileAccess("benchmark::integer::2mb", "thor_160", 120, "random"); + } + platform.checkInHPCCWsClient(connector); - for (DFUDataColumnWrapper wrapper : newgetFileDataColumns) - { - System.out.println("Col name: " + wrapper.getColumnLabel() + " ecl: " + wrapper.getColumnEclType() + " col type " + wrapper.getColumnType()); - } + connector = platform.checkOutHPCCWsClient(); + System.out.println("wsfileio ver: " + connector.getWsFileIOClientVer()); + System.out.println("wssmc ver: " + connector.getWsSMCClientClientVer()); + System.out.println("wspackageprocess ver: " + connector.getHPCCWsPackageProcessClientVer()); - try - { - //WSSQL Test - HPCCWsSQLClient wsSQLClient = platform.getWsClient().getWsSQLClient(wssqlport); - String s = "CREATE TABLE newtablename (FirstName VARCHAR(15), LastName VARCHAR(25), MiddleName VARCHAR(15) ,StreetAddress VARCHAR(42), city VARCHAR(20), state VARCHAR(2), zip VARCHAR(5)) LOAD DATA INFILE 'people-small' CONNECTION '10.0.2.15' DIRECTORY '/var/lib/HPCCSystems/mydropzone' INTO TABLE newtablename"; - ExecuteSQLResponseWrapper executeSQLFullResponse = wsSQLClient.executeSQLFullResponse(s, "thor", "thor", Integer.valueOf(0), Integer.valueOf(0),Integer.valueOf(0), false, false, "me", Integer.valueOf(-1)); - System.out.println(executeSQLFullResponse.getResult()); - - s = "SELECT * from newtablename where state = 'FL' limit 10;"; - executeSQLFullResponse = wsSQLClient.executeSQLFullResponse(s, "thor", "thor", Integer.valueOf(0), Integer.valueOf(0),Integer.valueOf(0), false, false, "me", Integer.valueOf(-1)); - System.out.println(executeSQLFullResponse.getResult()); - } - catch (java.net.ConnectException e) - { - System.out.println("Could not connect to WsSQL on port: " + wssqlport + "\n>>" + e.getLocalizedMessage()); - } - catch (ArrayOfBaseExceptionWrapper e) - { - e.printStackTrace(); + List newgetFileDataColumns = wsDFUClient.getFileMetaData(".::kw_test_sup", null); + + for (DFUDataColumnWrapper wrapper : newgetFileDataColumns) + { + System.out.println("Col name: " + wrapper.getColumnLabel() + " ecl: " + wrapper.getColumnEclType() + " col type " + wrapper.getColumnType()); + } + + try + { + //WSSQL Test + HPCCWsSQLClient wsSQLClient = platform.getWsClient().getWsSQLClient(wssqlport); + String s = "CREATE TABLE newtablename (FirstName VARCHAR(15), LastName VARCHAR(25), MiddleName VARCHAR(15) ,StreetAddress VARCHAR(42), city VARCHAR(20), state VARCHAR(2), zip VARCHAR(5)) LOAD DATA INFILE 'people-small' CONNECTION '10.0.2.15' DIRECTORY '/var/lib/HPCCSystems/mydropzone' INTO TABLE newtablename"; + ExecuteSQLResponseWrapper executeSQLFullResponse = wsSQLClient.executeSQLFullResponse(s, "thor", "thor", Integer.valueOf(0), Integer.valueOf(0),Integer.valueOf(0), false, false, "me", Integer.valueOf(-1)); + System.out.println(executeSQLFullResponse.getResult()); + + s = "SELECT * from newtablename where state = 'FL' limit 10;"; + executeSQLFullResponse = wsSQLClient.executeSQLFullResponse(s, "thor", "thor", Integer.valueOf(0), Integer.valueOf(0),Integer.valueOf(0), false, false, "me", Integer.valueOf(-1)); + System.out.println(executeSQLFullResponse.getResult()); + } + catch (java.net.ConnectException e) + { + System.out.println("Could not connect to WsSQL on port: " + wssqlport + "\n>>" + e.getLocalizedMessage()); + } + catch (ArrayOfBaseExceptionWrapper e) + { + e.printStackTrace(); + } + catch (Exception e) + { + System.out.println("Encountered issue while testing WsSQL on port: " + wssqlport + "\n>>" + e.getLocalizedMessage()); + } } - catch (Exception e) + finally { - System.out.println("Encountered issue while testing WsSQL on port: " + wssqlport + "\n>>" + e.getLocalizedMessage()); + rootSpan.end(); } } @@ -401,7 +447,6 @@ else if (v.getMajor() == 7 && v.getMinor() > 0) { System.out.println("\n****WsClient HPCC platform tester has finished****\n" ); } - } }