From b6fb2e6b54be0a1776589154b1f545ff3c0eba32 Mon Sep 17 00:00:00 2001 From: albertshau Date: Thu, 2 Nov 2023 09:43:21 -0700 Subject: [PATCH] CDAP-20858 checkstyle fixes --- .../runtime/spi/common/DataprocMetric.java | 3 +- .../runtime/spi/common/DataprocUtils.java | 6 ++++ .../runtimejob/DataprocRuntimeJobManager.java | 35 ++++++++++++++----- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/common/DataprocMetric.java b/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/common/DataprocMetric.java index 738c0fb6616f..1e3933bca5e9 100644 --- a/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/common/DataprocMetric.java +++ b/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/common/DataprocMetric.java @@ -20,7 +20,7 @@ import javax.annotation.Nullable; /** - * Dataproc related metric + * Dataproc related metric. */ public class DataprocMetric { private final String region; @@ -98,6 +98,7 @@ public Builder setLaunchMode(@Nullable LaunchMode launchMode) { /** * Returns a DataprocMetric. + * * @return DataprocMetric. */ public DataprocMetric build() { diff --git a/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/common/DataprocUtils.java b/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/common/DataprocUtils.java index 5efb82abc50f..d29b63817325 100644 --- a/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/common/DataprocUtils.java +++ b/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/common/DataprocUtils.java @@ -462,6 +462,12 @@ private static void updateTemporaryHoldOnGcsObject(Storage storage, String bucke } } + /** + * Get a user friendly message pointing to an external troubleshooting doc. + * + * @param troubleshootingDocsUrl Url for the troubleshooting doc + * @return user friendly message pointing to an external troubleshooting doc. + */ public static String getTroubleshootingHelpMessage(@Nullable String troubleshootingDocsUrl) { if (Strings.isNullOrEmpty(troubleshootingDocsUrl)) { return ""; diff --git a/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/runtimejob/DataprocRuntimeJobManager.java b/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/runtimejob/DataprocRuntimeJobManager.java index b59b6bf534af..0d252dc0462e 100644 --- a/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/runtimejob/DataprocRuntimeJobManager.java +++ b/cdap-runtime-ext-dataproc/src/main/java/io/cdap/cdap/runtime/spi/runtimejob/DataprocRuntimeJobManager.java @@ -544,7 +544,7 @@ private LocalFile uploadCacheableFile(String bucket, String targetFilePath, try { LOG.debug("Uploading a file of size {} bytes from {} to gs://{}/{}", localFile.getSize(), localFile.getURI(), bucket, targetFilePath); - uploadToGCSUtil(localFile, storage, targetFilePath, newBlobInfo, + uploadToGcsUtil(localFile, storage, targetFilePath, newBlobInfo, Storage.BlobWriteOption.generationMatch(), Storage.BlobWriteOption.metagenerationMatch()); } catch (StorageException e) { @@ -600,7 +600,7 @@ private LocalFile uploadFile(String bucket, String targetFilePath, localFile.getSize(), localFile.getURI(), bucket, targetFilePath, bucketObj.getLocationType(), bucketObj.getLocation()); try { - uploadToGCSUtil(localFile, storage, targetFilePath, blobInfo, + uploadToGcsUtil(localFile, storage, targetFilePath, blobInfo, Storage.BlobWriteOption.doesNotExist()); } catch (StorageException e) { if (e.getCode() != HttpURLConnection.HTTP_PRECON_FAILED) { @@ -613,7 +613,7 @@ private LocalFile uploadFile(String bucket, String targetFilePath, // Overwrite the file Blob existingBlob = storage.get(blobId); BlobInfo newBlobInfo = existingBlob.toBuilder().setContentType(contentType).build(); - uploadToGCSUtil(localFile, storage, targetFilePath, newBlobInfo, + uploadToGcsUtil(localFile, storage, targetFilePath, newBlobInfo, Storage.BlobWriteOption.generationNotMatch()); } else { LOG.debug("Skip uploading file {} to gs://{}/{} because it exists.", @@ -637,11 +637,11 @@ private long getCustomTime() { /** * Uploads the file to GCS Bucket. */ - private void uploadToGCSUtil(LocalFile localFile, Storage storage, String targetFilePath, + private void uploadToGcsUtil(LocalFile localFile, Storage storage, String targetFilePath, BlobInfo blobInfo, Storage.BlobWriteOption... blobWriteOptions) throws IOException, StorageException { long start = System.nanoTime(); - uploadToGCS(localFile.getURI(), storage, blobInfo, blobWriteOptions); + uploadToGcs(localFile.getURI(), storage, blobInfo, blobWriteOptions); long end = System.nanoTime(); LOG.debug("Successfully uploaded file {} to gs://{}/{} in {} ms.", localFile.getURI(), bucket, targetFilePath, TimeUnit.NANOSECONDS.toMillis(end - start)); @@ -650,7 +650,7 @@ private void uploadToGCSUtil(LocalFile localFile, Storage storage, String target /** * Uploads the file to GCS bucket. */ - private void uploadToGCS(java.net.URI localFileUri, Storage storage, BlobInfo blobInfo, + private void uploadToGcs(java.net.URI localFileUri, Storage storage, BlobInfo blobInfo, Storage.BlobWriteOption... blobWriteOptions) throws IOException, StorageException { try (InputStream inputStream = openStream(localFileUri); WriteChannel writer = storage.writer(blobInfo, blobWriteOptions)) { @@ -753,6 +753,17 @@ private SubmitJobRequest getSubmitJobRequest(RuntimeJobInfo runtimeJobInfo, .build(); } + /** + * Get the list of arguments to pass to the runtime job on the command line. + * The DataprocJobMain argument is [class-name] [spark-compat] [list of archive files...] + * + * @param runtimeJobInfo information about the runtime job + * @param localFiles files to localize + * @param sparkCompat spark compat version + * @param applicationJarLocalizedName localized application jar name + * @param launchMode launch mode for the job + * @return list of arguments to pass to the runtime job on the command line + */ @VisibleForTesting public static List getArguments(RuntimeJobInfo runtimeJobInfo, List localFiles, String sparkCompat, String applicationJarLocalizedName, @@ -773,6 +784,12 @@ public static List getArguments(RuntimeJobInfo runtimeJobInfo, List getProperties(RuntimeJobInfo runtimeJobInfo) { ProgramRunInfo runInfo = runtimeJobInfo.getProgramRunInfo(); @@ -870,9 +887,9 @@ private String getPath(String... pathSubComponents) { /** * Returns job name from run info. namespace, application, program, run(36 characters) Example: * namespace_application_program_8e1cb2ce-a102-48cf-a959-c4f991a2b475 - *

- * The ID must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), or hyphens (-). - * The maximum length is 100 characters. + * + *

The ID must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), or hyphens (-). + * The maximum length is 100 characters.

* * @throws IllegalArgumentException if provided id does not comply with naming restrictions */