S3A: Use an array-backed buffer for uploading.

Signed-off-by: Pascal Spörri <[email protected]>
IBM · Sep 20, 2023 · 0b56e2b · 0b56e2b
1 parent 6b68291
commit 0b56e2b
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 6 deletions.
diff --git a/examples/sql/run_benchmark.sh b/examples/sql/run_benchmark.sh
@@ -51,12 +51,10 @@ SPARK_HADOOP_S3A_CONFIG=(
     --conf spark.hadoop.fs.s3a.path.style.access=true
     --conf spark.hadoop.fs.s3a.fast.upload=true
     --conf spark.hadoop.fs.s3a.block.size=$(($BLOCK_SIZE * 1024 * 1024))
+    --conf spark.hadoop.fs.s3a.fast.upload.buffer=array
 )
 
 SPARK_S3_SHUFFLE_CONFIG=(
-    --conf spark.hadoop.fs.s3a.access.key=${S3A_ACCESS_KEY}
-    --conf spark.hadoop.fs.s3a.secret.key=${S3A_SECRET_KEY}
-    --conf spark.hadoop.fs.s3a.endpoint=${S3A_ENDPOINT}
     --conf spark.shuffle.manager="org.apache.spark.shuffle.sort.S3ShuffleManager"
     --conf spark.shuffle.sort.io.plugin.class=org.apache.spark.shuffle.S3ShuffleDataIO
     --conf spark.shuffle.checksum.enabled=${CHECKSUM_ENABLED}

diff --git a/examples/terasort/run.sh b/examples/terasort/run.sh
@@ -52,13 +52,11 @@ SPARK_HADOOP_S3A_CONFIG=(
     --conf spark.hadoop.fs.s3a.path.style.access=true
     --conf spark.hadoop.fs.s3a.fast.upload=true
     --conf spark.hadoop.fs.s3a.block.size=$(($BLOCK_SIZE * 1024 * 1024))
+    --conf spark.hadoop.fs.s3a.fast.upload.buffer=array
 )
 
 
 SPARK_S3_SHUFFLE_CONFIG=(
-    --conf spark.hadoop.fs.s3a.access.key=${S3A_ACCESS_KEY}
-    --conf spark.hadoop.fs.s3a.secret.key=${S3A_SECRET_KEY}
-    --conf spark.hadoop.fs.s3a.endpoint=${S3A_ENDPOINT}
     --conf spark.shuffle.manager="org.apache.spark.shuffle.sort.S3ShuffleManager"
     --conf spark.shuffle.sort.io.plugin.class=org.apache.spark.shuffle.S3ShuffleDataIO
     --conf spark.shuffle.checksum.enabled=${CHECKSUM_ENABLED}