#216: Added S3 endpoint region support (#221)

Fixes #216

doc/changes/changelog.md

@@ -1,6 +1,6 @@
 # Changes
 
-* [2.4.1](changes_2.4.1.md)
+* [2.5.0](changes_2.5.0.md)
 * [2.4.0](changes_2.4.0.md)
 * [2.3.2](changes_2.3.2.md)
 * [2.3.1](changes_2.3.1.md)

doc/changes/changes_2.4.1.md → doc/changes/changes_2.5.0.md

@@ -1,6 +1,14 @@
-# Cloud Storage Extension 2.4.1, released 2022-??-??
+# Cloud Storage Extension 2.5.0, released 2022-09-29
 
-Code name:
+Code name: S3 Endpoint Region Support
+
+## Summary
+
+In this release, we added support for accessing S3 buckets using VPC interface endpoints that are behind [PrivateLink](https://docs.aws.amazon.com/AmazonS3/latest/userguide/privatelink-interface-endpoints.html). Moreover, we updated documentation and dependencies.
+
+## Features
+
+* #216: Added support for S3 endpoint region
 
 ## Documentation
 
@@ -18,7 +26,10 @@ Code name:
 
 * Updated `com.exasol:error-reporting-java:0.4.1` to `1.0.0`
 * Updated `com.exasol:parquet-io-java:1.3.1` to `1.3.3`
+* Updated `com.fasterxml.jackson.core:jackson-databind:2.13.3` to `2.13.4`
+* Updated `com.fasterxml.jackson.module:jackson-module-scala_2.13:2.13.3` to `2.13.4`
 * Updated `com.google.protobuf:protobuf-java:3.21.2` to `3.21.6`
+* Updated `io.delta:delta-core_2.13:1.2.1` to `2.1.0`
 * Updated `io.grpc:grpc-netty:1.47.0` to `1.49.1`
 * Updated `io.netty:netty-all:4.1.79.Final` to `4.1.82.Final`
 * Updated `org.alluxio:alluxio-core-client-hdfs:2.8.0-2` to `2.8.1`
@@ -34,8 +45,8 @@ Code name:
 * Added `org.apache.logging.log4j:log4j-core:2.19.0`
 * Updated `org.apache.orc:orc-core:1.7.5` to `1.8.0`
 * Updated `org.scala-lang:scala-library:2.13.8` to `2.13.9`
-* Updated `org.slf4j:jul-to-slf4j:1.7.36` to `2.0.2`
-* Updated `org.slf4j:slf4j-log4j12:1.7.36` to `2.0.2`
+* Updated `org.slf4j:jul-to-slf4j:1.7.36` to `2.0.3`
+* Updated `org.slf4j:slf4j-log4j12:1.7.36` to `2.0.3`
 
 ### Test Dependency Updates
 

doc/user_guide/user_guide.md

@@ -140,7 +140,7 @@ downloaded jar file is the same as the checksum provided in the releases.
 To check the SHA256 result of the local jar, run the command:
 
 ```sh
-sha256sum exasol-cloud-storage-extension-2.4.1.jar
+sha256sum exasol-cloud-storage-extension-2.5.0.jar
 ```
 
 ### Building From Source
@@ -170,7 +170,7 @@ mvn clean package -DskipTests=true
 ```
 
 The assembled jar file should be located at
-`target/scala-2.12/exasol-cloud-storage-extension-2.4.1.jar`.
+`target/scala-2.12/exasol-cloud-storage-extension-2.5.0.jar`.
 
 ### Create an Exasol Bucket
 
@@ -192,8 +192,8 @@ for the HTTP protocol.
 Upload the jar file using curl command:
 
 ```sh
-curl -X PUT -T exasol-cloud-storage-extension-2.4.1.jar \
-  http://w:<WRITE_PASSWORD>@exasol.datanode.domain.com:2580/<BUCKET>/exasol-cloud-storage-extension-2.4.1.jar
+curl -X PUT -T exasol-cloud-storage-extension-2.5.0.jar \
+  http://w:<WRITE_PASSWORD>@exasol.datanode.domain.com:2580/<BUCKET>/exasol-cloud-storage-extension-2.5.0.jar
 ```
 
 You can also check out the Exasol [BucketFS Explorer][bucketfs-explorer] as an
@@ -226,7 +226,7 @@ OPEN SCHEMA CLOUD_STORAGE_EXTENSION;
 
 CREATE OR REPLACE JAVA SET SCRIPT IMPORT_PATH(...) EMITS (...) AS
   %scriptclass com.exasol.cloudetl.scriptclasses.FilesImportQueryGenerator;
-  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.4.1.jar;
+  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.5.0.jar;
 /
 
 CREATE OR REPLACE JAVA SCALAR SCRIPT IMPORT_METADATA(...) EMITS (
@@ -236,12 +236,12 @@ CREATE OR REPLACE JAVA SCALAR SCRIPT IMPORT_METADATA(...) EMITS (
   end_index DECIMAL(36, 0)
 ) AS
   %scriptclass com.exasol.cloudetl.scriptclasses.FilesMetadataReader;
-  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.4.1.jar;
+  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.5.0.jar;
 /
 
 CREATE OR REPLACE JAVA SET SCRIPT IMPORT_FILES(...) EMITS (...) AS
   %scriptclass com.exasol.cloudetl.scriptclasses.FilesDataImporter;
-  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.4.1.jar;
+  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.5.0.jar;
 /
 ```
 
@@ -261,7 +261,7 @@ OPEN SCHEMA CLOUD_STORAGE_EXTENSION;
 
 CREATE OR REPLACE JAVA SET SCRIPT IMPORT_PATH(...) EMITS (...) AS
   %scriptclass com.exasol.cloudetl.scriptclasses.DockerFilesImportQueryGenerator;
-  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.4.1.jar;
+  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.5.0.jar;
 /
 
 CREATE OR REPLACE JAVA SCALAR SCRIPT IMPORT_METADATA(...) EMITS (
@@ -271,12 +271,12 @@ CREATE OR REPLACE JAVA SCALAR SCRIPT IMPORT_METADATA(...) EMITS (
   end_index DECIMAL(36, 0)
 ) AS
   %scriptclass com.exasol.cloudetl.scriptclasses.DockerFilesMetadataReader;
-  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.4.1.jar;
+  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.5.0.jar;
 /
 
 CREATE OR REPLACE JAVA SET SCRIPT IMPORT_FILES(...) EMITS (...) AS
   %scriptclass com.exasol.cloudetl.scriptclasses.DockerFilesDataImporter;
-  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.4.1.jar;
+  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.5.0.jar;
 /
 ```
 
@@ -292,12 +292,12 @@ OPEN SCHEMA CLOUD_STORAGE_EXTENSION;
 
 CREATE OR REPLACE JAVA SET SCRIPT EXPORT_PATH(...) EMITS (...) AS
   %scriptclass com.exasol.cloudetl.scriptclasses.TableExportQueryGenerator;
-  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.4.1.jar;
+  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.5.0.jar;
 /
 
 CREATE OR REPLACE JAVA SET SCRIPT EXPORT_TABLE(...) EMITS (ROWS_AFFECTED INT) AS
   %scriptclass com.exasol.cloudetl.scriptclasses.TableDataExporter;
-  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.4.1.jar;
+  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.5.0.jar;
 /
 ```
 
@@ -317,12 +317,12 @@ OPEN SCHEMA CLOUD_STORAGE_EXTENSION;
 
 CREATE OR REPLACE JAVA SET SCRIPT EXPORT_PATH(...) EMITS (...) AS
   %scriptclass com.exasol.cloudetl.scriptclasses.DockerTableExportQueryGenerator;
-  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.4.1.jar;
+  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.5.0.jar;
 /
 
 CREATE OR REPLACE JAVA SET SCRIPT EXPORT_TABLE(...) EMITS (ROWS_AFFECTED INT) AS
   %scriptclass com.exasol.cloudetl.scriptclasses.DockerTableDataExporter;
-  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.4.1.jar;
+  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.5.0.jar;
 /
 ```
 
@@ -453,13 +453,13 @@ CREATE OR REPLACE JAVA SCALAR SCRIPT IMPORT_METADATA(...) EMITS (
 ) AS
   %jvmoption -DHTTPS_PROXY=http://username:password@10.10.1.10:1180
   %scriptclass com.exasol.cloudetl.scriptclasses.FilesMetadataReader;
-  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.4.1.jar;
+  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.5.0.jar;
 /
 
 CREATE OR REPLACE JAVA SET SCRIPT IMPORT_FILES(...) EMITS (...) AS
   %jvmoption -DHTTPS_PROXY=http://username:password@10.10.1.10:1180
   %scriptclass com.exasol.cloudetl.scriptclasses.FilesDataImporter;
-  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.4.1.jar;
+  %jar /buckets/bfsdefault/<BUCKET>/exasol-cloud-storage-extension-2.5.0.jar;
 /
 ```
 

pk_generated_parent.pom

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.exasol</groupId>
     <artifactId>cloud-storage-extension-generated-parent</artifactId>
-    <version>2.4.1</version>
+    <version>2.5.0</version>
     <packaging>pom</packaging>
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.exasol</groupId>
     <artifactId>cloud-storage-extension</artifactId>
-    <version>2.4.1</version>
+    <version>2.5.0</version>
     <name>Cloud Storage Extension</name>
     <description>Exasol Cloud Storage Import And Export Extension</description>
     <url>https://github.com/exasol/cloud-storage-extension/</url>
@@ -17,7 +17,7 @@
     <parent>
         <artifactId>cloud-storage-extension-generated-parent</artifactId>
         <groupId>com.exasol</groupId>
-        <version>2.4.1</version>
+        <version>2.5.0</version>
         <relativePath>pk_generated_parent.pom</relativePath>
     </parent>
     <repositories>
@@ -55,12 +55,12 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.13.3</version>
+            <version>2.13.4</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.module</groupId>
             <artifactId>jackson-module-scala_${scala.compat.version}</artifactId>
-            <version>2.13.3</version>
+            <version>2.13.4</version>
         </dependency>
         <dependency>
             <groupId>io.grpc</groupId>
@@ -264,8 +264,9 @@
         <dependency>
             <groupId>io.delta</groupId>
             <artifactId>delta-core_${scala.compat.version}</artifactId>
-            <version>1.2.1</version>
+            <version>2.1.0</version>
         </dependency>
+        <!-- Do not remove until hadoop version 3.4.0, see https://github.com/exasol/cloud-storage-extension/issues/195 -->
         <dependency>
             <groupId>org.codehaus.jackson</groupId>
             <artifactId>jackson-mapper-asl</artifactId>
@@ -306,12 +307,12 @@
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>jul-to-slf4j</artifactId>
-            <version>2.0.2</version>
+            <version>2.0.3</version>
         </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-log4j12</artifactId>
-            <version>2.0.2</version>
+            <version>2.0.3</version>
             <exclusions>
                 <exclusion>
                     <groupId>log4j</groupId>

src/main/scala/com/exasol/cloudetl/bucket/S3Bucket.scala

@@ -10,6 +10,7 @@ import org.apache.hadoop.fs.Path
 final case class S3Bucket(path: String, params: StorageProperties) extends Bucket with SecureBucket {
 
   private[this] val S3_ENDPOINT: String = "S3_ENDPOINT"
+  private[this] val S3_ENDPOINT_REGION: String = "S3_ENDPOINT_REGION"
   private[this] val S3_ACCESS_KEY: String = "S3_ACCESS_KEY"
   private[this] val S3_SECRET_KEY: String = "S3_SECRET_KEY"
   private[this] val S3_SESSION_TOKEN: String = "S3_SESSION_TOKEN"
@@ -63,6 +64,9 @@ final case class S3Bucket(path: String, params: StorageProperties) extends Bucke
     conf.set("fs.file.impl", classOf[org.apache.hadoop.fs.LocalFileSystem].getName)
     conf.set("fs.s3a.impl", classOf[org.apache.hadoop.fs.s3a.S3AFileSystem].getName)
     conf.set("fs.s3a.endpoint", properties.getString(S3_ENDPOINT))
+    if (properties.containsKey(S3_ENDPOINT_REGION)) {
+      conf.set("fs.s3a.endpoint.region", properties.getString(S3_ENDPOINT_REGION))
+    }
     if (properties.containsKey(S3_CHANGE_DETECTION_MODE)) {
       conf.set("fs.s3a.change.detection.mode", properties.getString(S3_CHANGE_DETECTION_MODE))
     }

src/test/scala/com/exasol/cloudetl/bucket/S3BucketTest.scala

@@ -91,6 +91,12 @@ class S3BucketTest extends AbstractBucketTest {
     assertConfigurationProperties(bucketWithDefaultConnectionString(properties), extraConfigs)
   }
 
+  test("apply returns S3Bucket with endpoint region") {
+    val properties = defaultProperties ++ Map("S3_ENDPOINT_REGION" -> "eu-central-1")
+    val extraConfigs = configMappings ++ Map("fs.s3a.endpoint.region" -> "eu-central-1")
+    assertConfigurationProperties(bucketWithDefaultConnectionString(properties), extraConfigs)
+  }
+
   test("apply returns S3Bucket with ssl enabled by default") {
     val extraConfigs = configMappings ++ Map("fs.s3a.connection.ssl.enabled" -> "true")
     assertConfigurationProperties(bucketWithDefaultConnectionString(defaultProperties), extraConfigs)