Core: Add support for view-default property in catalog

apache · Sep 2, 2024 · c4aa6ae · c4aa6ae
1 parent 69fab2b
commit c4aa6ae
Show file tree

Hide file tree

Showing 10 changed files with 28 additions and 4 deletions.
diff --git a/core/src/main/java/org/apache/iceberg/CatalogProperties.java b/core/src/main/java/org/apache/iceberg/CatalogProperties.java
@@ -29,6 +29,7 @@ private CatalogProperties() {}
   public static final String WAREHOUSE_LOCATION = "warehouse";
   public static final String TABLE_DEFAULT_PREFIX = "table-default.";
   public static final String TABLE_OVERRIDE_PREFIX = "table-override.";
+  public static final String VIEW_DEFAULT_PREFIX = "view-default.";
   public static final String METRICS_REPORTER_IMPL = "metrics-reporter-impl";
 
   /**

diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java
@@ -1200,6 +1200,8 @@ private RESTViewBuilder(SessionContext context, TableIdentifier identifier) {
       checkViewIdentifierIsValid(identifier);
       this.identifier = identifier;
       this.context = context;
+      properties.putAll(
+          PropertyUtil.propertiesWithPrefix(properties(), CatalogProperties.VIEW_DEFAULT_PREFIX));
     }
 
     @Override

diff --git a/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java b/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java
@@ -21,6 +21,7 @@
 import java.util.List;
 import java.util.Map;
 import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
 import org.apache.iceberg.EnvironmentContext;
 import org.apache.iceberg.Schema;
 import org.apache.iceberg.Transaction;
@@ -33,6 +34,7 @@
 import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
 import org.apache.iceberg.relocated.com.google.common.collect.Lists;
 import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.PropertyUtil;
 
 public abstract class BaseMetastoreViewCatalog extends BaseMetastoreCatalog implements ViewCatalog {
   protected abstract ViewOperations newViewOps(TableIdentifier identifier);
@@ -79,6 +81,8 @@ protected BaseViewBuilder(TableIdentifier identifier) {
       Preconditions.checkArgument(
           isValidIdentifier(identifier), "Invalid view identifier: %s", identifier);
       this.identifier = identifier;
+      this.properties.putAll(
+          PropertyUtil.propertiesWithPrefix(properties(), CatalogProperties.VIEW_DEFAULT_PREFIX));
     }
 
     @Override

diff --git a/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryViewCatalog.java b/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryViewCatalog.java
@@ -18,6 +18,7 @@
  */
 package org.apache.iceberg.inmemory;
 
+import org.apache.iceberg.CatalogProperties;
 import org.apache.iceberg.catalog.Catalog;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
 import org.apache.iceberg.view.ViewCatalogTests;
@@ -29,7 +30,11 @@ public class TestInMemoryViewCatalog extends ViewCatalogTests<InMemoryCatalog> {
   @BeforeEach
   public void before() {
     this.catalog = new InMemoryCatalog();
-    this.catalog.initialize("in-memory-catalog", ImmutableMap.of());
+    this.catalog.initialize(
+        "in-memory-catalog",
+        ImmutableMap.<String, String>builder()
+            .put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1")
+            .build());
   }
 
   @Override

diff --git a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcViewCatalog.java b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcViewCatalog.java
@@ -57,6 +57,7 @@ public void before() {
     properties.put(JdbcCatalog.PROPERTY_PREFIX + "password", "password");
     properties.put(CatalogProperties.WAREHOUSE_LOCATION, tableDir.toAbsolutePath().toString());
     properties.put(JdbcUtil.SCHEMA_VERSION_PROPERTY, JdbcUtil.SchemaVersion.V1.name());
+    properties.put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1");
 
     catalog = new JdbcCatalog();
     catalog.setConf(new Configuration());

diff --git a/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalog.java b/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalog.java
@@ -70,7 +70,10 @@ public void createCatalog() throws Exception {
     this.backendCatalog = new InMemoryCatalog();
     this.backendCatalog.initialize(
         "in-memory",
-        ImmutableMap.of(CatalogProperties.WAREHOUSE_LOCATION, warehouse.getAbsolutePath()));
+        ImmutableMap.<String, String>builder()
+            .put(CatalogProperties.WAREHOUSE_LOCATION, warehouse.getAbsolutePath())
+            .put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1")
+            .build());
 
     RESTCatalogAdapter adaptor =
         new RESTCatalogAdapter(backendCatalog) {

diff --git a/core/src/test/java/org/apache/iceberg/view/ViewCatalogTests.java b/core/src/test/java/org/apache/iceberg/view/ViewCatalogTests.java
@@ -21,6 +21,7 @@
 import static org.apache.iceberg.types.Types.NestedField.required;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.entry;
 
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -107,6 +108,7 @@ public void basicCreateView() {
     assertThat(view.currentVersion().operation()).isEqualTo("create");
     assertThat(view.schemas()).hasSize(1).containsKey(0);
     assertThat(view.versions()).hasSize(1).containsExactly(view.currentVersion());
+    assertThat(view.properties()).contains(entry("key1", "catalog-default-key1"));
 
     assertThat(view.currentVersion())
         .isEqualTo(
@@ -871,6 +873,7 @@ public void createOrReplaceView(boolean useCreateOrReplace) {
     assertThat(replacedView.name()).isEqualTo(ViewUtil.fullViewName(catalog().name(), identifier));
     assertThat(((BaseView) replacedView).operations().current().metadataFileLocation()).isNotNull();
     assertThat(replacedView.properties())
+        .containsEntry("key1", "catalog-default-key1")
         .containsEntry("prop1", "val1")
         .containsEntry("prop2", "val2")
         .containsEntry("replacedProp1", "val1")

diff --git a/docs/docs/spark-configuration.md b/docs/docs/spark-configuration.md
@@ -77,6 +77,8 @@ Both catalogs are configured using properties nested under the catalog name. Com
 | spark.sql.catalog._catalog-name_.cache.expiration-interval-ms | `30000` (30 seconds) | Duration after which cached catalog entries are expired; Only effective if `cache-enabled` is `true`. `-1` disables cache expiration and `0` disables caching entirely, irrespective of `cache-enabled`. Default is `30000` (30 seconds) |
 | spark.sql.catalog._catalog-name_.table-default._propertyKey_  |                               | Default Iceberg table property value for property key _propertyKey_, which will be set on tables created by this catalog if not overridden                                                                                               |
 | spark.sql.catalog._catalog-name_.table-override._propertyKey_ |                               | Enforced Iceberg table property value for property key _propertyKey_, which cannot be overridden by user                                                                                                                                 |
+| spark.sql.catalog._catalog-name_.view-default._propertyKey_  |                               | Default Iceberg view property value for property key _propertyKey_, which will be set on views created by this catalog if not overridden                                                                                               |
+| spark.sql.catalog._catalog-name_.view-override._propertyKey_ |                               | Enforced Iceberg view property value for property key _propertyKey_, which cannot be overridden by user                                                                                                                                 |
 | spark.sql.catalog._catalog-name_.use-nullable-query-schema | `true` or `false` | Whether to preserve fields' nullability when creating the table using CTAS and RTAS. If set to `true`, all fields will be marked as nullable. If set to `false`, fields' nullability will be preserved. The default value is `true`. Available in Spark 3.5 and above.   |
 
 Additional properties can be found in common [catalog configuration](configuration.md#catalog-properties).

diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieViewCatalog.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieViewCatalog.java
@@ -128,7 +128,9 @@ private NessieCatalog initNessieCatalog(String ref) {
             CatalogProperties.WAREHOUSE_LOCATION,
             temp.toUri().toString(),
             "client-api-version",
-            apiVersion == NessieApiVersion.V2 ? "2" : "1");
+            apiVersion == NessieApiVersion.V2 ? "2" : "1",
+            CatalogProperties.VIEW_DEFAULT_PREFIX + "key1",
+            "catalog-default-key1");
     newCatalog.initialize("nessie", options);
     return newCatalog;
   }

diff --git a/open-api/src/testFixtures/java/org/apache/iceberg/rest/RCKUtils.java b/open-api/src/testFixtures/java/org/apache/iceberg/rest/RCKUtils.java
@@ -85,7 +85,8 @@ static RESTCatalog initCatalogClient() {
     catalogProperties.putIfAbsent(
         CatalogProperties.URI,
         String.format("http://localhost:%s/", RESTCatalogServer.REST_PORT_DEFAULT));
-    catalogProperties.putIfAbsent(CatalogProperties.WAREHOUSE_LOCATION, "rck_warehouse");
+    catalogProperties.putIfAbsent(
+        CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1");
 
     RESTCatalog catalog = new RESTCatalog();
     catalog.setConf(new Configuration());