From a2e5fd05c6417e1277e39e67e85aadb63f4b1fd8 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Mon, 2 Sep 2024 11:36:57 +0900 Subject: [PATCH 1/2] Core: Expose properties in InMemoryCatalog --- .../java/org/apache/iceberg/inmemory/InMemoryCatalog.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java b/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java index a880f94f4385..7a6137041b3f 100644 --- a/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java +++ b/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java @@ -68,6 +68,7 @@ public class InMemoryCatalog extends BaseMetastoreViewCatalog private final ConcurrentMap views; private FileIO io; private String catalogName; + private Map catalogProperties; private String warehouseLocation; private CloseableGroup closeableGroup; @@ -85,6 +86,7 @@ public String name() { @Override public void initialize(String name, Map properties) { this.catalogName = name != null ? name : InMemoryCatalog.class.getSimpleName(); + this.catalogProperties = ImmutableMap.copyOf(properties); String warehouse = properties.getOrDefault(CatalogProperties.WAREHOUSE_LOCATION, ""); this.warehouseLocation = warehouse.replaceAll("/*$", ""); @@ -94,6 +96,11 @@ public void initialize(String name, Map properties) { closeableGroup.setSuppressCloseFailure(true); } + @Override + protected Map properties() { + return catalogProperties == null ? ImmutableMap.of() : catalogProperties; + } + @Override protected TableOperations newTableOps(TableIdentifier tableIdentifier) { return new InMemoryTableOperations(io, tableIdentifier); From 08cb7058f2e89b6366bc44523b55a21643909da6 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Mon, 2 Sep 2024 11:38:34 +0900 Subject: [PATCH 2/2] Core: Add support for view-default property in catalog --- .../org/apache/iceberg/CatalogProperties.java | 1 + .../iceberg/rest/RESTSessionCatalog.java | 15 +++++++++ .../view/BaseMetastoreViewCatalog.java | 21 +++++++++++++ .../inmemory/TestInMemoryViewCatalog.java | 8 ++++- .../iceberg/jdbc/TestJdbcViewCatalog.java | 2 ++ .../iceberg/rest/TestRESTViewCatalog.java | 6 +++- ...RESTViewCatalogWithAssumedViewSupport.java | 6 +++- .../apache/iceberg/view/ViewCatalogTests.java | 31 +++++++++++++++++++ docs/docs/spark-configuration.md | 1 + .../iceberg/hive/TestHiveViewCatalog.java | 6 +++- .../iceberg/nessie/TestNessieViewCatalog.java | 6 +++- .../org/apache/iceberg/rest/RCKUtils.java | 4 +++ 12 files changed, 102 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/CatalogProperties.java b/core/src/main/java/org/apache/iceberg/CatalogProperties.java index 339c59b45d1b..25d331c8323b 100644 --- a/core/src/main/java/org/apache/iceberg/CatalogProperties.java +++ b/core/src/main/java/org/apache/iceberg/CatalogProperties.java @@ -29,6 +29,7 @@ private CatalogProperties() {} public static final String WAREHOUSE_LOCATION = "warehouse"; public static final String TABLE_DEFAULT_PREFIX = "table-default."; public static final String TABLE_OVERRIDE_PREFIX = "table-override."; + public static final String VIEW_DEFAULT_PREFIX = "view-default."; public static final String METRICS_REPORTER_IMPL = "metrics-reporter-impl"; /** diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java index cc42604f700d..9e2c828e95c5 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java @@ -1275,6 +1275,21 @@ private RESTViewBuilder(SessionContext context, TableIdentifier identifier) { checkViewIdentifierIsValid(identifier); this.identifier = identifier; this.context = context; + this.properties.putAll(viewDefaultProperties()); + } + + /** + * Get default view properties set at Catalog level through catalog properties. + * + * @return default view properties specified in catalog properties + */ + private Map viewDefaultProperties() { + Map viewDefaultProperties = + PropertyUtil.propertiesWithPrefix(properties(), CatalogProperties.VIEW_DEFAULT_PREFIX); + LOG.info( + "View properties set at catalog level through catalog properties: {}", + viewDefaultProperties); + return viewDefaultProperties; } @Override diff --git a/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java b/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java index 6e2d6ff5e864..2f747c54556e 100644 --- a/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java +++ b/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import org.apache.iceberg.BaseMetastoreCatalog; +import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.EnvironmentContext; import org.apache.iceberg.Schema; import org.apache.iceberg.Transaction; @@ -33,8 +34,13 @@ import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.util.PropertyUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class BaseMetastoreViewCatalog extends BaseMetastoreCatalog implements ViewCatalog { + private static final Logger LOG = LoggerFactory.getLogger(BaseMetastoreViewCatalog.class); + protected abstract ViewOperations newViewOps(TableIdentifier identifier); @Override @@ -79,6 +85,21 @@ protected BaseViewBuilder(TableIdentifier identifier) { Preconditions.checkArgument( isValidIdentifier(identifier), "Invalid view identifier: %s", identifier); this.identifier = identifier; + this.properties.putAll(viewDefaultProperties()); + } + + /** + * Get default view properties set at Catalog level through catalog properties. + * + * @return default view properties specified in catalog properties + */ + private Map viewDefaultProperties() { + Map viewDefaultProperties = + PropertyUtil.propertiesWithPrefix(properties(), CatalogProperties.VIEW_DEFAULT_PREFIX); + LOG.info( + "View properties set at catalog level through catalog properties: {}", + viewDefaultProperties); + return viewDefaultProperties; } @Override diff --git a/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryViewCatalog.java b/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryViewCatalog.java index 76731f58a6be..65e4e26315c6 100644 --- a/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryViewCatalog.java +++ b/core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryViewCatalog.java @@ -18,6 +18,7 @@ */ package org.apache.iceberg.inmemory; +import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.view.ViewCatalogTests; @@ -29,7 +30,12 @@ public class TestInMemoryViewCatalog extends ViewCatalogTests { @BeforeEach public void before() { this.catalog = new InMemoryCatalog(); - this.catalog.initialize("in-memory-catalog", ImmutableMap.of()); + this.catalog.initialize( + "in-memory-catalog", + ImmutableMap.builder() + .put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1") + .put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", "catalog-default-key2") + .build()); } @Override diff --git a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcViewCatalog.java b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcViewCatalog.java index a66532d90f63..9b8ca3a90609 100644 --- a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcViewCatalog.java +++ b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcViewCatalog.java @@ -57,6 +57,8 @@ public void before() { properties.put(JdbcCatalog.PROPERTY_PREFIX + "password", "password"); properties.put(CatalogProperties.WAREHOUSE_LOCATION, tableDir.toAbsolutePath().toString()); properties.put(JdbcUtil.SCHEMA_VERSION_PROPERTY, JdbcUtil.SchemaVersion.V1.name()); + properties.put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1"); + properties.put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", "catalog-default-key2"); catalog = new JdbcCatalog(); catalog.setConf(new Configuration()); diff --git a/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalog.java b/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalog.java index 85ccdc8f5ddd..c649619867ab 100644 --- a/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalog.java +++ b/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalog.java @@ -71,7 +71,11 @@ public void createCatalog() throws Exception { this.backendCatalog = new InMemoryCatalog(); this.backendCatalog.initialize( "in-memory", - ImmutableMap.of(CatalogProperties.WAREHOUSE_LOCATION, warehouse.getAbsolutePath())); + ImmutableMap.builder() + .put(CatalogProperties.WAREHOUSE_LOCATION, warehouse.getAbsolutePath()) + .put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1") + .put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", "catalog-default-key2") + .build()); RESTCatalogAdapter adaptor = new RESTCatalogAdapter(backendCatalog) { diff --git a/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalogWithAssumedViewSupport.java b/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalogWithAssumedViewSupport.java index 3d7d64ddb794..ee9d8cdfd24c 100644 --- a/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalogWithAssumedViewSupport.java +++ b/core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalogWithAssumedViewSupport.java @@ -90,6 +90,10 @@ public T handleRequest( "catalog:12345", // assume that the server supports view endpoints RESTSessionCatalog.VIEW_ENDPOINTS_SUPPORTED, - "true")); + "true", + CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", + "catalog-default-key1", + CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", + "catalog-default-key2")); } } diff --git a/core/src/test/java/org/apache/iceberg/view/ViewCatalogTests.java b/core/src/test/java/org/apache/iceberg/view/ViewCatalogTests.java index b3765bb1eae7..d365defb29c8 100644 --- a/core/src/test/java/org/apache/iceberg/view/ViewCatalogTests.java +++ b/core/src/test/java/org/apache/iceberg/view/ViewCatalogTests.java @@ -127,6 +127,37 @@ public void basicCreateView() { assertThat(catalog().viewExists(identifier)).as("View should not exist").isFalse(); } + @Test + public void defaultViewProperties() { + TableIdentifier identifier = TableIdentifier.of("ns", "view"); + + if (requiresNamespaceCreate()) { + catalog().createNamespace(identifier.namespace()); + } + + assertThat(catalog().viewExists(identifier)).as("View should not exist").isFalse(); + + View view = + catalog() + .buildView(identifier) + .withSchema(SCHEMA) + .withDefaultNamespace(identifier.namespace()) + .withDefaultCatalog(catalog().name()) + .withQuery("spark", "select * from ns.tbl") + .withProperty("key2", "catalog-overridden-key2") + .withProperty("prop1", "val1") + .create(); + + assertThat(view).isNotNull(); + assertThat(view.properties()) + .containsEntry("key1", "catalog-default-key1") + .containsEntry("key2", "catalog-overridden-key2") + .containsEntry("prop1", "val1"); + + assertThat(catalog().dropView(identifier)).isTrue(); + assertThat(catalog().viewExists(identifier)).as("View should not exist").isFalse(); + } + @Test public void completeCreateView() { TableIdentifier identifier = TableIdentifier.of("ns", "view"); diff --git a/docs/docs/spark-configuration.md b/docs/docs/spark-configuration.md index 5b281b19891a..3b0f8b54a304 100644 --- a/docs/docs/spark-configuration.md +++ b/docs/docs/spark-configuration.md @@ -77,6 +77,7 @@ Both catalogs are configured using properties nested under the catalog name. Com | spark.sql.catalog._catalog-name_.cache.expiration-interval-ms | `30000` (30 seconds) | Duration after which cached catalog entries are expired; Only effective if `cache-enabled` is `true`. `-1` disables cache expiration and `0` disables caching entirely, irrespective of `cache-enabled`. Default is `30000` (30 seconds) | | spark.sql.catalog._catalog-name_.table-default._propertyKey_ | | Default Iceberg table property value for property key _propertyKey_, which will be set on tables created by this catalog if not overridden | | spark.sql.catalog._catalog-name_.table-override._propertyKey_ | | Enforced Iceberg table property value for property key _propertyKey_, which cannot be overridden by user | +| spark.sql.catalog._catalog-name_.view-default._propertyKey_ | | Default Iceberg view property value for property key _propertyKey_, which will be set on views created by this catalog if not overridden | | spark.sql.catalog._catalog-name_.use-nullable-query-schema | `true` or `false` | Whether to preserve fields' nullability when creating the table using CTAS and RTAS. If set to `true`, all fields will be marked as nullable. If set to `false`, fields' nullability will be preserved. The default value is `true`. Available in Spark 3.5 and above. | Additional properties can be found in common [catalog configuration](configuration.md#catalog-properties). diff --git a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveViewCatalog.java b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveViewCatalog.java index 3c195e256520..32889d9ddf2a 100644 --- a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveViewCatalog.java +++ b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveViewCatalog.java @@ -66,7 +66,11 @@ public void before() throws TException { CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE, ImmutableMap.of( CatalogProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS, - String.valueOf(TimeUnit.SECONDS.toMillis(10))), + String.valueOf(TimeUnit.SECONDS.toMillis(10)), + CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", + "catalog-default-key1", + CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", + "catalog-default-key2"), HIVE_METASTORE_EXTENSION.hiveConf()); } diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieViewCatalog.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieViewCatalog.java index cb2c61e9dcb8..94a1c1ec34ec 100644 --- a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieViewCatalog.java +++ b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieViewCatalog.java @@ -128,7 +128,11 @@ private NessieCatalog initNessieCatalog(String ref) { CatalogProperties.WAREHOUSE_LOCATION, temp.toUri().toString(), "client-api-version", - apiVersion == NessieApiVersion.V2 ? "2" : "1"); + apiVersion == NessieApiVersion.V2 ? "2" : "1", + CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", + "catalog-default-key1", + CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", + "catalog-default-key2"); newCatalog.initialize("nessie", options); return newCatalog; } diff --git a/open-api/src/testFixtures/java/org/apache/iceberg/rest/RCKUtils.java b/open-api/src/testFixtures/java/org/apache/iceberg/rest/RCKUtils.java index 0f1571d362e3..9bf386cda087 100644 --- a/open-api/src/testFixtures/java/org/apache/iceberg/rest/RCKUtils.java +++ b/open-api/src/testFixtures/java/org/apache/iceberg/rest/RCKUtils.java @@ -86,6 +86,10 @@ static RESTCatalog initCatalogClient() { CatalogProperties.URI, String.format("http://localhost:%s/", RESTCatalogServer.REST_PORT_DEFAULT)); catalogProperties.putIfAbsent(CatalogProperties.WAREHOUSE_LOCATION, "rck_warehouse"); + catalogProperties.putIfAbsent( + CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1"); + catalogProperties.putIfAbsent( + CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", "catalog-default-key2"); RESTCatalog catalog = new RESTCatalog(); catalog.setConf(new Configuration());