diff --git a/README.md b/README.md index eadae7943..e54f08a85 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,8 @@ Example: - my_writable_db2 - user_db_.* - ... + configuration-properties: + hive.metastore.kerberos.principal: hive/_HOST@HADOOP.COM federated-meta-stores: # List of read only metastores to federate - remote-meta-store-uris: thrift://10.0.0.1:9083 name: secondary @@ -147,6 +149,8 @@ Example: - database: prod_db2 mapped-tables: - tbl2 + configuration-properties: + hive.metastore.kerberos.principal: hive/clustername@HADOOP.COM - ... The table below describes all the available configuration values for Waggle Dance federations: @@ -166,6 +170,7 @@ The table below describes all the available configuration values for Waggle Danc | `primary-meta-store.mapped-tables` | No | List of mappings from databases to tables to federate from the primary metastore, similar to `mapped-databases`. By default, all tables are available. See `mapped-tables` configuration below. | | `primary-meta-stores.hive-metastore-filter-hook` | No | Name of the class which implements the `MetaStoreFilterHook` interface from Hive. This allows a metastore filter hook to be applied to the corresponding Hive metastore calls. Can be configured with the `configuration-properties` specified in the `waggle-dance-server.yml` configuration. They will be added in the HiveConf object that is given to the constructor of the `MetaStoreFilterHook` implementation you provide. | | `primary-meta-stores.database-name-mapping` | No | BiDirectional Map of database names and mapped name, where key=`` and value=``. See the [Database Name Mapping](#database-name-mapping) section.| +| `primary-meta-stores.configuration-properties` | No | Map of the primary metastore personalized properties that will be added to the HiveConf used when creating the Thrift clients (they will be effect only on this client),the priority is higher than the properites of the same name in waggle-dance-server.yml. | | `federated-meta-stores` | No | Possible empty list of read only federated metastores. | | `federated-meta-stores[n].remote-meta-store-uris` | Yes | Thrift URIs of the federated read-only metastore. | | `federated-meta-stores[n].name` | Yes | Name that uniquely identifies this metastore. Used internally. Cannot be empty. | @@ -178,6 +183,7 @@ The table below describes all the available configuration values for Waggle Danc | `federated-meta-stores[n].hive-metastore-filter-hook` | No | Name of the class which implements the `MetaStoreFilterHook` interface from Hive. This allows a metastore filter hook to be applied to the corresponding Hive metastore calls. Can be configured with the `configuration-properties` specified in the `waggle-dance-server.yml` configuration. They will be added in the HiveConf object that is given to the constructor of the `MetaStoreFilterHook` implementation you provide. | | `federated-meta-stores[n].database-name-mapping` | No | BiDirectional Map of database names and mapped names where key=`` and value=``. See the [Database Name Mapping](#database-name-mapping) section.| | `federated-meta-stores[n].writable-database-white-list` | No | White-list of databases used to verify write access used in conjunction with `federated-meta-stores[n].access-control-type`. The list of databases should be listed without a `federated-meta-stores[n].database-prefix`. This property supports both full database names and (case-insensitive) [Java RegEx patterns](https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html).| +| `federated-meta-stores[n].configuration-properties` | No | Map of the federate metastore personalized properties that will be added to the HiveConf used when creating the Thrift clients (they will be effect only on this client),the priority is higher than the properites of the same name in waggle-dance-server.yml. | #### Metastore tunnel The table below describes the metastore tunnel configuration values: diff --git a/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java b/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java index 4e8bc71e0..3eab7217d 100644 --- a/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java +++ b/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java @@ -20,6 +20,7 @@ import java.beans.Transient; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -51,7 +52,7 @@ public abstract class AbstractMetaStore { private List writableDatabaseWhitelist; private List mappedDatabases; private @Valid List mappedTables; - private Map databaseNameMapping = Collections.emptyMap(); + private Map databaseNameMapping = new HashMap<>(); private @NotBlank String name; private @NotBlank String remoteMetaStoreUris; private @Valid MetastoreTunnel metastoreTunnel; @@ -60,6 +61,8 @@ public abstract class AbstractMetaStore { private long latency = 0; private transient @JsonIgnore HashBiMap databaseNameBiMapping = HashBiMap.create(); private boolean impersonationEnabled; + private Map configurationProperties = new HashMap<>(); + public AbstractMetaStore(String name, String remoteMetaStoreUris, AccessControlType accessControlType) { this.name = name; this.remoteMetaStoreUris = remoteMetaStoreUris; @@ -201,6 +204,15 @@ public HashBiMap getDatabaseNameBiMapping() { return databaseNameBiMapping; } + public Map getConfigurationProperties() { + return configurationProperties; + } + + public void setConfigurationProperties( + Map configurationProperties) { + this.configurationProperties = configurationProperties; + } + @Transient public MetaStoreStatus getStatus() { return status; diff --git a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java index ec4b9cf98..fc6228e75 100644 --- a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java +++ b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java @@ -72,7 +72,7 @@ public void nullDatabasePrefix() { @Test public void toJson() throws Exception { - String expected = "{\"accessControlType\":\"READ_ONLY\",\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"name_\",\"federationType\":\"FEDERATED\",\"hiveMetastoreFilterHook\":null,\"impersonationEnabled\":false,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; + String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":{},\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"name_\",\"federationType\":\"FEDERATED\",\"hiveMetastoreFilterHook\":null,\"impersonationEnabled\":false,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; ObjectMapper mapper = new ObjectMapper(); // Sorting to get deterministic test behaviour mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY); diff --git a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java index feb719680..5da2d6316 100644 --- a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java +++ b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java @@ -89,7 +89,7 @@ public void nonEmptyDatabasePrefix() { @Test public void toJson() throws Exception { - String expected = "{\"accessControlType\":\"READ_ONLY\",\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"\",\"federationType\":\"PRIMARY\",\"hiveMetastoreFilterHook\":null,\"impersonationEnabled\":false,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; + String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":{},\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"\",\"federationType\":\"PRIMARY\",\"hiveMetastoreFilterHook\":null,\"impersonationEnabled\":false,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; ObjectMapper mapper = new ObjectMapper(); // Sorting to get deterministic test behaviour mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY); diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java index 81c4f7aba..7d7bdf85c 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java @@ -48,6 +48,10 @@ public CloseableThriftHiveMetastoreIface newInstance(AbstractMetaStore metaStore if (waggleDanceConfiguration.getConfigurationProperties() != null) { properties.putAll(waggleDanceConfiguration.getConfigurationProperties()); } + if (metaStore.getConfigurationProperties() != null) { + properties.putAll(metaStore.getConfigurationProperties()); + } + return newHiveInstance(metaStore, properties); } diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/SaslThriftMetastoreClientManager.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/SaslThriftMetastoreClientManager.java index 57838c20e..8a9bef8a9 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/SaslThriftMetastoreClientManager.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/SaslThriftMetastoreClientManager.java @@ -25,8 +25,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import javax.security.sasl.SaslException; - import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.MetaException; diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactoryTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactoryTest.java index d011d50cd..f60dea237 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactoryTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactoryTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2021 Expedia, Inc. + * Copyright (C) 2016-2024 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,6 +24,7 @@ import static com.hotels.bdp.waggledance.api.model.AbstractMetaStore.newFederatedInstance; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -38,6 +39,7 @@ import org.mockito.junit.MockitoJUnitRunner; import com.hotels.bdp.waggledance.api.model.AbstractMetaStore; +import com.hotels.bdp.waggledance.api.model.FederatedMetaStore; import com.hotels.bdp.waggledance.client.tunnelling.TunnelingMetaStoreClientFactory; import com.hotels.bdp.waggledance.conf.WaggleDanceConfiguration; import com.hotels.hcommon.hive.metastore.client.tunnelling.MetastoreTunnel; @@ -68,8 +70,9 @@ public void setUp() { @Test public void defaultFactory() { ArgumentCaptor hiveConfCaptor = ArgumentCaptor.forClass(HiveConf.class); - - factory.newInstance(newFederatedInstance("fed1", THRIFT_URI)); + FederatedMetaStore fed1 = newFederatedInstance("fed1", THRIFT_URI); + fed1.setConfigurationProperties(Collections.singletonMap(ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname, "hive/_HOST@HADOOP.COM")); + factory.newInstance(fed1); verify(defaultMetaStoreClientFactory).newInstance(hiveConfCaptor.capture(), eq( "waggledance-fed1"), eq(3), eq(2000)); verifyNoInteractions(tunnelingMetaStoreClientFactory); @@ -80,6 +83,7 @@ public void defaultFactory() { assertThat(hiveConf.getTimeVar(ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS), is(5L)); assertThat(hiveConf.getBoolVar(ConfVars.METASTORE_USE_THRIFT_FRAMED_TRANSPORT), is(true)); assertThat(hiveConf.getBoolVar(ConfVars.METASTORE_USE_THRIFT_COMPACT_PROTOCOL), is(false)); + assertThat(hiveConf.getVar(ConfVars.METASTORE_KERBEROS_PRINCIPAL), is("hive/_HOST@HADOOP.COM")); } @Test diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java index 9a1e7e8a0..62d950266 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java @@ -23,6 +23,7 @@ import java.io.File; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.util.Collections; import java.util.List; import javax.validation.ConstraintViolationException; @@ -211,12 +212,13 @@ public void saveFederationWriteFederations() throws Exception { MappedTables mappedTables2 = new MappedTables("db2", Lists.newArrayList("tbl2")); newFederatedInstance.setMappedTables(Lists.newArrayList(mappedTables1, mappedTables2)); newFederatedInstance.setHiveMetastoreFilterHook("filter.hook.class"); + newFederatedInstance.setConfigurationProperties(Collections.singletonMap("hive.metastore.kerberos.principal", "hive/_HOST@REALM")); storage.insert(newFederatedInstance); storage.saveFederation(); List lines = Files.readAllLines(file.toPath(), StandardCharsets.UTF_8); - assertThat(lines.size(), is(26)); + assertThat(lines.size(), is(27)); int i = 0; - while (i < 26) { + while (i < lines.size()) { assertThat(lines.get(i++), is("primary-meta-store:")); assertThat(lines.get(i++), is(" access-control-type: READ_ONLY")); assertThat(lines.get(i++), is(" database-prefix: ''")); @@ -226,7 +228,8 @@ public void saveFederationWriteFederations() throws Exception { assertThat(lines.get(i++), is(" remote-meta-store-uris: thrift://localhost:19083")); assertThat(lines.get(i++), is("federated-meta-stores:")); assertThat(lines.get(i++), is("- access-control-type: READ_ONLY")); - assertThat(lines.get(i++), is(" database-name-mapping: {}")); + assertThat(lines.get(i++), is(" configuration-properties:")); + assertThat(lines.get(i++), is(" hive.metastore.kerberos.principal: hive/_HOST@REALM")); assertThat(lines.get(i++), is(" database-prefix: hcom_2_")); assertThat(lines.get(i++), is(" hive-metastore-filter-hook: filter.hook.class")); assertThat(lines.get(i++), is(" impersonation-enabled: false")); @@ -297,15 +300,18 @@ public void savePrimaryWriteFederations() throws Exception { MappedTables mappedTables1 = new MappedTables("db1", Lists.newArrayList("tbl1")); MappedTables mappedTables2 = new MappedTables("db2", Lists.newArrayList("tbl2")); primaryMetaStore.setMappedTables(Lists.newArrayList(mappedTables1, mappedTables2)); + primaryMetaStore.setConfigurationProperties(Collections.singletonMap("hive.metastore.kerberos.principal", "hive/_HOST@REALM")); storage.insert(primaryMetaStore); storage.insert(newFederatedInstance("hcom_2", "thrift://localhost:29083")); storage.saveFederation(); List lines = Files.readAllLines(file.toPath(), StandardCharsets.UTF_8); - assertThat(lines.size(), is(25)); + assertThat(lines.size(), is(26)); int i = 0; - while (i < 25) { + while (i < lines.size()) { assertThat(lines.get(i++), is("primary-meta-store:")); assertThat(lines.get(i++), is(" access-control-type: READ_ONLY")); + assertThat(lines.get(i++), is(" configuration-properties:")); + assertThat(lines.get(i++), is(" hive.metastore.kerberos.principal: hive/_HOST@REALM")); assertThat(lines.get(i++), is(" database-prefix: ''")); assertThat(lines.get(i++), is(" impersonation-enabled: false")); assertThat(lines.get(i++), is(" latency: 0")); @@ -323,7 +329,6 @@ public void savePrimaryWriteFederations() throws Exception { assertThat(lines.get(i++), is(" remote-meta-store-uris: thrift://localhost:19083")); assertThat(lines.get(i++), is("federated-meta-stores:")); assertThat(lines.get(i++), is("- access-control-type: READ_ONLY")); - assertThat(lines.get(i++), is(" database-name-mapping: {}")); assertThat(lines.get(i++), is(" database-prefix: hcom_2_")); assertThat(lines.get(i++), is(" impersonation-enabled: false")); assertThat(lines.get(i++), is(" latency: 0"));