Add personalized configuration parameters for each metastore. (#315)

* Add personalized configuration parameters for each metastore. * Add personalized configuration parameters for each metastore * Recover * Update junit test * Update Junit Test * Update Junit Test * Update Junit Test * Format the code and update the readme * Revert * Update FederatedMetaStoreTest.java * Update PrimaryMetaStoreTest.java * Update AbstractMetaStore.java using new HashMap so the generated Yaml doesn't generate an anchor (reference &id001) * Update YamlFederatedMetaStoreStorageTest.java fixing test --------- Co-authored-by: yangyx <[email protected]> Co-authored-by: Patrick Duin <[email protected]>
ExpediaGroup · May 27, 2024 · 842aa2e · 842aa2e
1 parent eba5da0
commit 842aa2e
Show file tree

Hide file tree

Showing 7 changed files with 98 additions and 71 deletions.
diff --git a/README.md b/README.md
diff --git a/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java b/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java
@@ -51,14 +51,15 @@ public abstract class AbstractMetaStore {
   private List<String> writableDatabaseWhitelist;
   private List<String> mappedDatabases;
   private @Valid List<MappedTables> mappedTables;
-  private Map<String, String> databaseNameMapping = Collections.emptyMap();
+  private Map<String, String> databaseNameMapping = new HashMap<>();
   private @NotBlank String name;
   private @NotBlank String remoteMetaStoreUris;
   private @Valid MetastoreTunnel metastoreTunnel;
   private @NotNull AccessControlType accessControlType = AccessControlType.READ_ONLY;
   private transient @JsonProperty @NotNull MetaStoreStatus status = MetaStoreStatus.UNKNOWN;
   private long latency = 0;
   private transient @JsonIgnore HashBiMap<String, String> databaseNameBiMapping = HashBiMap.create();
+  private Map<String, String> configurationProperties = new HashMap<>();
 
   public AbstractMetaStore(String name, String remoteMetaStoreUris, AccessControlType accessControlType) {
     this.name = name;
@@ -201,6 +202,15 @@ public HashBiMap<String, String> getDatabaseNameBiMapping() {
     return databaseNameBiMapping;
   }
 
+  public Map<String, String> getConfigurationProperties() {
+    return configurationProperties;
+  }
+
+  public void setConfigurationProperties(
+          Map<String, String> configurationProperties) {
+    this.configurationProperties = configurationProperties;
+  }
+
   @Transient
   public MetaStoreStatus getStatus() {
     return status;
@@ -242,5 +252,4 @@ public String toString() {
         .add("status", status)
         .toString();
   }
-
 }
diff --git a/...-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java b/...-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java
@@ -72,7 +72,7 @@ public void nullDatabasePrefix() {
 
   @Test
   public void toJson() throws Exception {
-    String expected = "{\"accessControlType\":\"READ_ONLY\",\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"name_\",\"federationType\":\"FEDERATED\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}";
+    String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":{},\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"name_\",\"federationType\":\"FEDERATED\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}";
     ObjectMapper mapper = new ObjectMapper();
     // Sorting to get deterministic test behaviour
     mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY);

diff --git a/...le-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java b/...le-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java
@@ -89,7 +89,7 @@ public void nonEmptyDatabasePrefix() {
 
   @Test
   public void toJson() throws Exception {
-    String expected = "{\"accessControlType\":\"READ_ONLY\",\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"\",\"federationType\":\"PRIMARY\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}";
+    String expected = "{\"accessControlType\":\"READ_ONLY\",\"configurationProperties\":{},\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"\",\"federationType\":\"PRIMARY\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}";
     ObjectMapper mapper = new ObjectMapper();
     // Sorting to get deterministic test behaviour
     mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY);

diff --git a/...ava/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java b/...ava/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java
@@ -47,6 +47,10 @@ public CloseableThriftHiveMetastoreIface newInstance(AbstractMetaStore metaStore
     if (waggleDanceConfiguration.getConfigurationProperties() != null) {
       properties.putAll(waggleDanceConfiguration.getConfigurationProperties());
     }
+    if (metaStore.getConfigurationProperties() != null) {
+      properties.putAll(metaStore.getConfigurationProperties());
+    }
+
     return newHiveInstance(metaStore, properties);
   }
 

diff --git a/...com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactoryTest.java b/...com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactoryTest.java
@@ -24,10 +24,12 @@
 
 import static com.hotels.bdp.waggledance.api.model.AbstractMetaStore.newFederatedInstance;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
+import com.hotels.bdp.waggledance.api.model.FederatedMetaStore;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.junit.Before;
@@ -68,8 +70,9 @@ public void setUp() {
   @Test
   public void defaultFactory() {
     ArgumentCaptor<HiveConf> hiveConfCaptor = ArgumentCaptor.forClass(HiveConf.class);
-
-    factory.newInstance(newFederatedInstance("fed1", THRIFT_URI));
+    FederatedMetaStore fed1 = newFederatedInstance("fed1", THRIFT_URI);
+    fed1.setConfigurationProperties(Collections.singletonMap(ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname, "hive/[email protected]"));
+    factory.newInstance(fed1);
     verify(defaultMetaStoreClientFactory).newInstance(hiveConfCaptor.capture(), eq(
         "waggledance-fed1"), eq(3), eq(2000));
     verifyNoInteractions(tunnelingMetaStoreClientFactory);
@@ -80,6 +83,7 @@ public void defaultFactory() {
     assertThat(hiveConf.getTimeVar(ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS), is(5L));
     assertThat(hiveConf.getBoolVar(ConfVars.METASTORE_USE_THRIFT_FRAMED_TRANSPORT), is(true));
     assertThat(hiveConf.getBoolVar(ConfVars.METASTORE_USE_THRIFT_COMPACT_PROTOCOL), is(false));
+    assertThat(hiveConf.getVar(ConfVars.METASTORE_KERBEROS_PRINCIPAL), is("hive/[email protected]"));
   }
 
   @Test

diff --git a/...va/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java b/...va/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java
@@ -23,6 +23,7 @@
 import java.io.File;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
+import java.util.Collections;
 import java.util.List;
 
 import javax.validation.ConstraintViolationException;
@@ -211,10 +212,11 @@ public void saveFederationWriteFederations() throws Exception {
     MappedTables mappedTables2 = new MappedTables("db2", Lists.newArrayList("tbl2"));
     newFederatedInstance.setMappedTables(Lists.newArrayList(mappedTables1, mappedTables2));
     newFederatedInstance.setHiveMetastoreFilterHook("filter.hook.class");
+    newFederatedInstance.setConfigurationProperties(Collections.singletonMap("hive.metastore.kerberos.principal", "hive/_HOST@REALM"));
     storage.insert(newFederatedInstance);
     storage.saveFederation();
     List<String> lines = Files.readAllLines(file.toPath(), StandardCharsets.UTF_8);
-    assertThat(lines.size(), is(24));
+    assertThat(lines.size(), is(25));
     assertThat(lines.get(0), is("primary-meta-store:"));
     assertThat(lines.get(1), is("  access-control-type: READ_ONLY"));
     assertThat(lines.get(2), is("  database-prefix: ''"));
@@ -223,22 +225,23 @@ public void saveFederationWriteFederations() throws Exception {
     assertThat(lines.get(5), is("  remote-meta-store-uris: thrift://localhost:19083"));
     assertThat(lines.get(6), is("federated-meta-stores:"));
     assertThat(lines.get(7), is("- access-control-type: READ_ONLY"));
-    assertThat(lines.get(8), is("  database-name-mapping: {}"));
-    assertThat(lines.get(9), is("  database-prefix: hcom_2_"));
-    assertThat(lines.get(10), is("  hive-metastore-filter-hook: filter.hook.class"));
-    assertThat(lines.get(11), is("  latency: 0"));
-    assertThat(lines.get(12), is("  mapped-databases:"));
-    assertThat(lines.get(13), is("  - db1"));
-    assertThat(lines.get(14), is("  - db2"));
-    assertThat(lines.get(15), is("  mapped-tables:"));
-    assertThat(lines.get(16), is("  - database: db1"));
-    assertThat(lines.get(17), is("    mapped-tables:"));
-    assertThat(lines.get(18), is("    - tbl1"));
-    assertThat(lines.get(19), is("  - database: db2"));
-    assertThat(lines.get(20), is("    mapped-tables:"));
-    assertThat(lines.get(21), is("    - tbl2"));
-    assertThat(lines.get(22), is("  name: hcom_2"));
-    assertThat(lines.get(23), is("  remote-meta-store-uris: thrift://localhost:29083"));
+    assertThat(lines.get(8), is("  configuration-properties:"));
+    assertThat(lines.get(9), is("    hive.metastore.kerberos.principal: hive/_HOST@REALM"));
+    assertThat(lines.get(10), is("  database-prefix: hcom_2_"));
+    assertThat(lines.get(11), is("  hive-metastore-filter-hook: filter.hook.class"));
+    assertThat(lines.get(12), is("  latency: 0"));
+    assertThat(lines.get(13), is("  mapped-databases:"));
+    assertThat(lines.get(14), is("  - db1"));
+    assertThat(lines.get(15), is("  - db2"));
+    assertThat(lines.get(16), is("  mapped-tables:"));
+    assertThat(lines.get(17), is("  - database: db1"));
+    assertThat(lines.get(18), is("    mapped-tables:"));
+    assertThat(lines.get(19), is("    - tbl1"));
+    assertThat(lines.get(20), is("  - database: db2"));
+    assertThat(lines.get(21), is("    mapped-tables:"));
+    assertThat(lines.get(22), is("    - tbl2"));
+    assertThat(lines.get(23), is("  name: hcom_2"));
+    assertThat(lines.get(24), is("  remote-meta-store-uris: thrift://localhost:29083"));
   }
 
   @Test
@@ -292,34 +295,36 @@ public void savePrimaryWriteFederations() throws Exception {
     MappedTables mappedTables1 = new MappedTables("db1", Lists.newArrayList("tbl1"));
     MappedTables mappedTables2 = new MappedTables("db2", Lists.newArrayList("tbl2"));
     primaryMetaStore.setMappedTables(Lists.newArrayList(mappedTables1, mappedTables2));
+    primaryMetaStore.setConfigurationProperties(Collections.singletonMap("hive.metastore.kerberos.principal", "hive/_HOST@REALM"));
     storage.insert(primaryMetaStore);
     storage.insert(newFederatedInstance("hcom_2", "thrift://localhost:29083"));
     storage.saveFederation();
     List<String> lines = Files.readAllLines(file.toPath(), StandardCharsets.UTF_8);
-    assertThat(lines.size(), is(23));
+    assertThat(lines.size(), is(24));
     assertThat(lines.get(0), is("primary-meta-store:"));
     assertThat(lines.get(1), is("  access-control-type: READ_ONLY"));
-    assertThat(lines.get(2), is("  database-prefix: ''"));
-    assertThat(lines.get(3), is("  latency: 0"));
-    assertThat(lines.get(4), is("  mapped-databases:"));
-    assertThat(lines.get(5), is("  - db1"));
-    assertThat(lines.get(6), is("  - db2"));
-    assertThat(lines.get(7), is("  mapped-tables:"));
-    assertThat(lines.get(8), is("  - database: db1"));
-    assertThat(lines.get(9), is("    mapped-tables:"));
-    assertThat(lines.get(10), is("    - tbl1"));
-    assertThat(lines.get(11), is("  - database: db2"));
-    assertThat(lines.get(12), is("    mapped-tables:"));
-    assertThat(lines.get(13), is("    - tbl2"));
-    assertThat(lines.get(14), is("  name: hcom_1"));
-    assertThat(lines.get(15), is("  remote-meta-store-uris: thrift://localhost:19083"));
-    assertThat(lines.get(16), is("federated-meta-stores:"));
-    assertThat(lines.get(17), is("- access-control-type: READ_ONLY"));
-    assertThat(lines.get(18), is("  database-name-mapping: {}"));
-    assertThat(lines.get(19), is("  database-prefix: hcom_2_"));
-    assertThat(lines.get(20), is("  latency: 0"));
-    assertThat(lines.get(21), is("  name: hcom_2"));
-    assertThat(lines.get(22), is("  remote-meta-store-uris: thrift://localhost:29083"));
+    assertThat(lines.get(2), is("  configuration-properties:"));
+    assertThat(lines.get(3), is("    hive.metastore.kerberos.principal: hive/_HOST@REALM"));
+    assertThat(lines.get(4), is("  database-prefix: ''"));
+    assertThat(lines.get(5), is("  latency: 0"));
+    assertThat(lines.get(6), is("  mapped-databases:"));
+    assertThat(lines.get(7), is("  - db1"));
+    assertThat(lines.get(8), is("  - db2"));
+    assertThat(lines.get(9), is("  mapped-tables:"));
+    assertThat(lines.get(10), is("  - database: db1"));
+    assertThat(lines.get(11), is("    mapped-tables:"));
+    assertThat(lines.get(12), is("    - tbl1"));
+    assertThat(lines.get(13), is("  - database: db2"));
+    assertThat(lines.get(14), is("    mapped-tables:"));
+    assertThat(lines.get(15), is("    - tbl2"));
+    assertThat(lines.get(16), is("  name: hcom_1"));
+    assertThat(lines.get(17), is("  remote-meta-store-uris: thrift://localhost:19083"));
+    assertThat(lines.get(18), is("federated-meta-stores:"));
+    assertThat(lines.get(19), is("- access-control-type: READ_ONLY"));
+    assertThat(lines.get(20), is("  database-prefix: hcom_2_"));
+    assertThat(lines.get(21), is("  latency: 0"));
+    assertThat(lines.get(22), is("  name: hcom_2"));
+    assertThat(lines.get(23), is("  remote-meta-store-uris: thrift://localhost:29083"));
   }
 
   private PrimaryMetaStore newPrimaryInstance(String name, String remoteMetaStoreUris) {