[AMORO-2861]: Support display Hudi table metadata in Amoro Dashboard (a…

…pache#2877) * Hudi | Add format catalog and hadoop implements. * Refactor | Move hive related common class to core module. * Show Hudi tables in dashboard * show right table column field types. * show right table properties * partition keys && primary keys * hoodie table files * snapshot summary * snapshot details * optimizing infos * optimizing task infos * spotless apply * fix checkstyle * use amoro shade * fix complie problems * remove useless codes * support cluster instant * fix unit test problems * set hudi to provided * fix spark unit test problems. * fix checkstyle * fix reviews * fix conflict * fix checkstyle * dependency change * revert mock hive * no hudi format catalog in spark ut * revert optimizer config * test-jar * remove optimizer config * fix compile problem * add test-jar goal for jar-plugin * fix method --------- Co-authored-by: ZhouJinsong <[email protected]>
tcodehuber · Aug 19, 2024 · aa0f91f · aa0f91f
1 parent 9b05fbf
commit aa0f91f
Show file tree

Hide file tree

Showing 37 changed files with 2,093 additions and 64 deletions.
diff --git a/amoro-ams/amoro-ams-dashboard/src/assets/icons/svg/hudi.svg b/amoro-ams/amoro-ams-dashboard/src/assets/icons/svg/hudi.svg
diff --git a/amoro-ams/amoro-ams-dashboard/src/types/common.type.ts b/amoro-ams/amoro-ams-dashboard/src/types/common.type.ts
@@ -353,6 +353,7 @@ export enum tableTypeIconMap {
   ARCTIC = 'amoro',
   HIVE = 'hive',
   PAIMON = 'paimon',
+  HUDI = 'hudi',
 }
 
 export type ILineChartOriginalData = Record<string, Record<string, number>>

diff --git a/amoro-ams/amoro-ams-dashboard/src/views/catalogs/Detail.vue b/amoro-ams/amoro-ams-dashboard/src/views/catalogs/Detail.vue
@@ -113,17 +113,20 @@ const tableFormatMap = {
   ICEBERG: 'ICEBERG',
   MIXED_ICEBERG: 'MIXED_ICEBERG',
   PAIMON: 'PAIMON',
+  HUDI: 'HUDI',
 }
 
 const tableFormatText = {
   [tableFormatMap.ICEBERG]: 'Iceberg',
   [tableFormatMap.MIXED_HIVE]: 'Mixed Hive',
   [tableFormatMap.MIXED_ICEBERG]: 'Mixed Iceberg',
   [tableFormatMap.PAIMON]: 'Paimon',
+  [tableFormatMap.HUDI]: 'Hudi',
 }
 const storeSupportFormat: { [prop: string]: string[] } = {
   ams: [tableFormatMap.MIXED_ICEBERG, tableFormatMap.ICEBERG],
-  hive: [tableFormatMap.MIXED_HIVE, tableFormatMap.MIXED_ICEBERG, tableFormatMap.ICEBERG, tableFormatMap.PAIMON],
+  hive: [tableFormatMap.MIXED_HIVE, tableFormatMap.MIXED_ICEBERG,
+      tableFormatMap.ICEBERG, tableFormatMap.PAIMON, tableFormatMap.HUDI],
   hadoop: [tableFormatMap.MIXED_ICEBERG, tableFormatMap.ICEBERG, tableFormatMap.PAIMON],
   glue: [tableFormatMap.MIXED_ICEBERG, tableFormatMap.ICEBERG],
   custom: [tableFormatMap.MIXED_ICEBERG, tableFormatMap.ICEBERG],

diff --git a/amoro-ams/amoro-ams-server/pom.xml b/amoro-ams/amoro-ams-server/pom.xml
@@ -194,6 +194,12 @@
             <artifactId>derby</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-dbcp2</artifactId>
@@ -292,6 +298,22 @@
             </exclusions>
         </dependency>
 
+        <!--  apache hudi dependencies -->
+        <dependency>
+            <groupId>org.apache.hudi</groupId>
+            <artifactId>hudi-java-client</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.orc</groupId>
+                    <artifactId>orc-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>asm</groupId>
+                    <artifactId>asm</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
         <!-- runtime dependencies -->
         <dependency>
             <groupId>org.apache.amoro</groupId>

diff --git a/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java b/amoro-ams/amoro-ams-server/src/main/java/org/apache/amoro/server/catalog/CatalogBuilder.java
@@ -43,15 +43,20 @@ public class CatalogBuilder {
   private static final Map<String, Set<TableFormat>> formatSupportedMatrix =
       ImmutableMap.of(
           CATALOG_TYPE_HADOOP,
-              Sets.newHashSet(TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG, TableFormat.PAIMON),
+              Sets.newHashSet(
+                  TableFormat.ICEBERG,
+                  TableFormat.MIXED_ICEBERG,
+                  TableFormat.PAIMON,
+                  TableFormat.HUDI),
           CATALOG_TYPE_GLUE, Sets.newHashSet(TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG),
           CATALOG_TYPE_CUSTOM, Sets.newHashSet(TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG),
           CATALOG_TYPE_HIVE,
               Sets.newHashSet(
                   TableFormat.ICEBERG,
                   TableFormat.MIXED_ICEBERG,
                   TableFormat.MIXED_HIVE,
-                  TableFormat.PAIMON),
+                  TableFormat.PAIMON,
+                  TableFormat.HUDI),
           CATALOG_TYPE_AMS, Sets.newHashSet(TableFormat.ICEBERG, TableFormat.MIXED_ICEBERG));
 
   private static String getAmsURI(Configurations serviceConfig) {

diff --git a/...oro-ams-server/src/main/java/org/apache/amoro/server/dashboard/FormatTableDescriptor.java b/...oro-ams-server/src/main/java/org/apache/amoro/server/dashboard/FormatTableDescriptor.java
@@ -48,7 +48,7 @@ List<AmoroSnapshotsOfTable> getSnapshots(
       AmoroTable<?> amoroTable, String ref, OperationType operationType);
 
   /** Get the snapshot detail information of the {@link AmoroTable}. */
-  List<PartitionFileBaseInfo> getSnapshotDetail(AmoroTable<?> amoroTable, long snapshotId);
+  List<PartitionFileBaseInfo> getSnapshotDetail(AmoroTable<?> amoroTable, String snapshotId);
 
   /** Get the DDL information of the {@link AmoroTable}. */
   List<DDLInfo> getTableOperations(AmoroTable<?> amoroTable);
@@ -65,7 +65,7 @@ Pair<List<OptimizingProcessInfo>, Integer> getOptimizingProcessesInfo(
       AmoroTable<?> amoroTable, int limit, int offset);
 
   /** Get the paged optimizing process tasks information of the {@link AmoroTable}. */
-  List<OptimizingTaskInfo> getOptimizingTaskInfos(AmoroTable<?> amoroTable, long processId);
+  List<OptimizingTaskInfo> getOptimizingTaskInfos(AmoroTable<?> amoroTable, String processId);
 
   /** Get the tag information of the {@link AmoroTable}. */
   List<TagOrBranchInfo> getTableTags(AmoroTable<?> amoroTable);