Merge branch 'datahub-project:master' into master

acryldata · Dec 22, 2023 · 6d79251 · 6d79251
2 parents 9fbbca4 + 52687f3
commit 6d79251
Show file tree

Hide file tree

Showing 119 changed files with 2,138 additions and 465 deletions.
diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -917,13 +917,13 @@ jobs:
       ]
     steps:
       - uses: aws-actions/configure-aws-credentials@v1
-        if: ${{ needs.setup.outputs.publish != 'false' }}
+        if: ${{ needs.setup.outputs.publish != 'false' && github.repository_owner == 'datahub-project' && needs.setup.outputs.repository_name == 'datahub' }}
         with:
           aws-access-key-id: ${{ secrets.AWS_SQS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SQS_ACCESS_KEY }}
           aws-region: us-west-2
       - uses: isbang/[email protected]
-        if: ${{ needs.setup.outputs.publish != 'false' }}
+        if: ${{ needs.setup.outputs.publish != 'false' && github.repository_owner == 'datahub-project' && needs.setup.outputs.repository_name == 'datahub' }}
         with:
           sqs-url: ${{ secrets.DATAHUB_HEAD_SYNC_QUEUE }}
           message: '{ "command": "git-sync", "args" : {"repoName": "${{ needs.setup.outputs.repository_name }}", "repoOrg": "${{ github.repository_owner }}", "repoBranch": "${{ needs.setup.outputs.branch_name }}", "repoShaShort": "${{ needs.setup.outputs.short_sha }}" }}'
diff --git a/build.gradle b/build.gradle
@@ -46,6 +46,7 @@ plugins {
   id 'com.gorylenko.gradle-git-properties' version '2.4.1'
   id 'com.github.johnrengelman.shadow' version '8.1.1' apply false
   id 'com.palantir.docker' version '0.35.0' apply false
+  id 'com.avast.gradle.docker-compose' version '0.17.5'
   id "com.diffplug.spotless" version "6.23.3"
   // https://blog.ltgt.net/javax-jakarta-mess-and-gradle-solution/
   // TODO id "org.gradlex.java-ecosystem-capabilities" version "1.0"

diff --git a/...core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java b/...core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java
@@ -91,6 +91,7 @@ private List<AnalyticsChart> getProductAnalyticsCharts(Authentication authentica
     final List<AnalyticsChart> charts = new ArrayList<>();
     DateUtil dateUtil = new DateUtil();
     final DateTime startOfNextWeek = dateUtil.getStartOfNextWeek();
+    final DateTime startOfThisMonth = dateUtil.getStartOfThisMonth();
     final DateTime startOfNextMonth = dateUtil.getStartOfNextMonth();
     final DateRange trailingWeekDateRange = dateUtil.getTrailingWeekDateRange();
 
@@ -103,7 +104,7 @@ private List<AnalyticsChart> getProductAnalyticsCharts(Authentication authentica
     charts.add(
         getActiveUsersTimeSeriesChart(
             startOfNextMonth.minusMonths(12),
-            startOfNextMonth.minusMillis(1),
+            startOfThisMonth.minusMillis(1),
             "Monthly Active Users",
             DateInterval.MONTH));
 

diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/util/DateUtil.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/util/DateUtil.java
@@ -13,6 +13,10 @@ public DateTime getStartOfNextWeek() {
     return setTimeToZero(getNow().withDayOfWeek(DateTimeConstants.SUNDAY).plusDays(1));
   }
 
+  public DateTime getStartOfThisMonth() {
+    return setTimeToZero(getNow().withDayOfMonth(1));
+  }
+
   public DateTime getStartOfNextMonth() {
     return setTimeToZero(getNow().withDayOfMonth(1).plusMonths(1));
   }

diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeCleanupConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeCleanupConfig.java
@@ -7,13 +7,16 @@
 import com.linkedin.metadata.utils.elasticsearch.IndexConvention;
 import io.ebean.Database;
 import javax.annotation.Nonnull;
+import lombok.extern.slf4j.Slf4j;
 import org.opensearch.client.RestHighLevelClient;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.DependsOn;
 
+@Slf4j
 @Configuration
 public class NoCodeCleanupConfig {
 
@@ -26,6 +29,7 @@ public class NoCodeCleanupConfig {
     "elasticSearchRestHighLevelClient",
     INDEX_CONVENTION_BEAN
   })
+  @ConditionalOnProperty(name = "entityService.impl", havingValue = "ebean", matchIfMissing = true)
   @Nonnull
   public NoCodeCleanupUpgrade createInstance() {
     final Database ebeanServer = applicationContext.getBean(Database.class);
@@ -34,4 +38,12 @@ public NoCodeCleanupUpgrade createInstance() {
     final IndexConvention indexConvention = applicationContext.getBean(IndexConvention.class);
     return new NoCodeCleanupUpgrade(ebeanServer, graphClient, searchClient, indexConvention);
   }
+
+  @Bean(name = "noCodeCleanup")
+  @ConditionalOnProperty(name = "entityService.impl", havingValue = "cassandra")
+  @Nonnull
+  public NoCodeCleanupUpgrade createNotImplInstance() {
+    log.warn("NoCode is not supported for cassandra!");
+    return new NoCodeCleanupUpgrade(null, null, null, null);
+  }
 }
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeUpgradeConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeUpgradeConfig.java
@@ -6,19 +6,23 @@
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import io.ebean.Database;
 import javax.annotation.Nonnull;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.DependsOn;
 
+@Slf4j
 @Configuration
 public class NoCodeUpgradeConfig {
 
   @Autowired ApplicationContext applicationContext;
 
   @Bean(name = "noCodeUpgrade")
   @DependsOn({"ebeanServer", "entityService", "systemRestliEntityClient", "entityRegistry"})
+  @ConditionalOnProperty(name = "entityService.impl", havingValue = "ebean", matchIfMissing = true)
   @Nonnull
   public NoCodeUpgrade createInstance() {
     final Database ebeanServer = applicationContext.getBean(Database.class);
@@ -29,4 +33,12 @@ public NoCodeUpgrade createInstance() {
 
     return new NoCodeUpgrade(ebeanServer, entityService, entityRegistry, entityClient);
   }
+
+  @Bean(name = "noCodeUpgrade")
+  @ConditionalOnProperty(name = "entityService.impl", havingValue = "cassandra")
+  @Nonnull
+  public NoCodeUpgrade createNotImplInstance() {
+    log.warn("NoCode is not supported for cassandra!");
+    return new NoCodeUpgrade(null, null, null, null);
+  }
 }
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreBackupConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreBackupConfig.java
@@ -8,12 +8,15 @@
 import com.linkedin.metadata.search.EntitySearchService;
 import io.ebean.Database;
 import javax.annotation.Nonnull;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.DependsOn;
 
+@Slf4j
 @Configuration
 public class RestoreBackupConfig {
   @Autowired ApplicationContext applicationContext;
@@ -27,6 +30,7 @@ public class RestoreBackupConfig {
     "searchService",
     "entityRegistry"
   })
+  @ConditionalOnProperty(name = "entityService.impl", havingValue = "ebean", matchIfMissing = true)
   @Nonnull
   public RestoreBackup createInstance() {
     final Database ebeanServer = applicationContext.getBean(Database.class);
@@ -40,4 +44,12 @@ public RestoreBackup createInstance() {
     return new RestoreBackup(
         ebeanServer, entityService, entityRegistry, entityClient, graphClient, searchClient);
   }
+
+  @Bean(name = "restoreBackup")
+  @ConditionalOnProperty(name = "entityService.impl", havingValue = "cassandra")
+  @Nonnull
+  public RestoreBackup createNotImplInstance() {
+    log.warn("restoreIndices is not supported for cassandra!");
+    return new RestoreBackup(null, null, null, null, null, null);
+  }
 }
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreIndicesConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreIndicesConfig.java
@@ -7,18 +7,22 @@
 import com.linkedin.metadata.search.EntitySearchService;
 import io.ebean.Database;
 import javax.annotation.Nonnull;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.DependsOn;
 
+@Slf4j
 @Configuration
 public class RestoreIndicesConfig {
   @Autowired ApplicationContext applicationContext;
 
   @Bean(name = "restoreIndices")
   @DependsOn({"ebeanServer", "entityService", "searchService", "graphService", "entityRegistry"})
+  @ConditionalOnProperty(name = "entityService.impl", havingValue = "ebean", matchIfMissing = true)
   @Nonnull
   public RestoreIndices createInstance() {
     final Database ebeanServer = applicationContext.getBean(Database.class);
@@ -31,4 +35,12 @@ public RestoreIndices createInstance() {
     return new RestoreIndices(
         ebeanServer, entityService, entityRegistry, entitySearchService, graphService);
   }
+
+  @Bean(name = "restoreIndices")
+  @ConditionalOnProperty(name = "entityService.impl", havingValue = "cassandra")
+  @Nonnull
+  public RestoreIndices createNotImplInstance() {
+    log.warn("restoreIndices is not supported for cassandra!");
+    return new RestoreIndices(null, null, null, null, null);
+  }
 }
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java
@@ -13,6 +13,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import javax.annotation.Nullable;
 
 public class NoCodeUpgrade implements Upgrade {
 
@@ -26,12 +27,17 @@ public class NoCodeUpgrade implements Upgrade {
 
   // Upgrade requires the Database.
   public NoCodeUpgrade(
-      final Database server,
+      @Nullable final Database server,
       final EntityService entityService,
       final EntityRegistry entityRegistry,
       final SystemRestliEntityClient entityClient) {
-    _steps = buildUpgradeSteps(server, entityService, entityRegistry, entityClient);
-    _cleanupSteps = buildCleanupSteps();
+    if (server != null) {
+      _steps = buildUpgradeSteps(server, entityService, entityRegistry, entityClient);
+      _cleanupSteps = buildCleanupSteps();
+    } else {
+      _steps = List.of();
+      _cleanupSteps = List.of();
+    }
   }
 
   @Override

diff --git a/...pgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/NoCodeCleanupUpgrade.java b/...pgrade/src/main/java/com/linkedin/datahub/upgrade/nocodecleanup/NoCodeCleanupUpgrade.java
@@ -9,6 +9,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import javax.annotation.Nullable;
 import org.opensearch.client.RestHighLevelClient;
 
 public class NoCodeCleanupUpgrade implements Upgrade {
@@ -18,12 +19,17 @@ public class NoCodeCleanupUpgrade implements Upgrade {
 
   // Upgrade requires the Database.
   public NoCodeCleanupUpgrade(
-      final Database server,
+      @Nullable final Database server,
       final GraphService graphClient,
       final RestHighLevelClient searchClient,
       final IndexConvention indexConvention) {
-    _steps = buildUpgradeSteps(server, graphClient, searchClient, indexConvention);
-    _cleanupSteps = buildCleanupSteps();
+    if (server != null) {
+      _steps = buildUpgradeSteps(server, graphClient, searchClient, indexConvention);
+      _cleanupSteps = buildCleanupSteps();
+    } else {
+      _steps = List.of();
+      _cleanupSteps = List.of();
+    }
   }
 
   @Override

diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreBackup.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreBackup.java
@@ -16,20 +16,26 @@
 import io.ebean.Database;
 import java.util.ArrayList;
 import java.util.List;
+import javax.annotation.Nullable;
 
 public class RestoreBackup implements Upgrade {
 
   private final List<UpgradeStep> _steps;
 
   public RestoreBackup(
-      final Database server,
+      @Nullable final Database server,
       final EntityService entityService,
       final EntityRegistry entityRegistry,
       final SystemRestliEntityClient entityClient,
       final GraphService graphClient,
       final EntitySearchService searchClient) {
-    _steps =
-        buildSteps(server, entityService, entityRegistry, entityClient, graphClient, searchClient);
+    if (server != null) {
+      _steps =
+          buildSteps(
+              server, entityService, entityRegistry, entityClient, graphClient, searchClient);
+    } else {
+      _steps = List.of();
+    }
   }
 
   @Override

diff --git a/...hub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/RestoreIndices.java b/...hub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restoreindices/RestoreIndices.java
@@ -13,6 +13,7 @@
 import io.ebean.Database;
 import java.util.ArrayList;
 import java.util.List;
+import javax.annotation.Nullable;
 
 public class RestoreIndices implements Upgrade {
   public static final String BATCH_SIZE_ARG_NAME = "batchSize";
@@ -23,18 +24,23 @@ public class RestoreIndices implements Upgrade {
   public static final String WRITER_POOL_SIZE = "WRITER_POOL_SIZE";
   public static final String URN_ARG_NAME = "urn";
   public static final String URN_LIKE_ARG_NAME = "urnLike";
+  public static final String URN_BASED_PAGINATION_ARG_NAME = "urnBasedPagination";
 
   public static final String STARTING_OFFSET_ARG_NAME = "startingOffset";
 
   private final List<UpgradeStep> _steps;
 
   public RestoreIndices(
-      final Database server,
+      @Nullable final Database server,
       final EntityService entityService,
       final EntityRegistry entityRegistry,
       final EntitySearchService entitySearchService,
       final GraphService graphService) {
-    _steps = buildSteps(server, entityService, entityRegistry, entitySearchService, graphService);
+    if (server != null) {
+      _steps = buildSteps(server, entityService, entityRegistry, entitySearchService, graphService);
+    } else {
+      _steps = List.of();
+    }
   }
 
   @Override