Merge branch 'master' into add_grouplist_string_logic

.github/workflows/airflow-plugin.yml

@@ -32,20 +32,21 @@ jobs:
     strategy:
       matrix:
         include:
+          # Note: this should be kept in sync with tox.ini.
           - python-version: "3.8"
             extra_pip_requirements: "apache-airflow~=2.1.4"
             extra_pip_extras: plugin-v1
           - python-version: "3.8"
             extra_pip_requirements: "apache-airflow~=2.2.4"
             extra_pip_extras: plugin-v1
           - python-version: "3.10"
-            extra_pip_requirements: "apache-airflow~=2.4.0"
+            extra_pip_requirements: 'apache-airflow~=2.4.0 pluggy==1.0.0 "pendulum<3.0"'
             extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extra_pip_requirements: "apache-airflow~=2.6.0"
+            extra_pip_requirements: 'apache-airflow~=2.6.0 "pendulum<3.0"'
             extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extra_pip_requirements: "apache-airflow>=2.7.0"
+            extra_pip_requirements: "apache-airflow>=2.7.0 pydantic==2.4.2"
             extra_pip_extras: plugin-v2
       fail-fast: false
     steps:

.github/workflows/docker-unified.yml

@@ -911,13 +911,13 @@ jobs:
       ]
     steps:
       - uses: aws-actions/configure-aws-credentials@v1
-        if: ${{ needs.setup.outputs.publish != 'false' }}
+        if: ${{ needs.setup.outputs.publish != 'false' && github.repository_owner == 'datahub-project' && needs.setup.outputs.repository_name == 'datahub' }}
         with:
           aws-access-key-id: ${{ secrets.AWS_SQS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SQS_ACCESS_KEY }}
           aws-region: us-west-2
       - uses: isbang/[email protected]
-        if: ${{ needs.setup.outputs.publish != 'false' }}
+        if: ${{ needs.setup.outputs.publish != 'false' && github.repository_owner == 'datahub-project' && needs.setup.outputs.repository_name == 'datahub' }}
         with:
           sqs-url: ${{ secrets.DATAHUB_HEAD_SYNC_QUEUE }}
           message: '{ "command": "git-sync", "args" : {"repoName": "${{ needs.setup.outputs.repository_name }}", "repoOrg": "${{ github.repository_owner }}", "repoBranch": "${{ needs.setup.outputs.branch_name }}", "repoShaShort": "${{ needs.setup.outputs.short_sha }}" }}'

build.gradle

@@ -46,6 +46,7 @@ plugins {
   id 'com.gorylenko.gradle-git-properties' version '2.4.1'
   id 'com.github.johnrengelman.shadow' version '8.1.1' apply false
   id 'com.palantir.docker' version '0.35.0' apply false
+  id 'com.avast.gradle.docker-compose' version '0.17.5'
   id "com.diffplug.spotless" version "6.23.3"
   // https://blog.ltgt.net/javax-jakarta-mess-and-gradle-solution/
   // TODO id "org.gradlex.java-ecosystem-capabilities" version "1.0"

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java

@@ -91,6 +91,7 @@ private List<AnalyticsChart> getProductAnalyticsCharts(Authentication authentica
     final List<AnalyticsChart> charts = new ArrayList<>();
     DateUtil dateUtil = new DateUtil();
     final DateTime startOfNextWeek = dateUtil.getStartOfNextWeek();
+    final DateTime startOfThisMonth = dateUtil.getStartOfThisMonth();
     final DateTime startOfNextMonth = dateUtil.getStartOfNextMonth();
     final DateRange trailingWeekDateRange = dateUtil.getTrailingWeekDateRange();
 
@@ -103,7 +104,7 @@ private List<AnalyticsChart> getProductAnalyticsCharts(Authentication authentica
     charts.add(
         getActiveUsersTimeSeriesChart(
             startOfNextMonth.minusMonths(12),
-            startOfNextMonth.minusMillis(1),
+            startOfThisMonth.minusMillis(1),
             "Monthly Active Users",
             DateInterval.MONTH));
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java

@@ -12,6 +12,7 @@ public class FeatureFlags {
   private boolean readOnlyModeEnabled = false;
   private boolean showSearchFiltersV2 = false;
   private boolean showBrowseV2 = false;
+  private boolean platformBrowseV2 = false;
   private PreProcessHooks preProcessHooks;
   private boolean showAcrylInfo = false;
   private boolean showAccessManagement = false;

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java

@@ -2,14 +2,16 @@
 
 import static com.linkedin.datahub.graphql.Constants.BROWSE_PATH_V2_DELIMITER;
 import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument;
-import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.resolveView;
+import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.*;
 
+import com.google.common.collect.ImmutableList;
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.generated.BrowseResultGroupV2;
 import com.linkedin.datahub.graphql.generated.BrowseResultMetadata;
 import com.linkedin.datahub.graphql.generated.BrowseResultsV2;
 import com.linkedin.datahub.graphql.generated.BrowseV2Input;
+import com.linkedin.datahub.graphql.generated.EntityType;
 import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper;
 import com.linkedin.datahub.graphql.resolvers.ResolverUtils;
 import com.linkedin.datahub.graphql.resolvers.search.SearchUtils;
@@ -43,8 +45,8 @@ public class BrowseV2Resolver implements DataFetcher<CompletableFuture<BrowseRes
   public CompletableFuture<BrowseResultsV2> get(DataFetchingEnvironment environment) {
     final QueryContext context = environment.getContext();
     final BrowseV2Input input = bindArgument(environment.getArgument("input"), BrowseV2Input.class);
-    final String entityName = EntityTypeMapper.getName(input.getType());
 
+    final List<String> entityNames = getEntityNames(input);
     final int start = input.getStart() != null ? input.getStart() : DEFAULT_START;
     final int count = input.getCount() != null ? input.getCount() : DEFAULT_COUNT;
     final String query = input.getQuery() != null ? input.getQuery() : "*";
@@ -70,7 +72,7 @@ public CompletableFuture<BrowseResultsV2> get(DataFetchingEnvironment environmen
 
             BrowseResultV2 browseResults =
                 _entityClient.browseV2(
-                    entityName,
+                    entityNames,
                     pathStr,
                     maybeResolvedView != null
                         ? SearchUtils.combineFilters(
@@ -87,6 +89,18 @@ public CompletableFuture<BrowseResultsV2> get(DataFetchingEnvironment environmen
         });
   }
 
+  public static List<String> getEntityNames(BrowseV2Input input) {
+    List<EntityType> entityTypes;
+    if (input.getTypes() != null && input.getTypes().size() > 0) {
+      entityTypes = input.getTypes();
+    } else if (input.getType() != null) {
+      entityTypes = ImmutableList.of(input.getType());
+    } else {
+      entityTypes = BROWSE_ENTITY_TYPES;
+    }
+    return entityTypes.stream().map(EntityTypeMapper::getName).collect(Collectors.toList());
+  }
+
   private BrowseResultsV2 mapBrowseResults(BrowseResultV2 browseResults) {
     BrowseResultsV2 results = new BrowseResultsV2();
     results.setTotal(browseResults.getNumGroups());

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java

@@ -175,6 +175,7 @@ public CompletableFuture<AppConfig> get(final DataFetchingEnvironment environmen
             .setShowAcrylInfo(_featureFlags.isShowAcrylInfo())
             .setShowAccessManagement(_featureFlags.isShowAccessManagement())
             .setNestedDomainsEnabled(_featureFlags.isNestedDomainsEnabled())
+            .setPlatformBrowseV2(_featureFlags.isPlatformBrowseV2())
             .build();
 
     appConfig.setFeatureFlags(featureFlagsConfig);

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java

@@ -92,6 +92,20 @@ private SearchUtils() {}
           EntityType.NOTEBOOK,
           EntityType.DATA_PRODUCT);
 
+  /** Entities that are part of browse by default */
+  public static final List<EntityType> BROWSE_ENTITY_TYPES =
+      ImmutableList.of(
+          EntityType.DATASET,
+          EntityType.DASHBOARD,
+          EntityType.CHART,
+          EntityType.CONTAINER,
+          EntityType.MLMODEL,
+          EntityType.MLMODEL_GROUP,
+          EntityType.MLFEATURE_TABLE,
+          EntityType.DATA_FLOW,
+          EntityType.DATA_JOB,
+          EntityType.NOTEBOOK);
+
   /** A prioritized list of source filter types used to generate quick filters */
   public static final List<String> PRIORITIZED_SOURCE_ENTITY_TYPES =
       Stream.of(

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/util/DateUtil.java

@@ -13,6 +13,10 @@ public DateTime getStartOfNextWeek() {
     return setTimeToZero(getNow().withDayOfWeek(DateTimeConstants.SUNDAY).plusDays(1));
   }
 
+  public DateTime getStartOfThisMonth() {
+    return setTimeToZero(getNow().withDayOfMonth(1));
+  }
+
   public DateTime getStartOfNextMonth() {
     return setTimeToZero(getNow().withDayOfMonth(1).plusMonths(1));
   }

datahub-graphql-core/src/main/resources/app.graphql

@@ -437,6 +437,11 @@ type FeatureFlagsConfig {
   """
   showBrowseV2: Boolean!
 
+  """
+  Whether browse v2 is platform mode, which means that platforms are displayed instead of entity types at the root.
+  """
+  platformBrowseV2: Boolean!
+
   """
   Whether we should show CTAs in the UI related to moving to Managed DataHub by Acryl.
   """

datahub-graphql-core/src/main/resources/search.graphql

@@ -1176,9 +1176,14 @@ Input required for browse queries
 """
 input BrowseV2Input {
   """
-  The browse entity type
+  The browse entity type - deprecated use types instead
   """
-  type: EntityType!
+  type: EntityType
+
+  """
+  The browse entity type - deprecated use types instead. If not provided, all types will be used.
+  """
+  types: [EntityType!]
 
   """
   The browse path V2 - a list with each entry being part of the browse path V2

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/browse/BrowseV2ResolverTest.java

@@ -249,7 +249,7 @@ private static EntityClient initMockEntityClient(
     EntityClient client = Mockito.mock(EntityClient.class);
     Mockito.when(
             client.browseV2(
-                Mockito.eq(entityName),
+                Mockito.eq(ImmutableList.of(entityName)),
                 Mockito.eq(path),
                 Mockito.eq(filter),
                 Mockito.eq(query),

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeCleanupConfig.java

@@ -7,13 +7,16 @@
 import com.linkedin.metadata.utils.elasticsearch.IndexConvention;
 import io.ebean.Database;
 import javax.annotation.Nonnull;
+import lombok.extern.slf4j.Slf4j;
 import org.opensearch.client.RestHighLevelClient;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.DependsOn;
 
+@Slf4j
 @Configuration
 public class NoCodeCleanupConfig {
 
@@ -26,6 +29,7 @@ public class NoCodeCleanupConfig {
     "elasticSearchRestHighLevelClient",
     INDEX_CONVENTION_BEAN
   })
+  @ConditionalOnProperty(name = "entityService.impl", havingValue = "ebean", matchIfMissing = true)
   @Nonnull
   public NoCodeCleanupUpgrade createInstance() {
     final Database ebeanServer = applicationContext.getBean(Database.class);
@@ -34,4 +38,12 @@ public NoCodeCleanupUpgrade createInstance() {
     final IndexConvention indexConvention = applicationContext.getBean(IndexConvention.class);
     return new NoCodeCleanupUpgrade(ebeanServer, graphClient, searchClient, indexConvention);
   }
+
+  @Bean(name = "noCodeCleanup")
+  @ConditionalOnProperty(name = "entityService.impl", havingValue = "cassandra")
+  @Nonnull
+  public NoCodeCleanupUpgrade createNotImplInstance() {
+    log.warn("NoCode is not supported for cassandra!");
+    return new NoCodeCleanupUpgrade(null, null, null, null);
+  }
 }

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/NoCodeUpgradeConfig.java

@@ -6,19 +6,23 @@
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import io.ebean.Database;
 import javax.annotation.Nonnull;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.DependsOn;
 
+@Slf4j
 @Configuration
 public class NoCodeUpgradeConfig {
 
   @Autowired ApplicationContext applicationContext;
 
   @Bean(name = "noCodeUpgrade")
   @DependsOn({"ebeanServer", "entityService", "systemRestliEntityClient", "entityRegistry"})
+  @ConditionalOnProperty(name = "entityService.impl", havingValue = "ebean", matchIfMissing = true)
   @Nonnull
   public NoCodeUpgrade createInstance() {
     final Database ebeanServer = applicationContext.getBean(Database.class);
@@ -29,4 +33,12 @@ public NoCodeUpgrade createInstance() {
 
     return new NoCodeUpgrade(ebeanServer, entityService, entityRegistry, entityClient);
   }
+
+  @Bean(name = "noCodeUpgrade")
+  @ConditionalOnProperty(name = "entityService.impl", havingValue = "cassandra")
+  @Nonnull
+  public NoCodeUpgrade createNotImplInstance() {
+    log.warn("NoCode is not supported for cassandra!");
+    return new NoCodeUpgrade(null, null, null, null);
+  }
 }

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreBackupConfig.java

@@ -8,12 +8,15 @@
 import com.linkedin.metadata.search.EntitySearchService;
 import io.ebean.Database;
 import javax.annotation.Nonnull;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.DependsOn;
 
+@Slf4j
 @Configuration
 public class RestoreBackupConfig {
   @Autowired ApplicationContext applicationContext;
@@ -27,6 +30,7 @@ public class RestoreBackupConfig {
     "searchService",
     "entityRegistry"
   })
+  @ConditionalOnProperty(name = "entityService.impl", havingValue = "ebean", matchIfMissing = true)
   @Nonnull
   public RestoreBackup createInstance() {
     final Database ebeanServer = applicationContext.getBean(Database.class);
@@ -40,4 +44,12 @@ public RestoreBackup createInstance() {
     return new RestoreBackup(
         ebeanServer, entityService, entityRegistry, entityClient, graphClient, searchClient);
   }
+
+  @Bean(name = "restoreBackup")
+  @ConditionalOnProperty(name = "entityService.impl", havingValue = "cassandra")
+  @Nonnull
+  public RestoreBackup createNotImplInstance() {
+    log.warn("restoreIndices is not supported for cassandra!");
+    return new RestoreBackup(null, null, null, null, null, null);
+  }
 }

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/RestoreIndicesConfig.java

@@ -7,18 +7,22 @@
 import com.linkedin.metadata.search.EntitySearchService;
 import io.ebean.Database;
 import javax.annotation.Nonnull;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.DependsOn;
 
+@Slf4j
 @Configuration
 public class RestoreIndicesConfig {
   @Autowired ApplicationContext applicationContext;
 
   @Bean(name = "restoreIndices")
   @DependsOn({"ebeanServer", "entityService", "searchService", "graphService", "entityRegistry"})
+  @ConditionalOnProperty(name = "entityService.impl", havingValue = "ebean", matchIfMissing = true)
   @Nonnull
   public RestoreIndices createInstance() {
     final Database ebeanServer = applicationContext.getBean(Database.class);
@@ -31,4 +35,12 @@ public RestoreIndices createInstance() {
     return new RestoreIndices(
         ebeanServer, entityService, entityRegistry, entitySearchService, graphService);
   }
+
+  @Bean(name = "restoreIndices")
+  @ConditionalOnProperty(name = "entityService.impl", havingValue = "cassandra")
+  @Nonnull
+  public RestoreIndices createNotImplInstance() {
+    log.warn("restoreIndices is not supported for cassandra!");
+    return new RestoreIndices(null, null, null, null, null);
+  }
 }

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/NoCodeUpgrade.java

@@ -13,6 +13,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import javax.annotation.Nullable;
 
 public class NoCodeUpgrade implements Upgrade {
 
@@ -26,12 +27,17 @@ public class NoCodeUpgrade implements Upgrade {
 
   // Upgrade requires the Database.
   public NoCodeUpgrade(
-      final Database server,
+      @Nullable final Database server,
       final EntityService entityService,
       final EntityRegistry entityRegistry,
       final SystemRestliEntityClient entityClient) {
-    _steps = buildUpgradeSteps(server, entityService, entityRegistry, entityClient);
-    _cleanupSteps = buildCleanupSteps();
+    if (server != null) {
+      _steps = buildUpgradeSteps(server, entityService, entityRegistry, entityClient);
+      _cleanupSteps = buildCleanupSteps();
+    } else {
+      _steps = List.of();
+      _cleanupSteps = List.of();
+    }
   }
 
   @Override