Merge branch 'datahub-project:master' into master

acryldata · Feb 6, 2024 · c51122e · c51122e
2 parents 43e8eed + 92850ac
commit c51122e
Show file tree

Hide file tree

Showing 282 changed files with 9,979 additions and 2,306 deletions.
diff --git a/.github/scripts/check_python_package.py b/.github/scripts/check_python_package.py
@@ -0,0 +1,18 @@
+import setuptools
+
+folders = ["./smoke-test/tests"]
+
+for folder in folders:
+    print(f"Checking folder {folder}")
+    a = [i for i in setuptools.find_packages(folder) if "cypress" not in i]
+    b = [i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i]
+
+    in_a_not_b = set(a) - set(b)
+    in_b_not_a = set(b) - set(a)
+
+    assert (
+        len(in_a_not_b) == 0
+    ), f"Found packages in {folder} that are not in namespace packages: {in_a_not_b}"
+    assert (
+        len(in_b_not_a) == 0
+    ), f"Found namespace packages in {folder} that are not in packages: {in_b_not_a}"
diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -43,8 +43,29 @@ env:
   #### IMPORTANT ####
 
 jobs:
+  check_lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: hsheth2/sane-checkout-action@v1
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: "zulu"
+          java-version: 17
+      - name: Ensure packages are correct
+        run: |
+          python ./.github/scripts/check_python_package.py
+      - name: Run lint on smoke test
+        run: |
+          ./gradlew :smoke-test:lint
   setup:
     runs-on: ubuntu-latest
+    needs: check_lint
     outputs:
       tag: ${{ steps.tag.outputs.tag }}
       slim_tag: ${{ steps.tag.outputs.slim_tag }}

diff --git a/datahub-frontend/build.gradle b/datahub-frontend/build.gradle
@@ -87,7 +87,7 @@ docker {
   }
 }
 
-task unversionZip(type: Copy, dependsOn: [':datahub-web-react:build', dist]) {
+task unversionZip(type: Copy, dependsOn: [':datahub-web-react:distZip', dist]) {
   from ("${buildDir}/distributions")
   include "datahub-frontend-${version}.zip"
   into "${buildDir}/docker/"

diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
@@ -92,6 +92,7 @@
 import com.linkedin.datahub.graphql.generated.QuerySubject;
 import com.linkedin.datahub.graphql.generated.QuickFilter;
 import com.linkedin.datahub.graphql.generated.RecommendationContent;
+import com.linkedin.datahub.graphql.generated.ResolvedAuditStamp;
 import com.linkedin.datahub.graphql.generated.SchemaField;
 import com.linkedin.datahub.graphql.generated.SchemaFieldEntity;
 import com.linkedin.datahub.graphql.generated.SearchAcrossLineageResult;
@@ -1642,7 +1643,7 @@ private void configureResolvedAuditStampResolvers(final RuntimeWiring.Builder bu
             typeWiring.dataFetcher(
                 "actor",
                 new LoadableTypeResolver<>(
-                    corpUserType, (env) -> ((CorpUser) env.getSource()).getUrn())));
+                    corpUserType, (env) -> ((ResolvedAuditStamp) env.getSource()).getActor().getUrn())));
   }
 
   /**

diff --git a/datahub-graphql-core/src/main/resources/forms.graphql b/datahub-graphql-core/src/main/resources/forms.graphql
@@ -275,20 +275,6 @@ input SubmitFormPromptInput {
     structuredPropertyParams: StructuredPropertyInputParams
 }
 
-"""
-Input for responding to a singular prompt in a form for a batch of entities
-"""
-input BatchSubmitFormPromptInput {
-    """
-    The urns of the entities this prompt submission is for
-    """
-    assetUrns: [String!]!
-
-    """
-    Input for responding to a specific prompt on a form
-    """
-    input: SubmitFormPromptInput
-}
 
 """
 Input for collecting structured property values to apply to entities
@@ -390,18 +376,3 @@ input VerifyFormInput {
     """
     entityUrn: String!
 }
-
-"""
-Input for verifying a batch of entities for a give form
-"""
-input BatchVerifyFormInput {
-    """
-    The urns of the entities getting verified for this form
-    """
-    assetUrns: [String!]!
-
-    """
-    The urn of the form being verified on the given entities
-    """
-    formUrn: String!
-}
diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/DataMigrationStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/DataMigrationStep.java
@@ -10,6 +10,7 @@
 import com.linkedin.datahub.upgrade.UpgradeStepResult;
 import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult;
 import com.linkedin.metadata.Constants;
+import com.linkedin.metadata.aspect.utils.DefaultAspectsUtil;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.entity.ebean.EbeanAspectV1;
 import com.linkedin.metadata.entity.ebean.EbeanAspectV2;
@@ -170,7 +171,7 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
             // Emit a browse path aspect.
             final BrowsePaths browsePaths;
             try {
-              browsePaths = _entityService.buildDefaultBrowsePath(urn);
+              browsePaths = DefaultAspectsUtil.buildDefaultBrowsePath(urn, _entityService);
 
               final AuditStamp browsePathsStamp = new AuditStamp();
               browsePathsStamp.setActor(Urn.createFromString(Constants.SYSTEM_ACTOR));

diff --git a/...main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2Step.java b/...main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2Step.java
@@ -15,6 +15,7 @@
 import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult;
 import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.Constants;
+import com.linkedin.metadata.aspect.utils.DefaultAspectsUtil;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.query.SearchFlags;
 import com.linkedin.metadata.query.filter.Condition;
@@ -181,7 +182,8 @@ private Filter backfillDefaultBrowsePathsV2Filter() {
   }
 
   private void ingestBrowsePathsV2(Urn urn, AuditStamp auditStamp) throws Exception {
-    BrowsePathsV2 browsePathsV2 = _entityService.buildDefaultBrowsePathV2(urn, true);
+    BrowsePathsV2 browsePathsV2 =
+        DefaultAspectsUtil.buildDefaultBrowsePathV2(urn, true, _entityService);
     log.debug(String.format("Adding browse path v2 for urn %s with value %s", urn, browsePathsV2));
     MetadataChangeProposal proposal = new MetadataChangeProposal();
     proposal.setEntityUrn(urn);

diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json
@@ -65,6 +65,7 @@
         "react-helmet-async": "^1.3.0",
         "react-highlighter": "^0.4.3",
         "react-icons": "4.3.1",
+        "react-intersection-observer": "^9.5.3",
         "react-js-cron": "^2.1.0",
         "react-router": "^5.3",
         "react-router-dom": "^5.3",

diff --git a/datahub-web-react/src/Mocks.tsx b/datahub-web-react/src/Mocks.tsx
@@ -298,6 +298,7 @@ export const dataset1 = {
     browsePathV2: { path: [{ name: 'test', entity: null }], __typename: 'BrowsePathV2' },
     autoRenderAspects: [],
     structuredProperties: null,
+    forms: null,
 };
 
 export const dataset2 = {
@@ -395,6 +396,7 @@ export const dataset2 = {
     browsePathV2: { path: [{ name: 'test', entity: null }], __typename: 'BrowsePathV2' },
     autoRenderAspects: [],
     structuredProperties: null,
+    forms: null,
 };
 
 export const dataset3 = {
@@ -629,6 +631,7 @@ export const dataset3 = {
     lastProfile: null,
     lastOperation: null,
     structuredProperties: null,
+    forms: null,
 } as Dataset;
 
 export const dataset3WithSchema = {

diff --git a/datahub-web-react/src/app/entity/Entity.tsx b/datahub-web-react/src/app/entity/Entity.tsx
@@ -1,6 +1,6 @@
 import { EntityType, SearchResult } from '../../types.generated';
 import { FetchedEntity } from '../lineage/types';
-import { GenericEntityProperties } from './shared/types';
+import { EntitySidebarSection, GenericEntityProperties } from './shared/types';
 
 export enum PreviewType {
     /**
@@ -176,4 +176,9 @@ export interface Entity<T> {
      * Returns the profile component to be displayed in our Chrome extension
      */
     renderEmbeddedProfile?: (urn: string) => JSX.Element;
+
+    /**
+     * Returns the entity profile sidebar sections for an entity type. Only implemented on Datasets for now.
+     */
+    getSidebarSections?: () => EntitySidebarSection[];
 }
diff --git a/datahub-web-react/src/app/entity/EntityRegistry.tsx b/datahub-web-react/src/app/entity/EntityRegistry.tsx
@@ -4,7 +4,7 @@ import { FetchedEntity } from '../lineage/types';
 import { SearchResultProvider } from '../search/context/SearchResultContext';
 import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from './Entity';
 import { GLOSSARY_ENTITY_TYPES } from './shared/constants';
-import { GenericEntityProperties } from './shared/types';
+import { EntitySidebarSection, GenericEntityProperties } from './shared/types';
 import { dictToQueryStringParams, getFineGrainedLineageWithSiblings, urlEncodeUrn } from './shared/utils';
 
 function validatedGet<K, V>(key: K, map: Map<K, V>): V {
@@ -194,6 +194,11 @@ export default class EntityRegistry {
         return entity.displayName(data);
     }
 
+    getSidebarSections(type: EntityType): EntitySidebarSection[] {
+        const entity = validatedGet(type, this.entityTypeToEntity);
+        return entity.getSidebarSections ? entity.getSidebarSections() : [];
+    }
+
     getGenericEntityProperties<T>(type: EntityType, data: T): GenericEntityProperties | null {
         const entity = validatedGet(type, this.entityTypeToEntity);
         return entity.getGenericEntityProperties(data);

diff --git a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx
@@ -190,51 +190,51 @@ export class DatasetEntity implements Entity<Dataset> {
                     },
                 },
             ]}
-            sidebarSections={[
-                {
-                    component: SidebarAboutSection,
-                },
-                {
-                    component: SidebarOwnerSection,
-                    properties: {
-                        defaultOwnerType: OwnershipType.TechnicalOwner,
-                    },
-                },
-                {
-                    component: SidebarSiblingsSection,
-                    display: {
-                        visible: (_, dataset: GetDatasetQuery) =>
-                            (dataset?.dataset?.siblings?.siblings?.length || 0) > 0,
-                    },
-                },
-                {
-                    component: SidebarViewDefinitionSection,
-                    display: {
-                        visible: (_, dataset: GetDatasetQuery) =>
-                            (dataset?.dataset?.viewProperties?.logic && true) || false,
-                    },
-                },
-                {
-                    component: SidebarTagsSection,
-                    properties: {
-                        hasTags: true,
-                        hasTerms: true,
-                    },
-                },
-                {
-                    component: SidebarDomainSection,
-                },
-                {
-                    component: DataProductSection,
-                },
-                // TODO: Add back once entity-level recommendations are complete.
-                // {
-                //    component: SidebarRecommendationsSection,
-                // },
-            ]}
+            sidebarSections={this.getSidebarSections()}
         />
     );
 
+    getSidebarSections = () => [
+        {
+            component: SidebarAboutSection,
+        },
+        {
+            component: SidebarOwnerSection,
+            properties: {
+                defaultOwnerType: OwnershipType.TechnicalOwner,
+            },
+        },
+        {
+            component: SidebarSiblingsSection,
+            display: {
+                visible: (_, dataset: GetDatasetQuery) => (dataset?.dataset?.siblings?.siblings?.length || 0) > 0,
+            },
+        },
+        {
+            component: SidebarViewDefinitionSection,
+            display: {
+                visible: (_, dataset: GetDatasetQuery) => (dataset?.dataset?.viewProperties?.logic && true) || false,
+            },
+        },
+        {
+            component: SidebarTagsSection,
+            properties: {
+                hasTags: true,
+                hasTerms: true,
+            },
+        },
+        {
+            component: SidebarDomainSection,
+        },
+        {
+            component: DataProductSection,
+        },
+        // TODO: Add back once entity-level recommendations are complete.
+        // {
+        //    component: SidebarRecommendationsSection,
+        // },
+    ];
+
     getOverridePropertiesFromEntity = (dataset?: Dataset | null): GenericEntityProperties => {
         // if dataset has subTypes filled out, pick the most specific subtype and return it
         const subTypes = dataset?.subTypes;

diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx
@@ -45,6 +45,7 @@ import {
 import { useAppConfig } from '../../../../useAppConfig';
 import { useUpdateDomainEntityDataOnChange } from '../../../../domain/utils';
 import ProfileSidebar from './sidebar/ProfileSidebar';
+import SidebarFormInfoWrapper from './sidebar/FormInfo/SidebarFormInfoWrapper';
 
 type Props<T, U> = {
     urn: string;
@@ -333,7 +334,10 @@ export const EntityProfile = <T, U>({
                                         </TabContent>
                                     </HeaderAndTabsFlex>
                                 </HeaderAndTabs>
-                                <ProfileSidebar sidebarSections={sidebarSections} />
+                                <ProfileSidebar
+                                    sidebarSections={sidebarSections}
+                                    topSection={{ component: SidebarFormInfoWrapper }}
+                                />
                             </>
                         )}
                     </ContentContainer>