Merge branch 'datahub-project:master' into master

datahub-web-react/src/app/entity/ermodelrelationships/preview/ERModelRelationshipAction.less

@@ -1,5 +1,3 @@
-@import '../../../../../node_modules/antd/dist/antd.less';
-
 .joinName {
     width: 385px;
     height: 24px;

datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/CreateERModelRelationModal.less

@@ -1,5 +1,3 @@
-@import '../../../../../../../node_modules/antd/dist/antd.less';
-
 .CreateERModelRelationModal {
     .ermodelrelation-name {
         padding: 8px 16px;

datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/ERModelRelationPreview.less

@@ -1,5 +1,3 @@
-@import '../../../../../../../node_modules/antd/dist/antd.less';
-
 .ERModelRelationPreview {
     .preview-main-div {
         display: flex;

datahub-web-react/src/app/entity/shared/tabs/Dataset/Relationship/RelationshipsTab.less

@@ -1,5 +1,3 @@
-@import '../../../../../../../node_modules/antd/dist/antd.less';
-
 .RelationshipsTab {
     .add-btn-link {
         height: 56px !important;

datahub-web-react/src/app/entity/shared/tabs/ERModelRelationship/ERModelRelationshipTab.less

@@ -1,5 +1,3 @@
-@import '../../../../../../node_modules/antd/dist/antd.less';
-
 .ERModelRelationTab {
     .add-btn-link {
         padding-left: 1155px !important;

datahub-web-react/src/app/ingest/ManageIngestionPage.tsx

@@ -2,6 +2,8 @@ import { Tabs, Typography } from 'antd';
 import React, { useState } from 'react';
 import styled from 'styled-components';
 import { IngestionSourceList } from './source/IngestionSourceList';
+import { useAppConfig } from '../useAppConfig';
+import { useUserContext } from '../context/useUserContext';
 import { SecretsList } from './secret/SecretsList';
 import { OnboardingTour } from '../onboarding/OnboardingTour';
 import {
@@ -48,7 +50,13 @@ export const ManageIngestionPage = () => {
     /**
      * Determines which view should be visible: ingestion sources or secrets.
      */
-    const [selectedTab, setSelectedTab] = useState<TabType>(TabType.Sources);
+    const me = useUserContext();
+    const { config } = useAppConfig();
+    const isIngestionEnabled = config?.managedIngestionConfig.enabled;
+    const showIngestionTab = isIngestionEnabled && me && me.platformPrivileges?.manageIngestion;
+    const showSecretsTab = isIngestionEnabled && me && me.platformPrivileges?.manageSecrets;
+    const defaultTab = showIngestionTab ? TabType.Sources : TabType.Secrets;
+    const [selectedTab, setSelectedTab] = useState<TabType>(defaultTab);
 
     const onClickTab = (newTab: string) => {
         setSelectedTab(TabType[newTab]);
@@ -64,8 +72,8 @@ export const ManageIngestionPage = () => {
                 </Typography.Paragraph>
             </PageHeaderContainer>
             <StyledTabs activeKey={selectedTab} size="large" onTabClick={(tab: string) => onClickTab(tab)}>
-                <Tab key={TabType.Sources} tab={TabType.Sources} />
-                <Tab key={TabType.Secrets} tab={TabType.Secrets} />
+                {showIngestionTab && <Tab key={TabType.Sources} tab={TabType.Sources} />}
+                {showSecretsTab && <Tab key={TabType.Secrets} tab={TabType.Secrets} />}
             </StyledTabs>
             <ListContainer>{selectedTab === TabType.Sources ? <IngestionSourceList /> : <SecretsList />}</ListContainer>
         </PageContainer>

datahub-web-react/src/app/ingest/secret/SecretsList.tsx

@@ -118,7 +118,7 @@ export const SecretsList = () => {
                     {
                         urn: res.data?.createSecret || '',
                         name: state.name,
-                        description: state.description,
+                        description: state.description || '',
                     },
                     client,
                     pageSize,
@@ -127,7 +127,7 @@ export const SecretsList = () => {
             .catch((e) => {
                 message.destroy();
                 message.error({
-                    content: `Failed to update ingestion source!: \n ${e.message || ''}`,
+                    content: `Failed to update secret!: \n ${e.message || ''}`,
                     duration: 3,
                 });
             });

datahub-web-react/src/app/shared/admin/HeaderLinks.tsx

@@ -75,7 +75,7 @@ export function HeaderLinks(props: Props) {
     const showAnalytics = (isAnalyticsEnabled && me && me?.platformPrivileges?.viewAnalytics) || false;
     const showSettings = true;
     const showIngestion =
-        isIngestionEnabled && me && me.platformPrivileges?.manageIngestion && me.platformPrivileges?.manageSecrets;
+        isIngestionEnabled && me && (me.platformPrivileges?.manageIngestion || me.platformPrivileges?.manageSecrets);
 
     useToggleEducationStepIdsAllowList(!!showIngestion, HOME_PAGE_INGESTION_ID);
 

datahub-web-react/src/app/shared/sidebar/components.tsx

@@ -8,6 +8,8 @@ export const SidebarWrapper = styled.div<{ width: number }>`
     width: ${(props) => props.width}px;
     min-width: ${(props) => props.width}px;
     display: ${(props) => (props.width ? 'block' : 'none')};
+    display: flex;
+    flex-direction: column;
 `;
 
 export function RotatingTriangle({ isOpen, onClick }: { isOpen: boolean; onClick?: () => void }) {

docs-website/graphql/generateGraphQLSchema.sh

@@ -1,22 +1,8 @@
 #!/bin/sh
-rm combined.graphql
+if [ -f "combined.graphql" ] ; then
+    rm "combined.graphql"
+fi
 touch combined.graphql
 echo "Generating combined GraphQL schema..."
 echo "# Auto Generated During Docs Build" >> combined.graphql
-cat ../../datahub-graphql-core/src/main/resources/actions.graphql >> combined.graphql
-cat ../../datahub-graphql-core/src/main/resources/analytics.graphql >> combined.graphql
-cat ../../datahub-graphql-core/src/main/resources/app.graphql >> combined.graphql
-cat ../../datahub-graphql-core/src/main/resources/auth.graphql >> combined.graphql
-cat ../../datahub-graphql-core/src/main/resources/constraints.graphql >> combined.graphql
-cat ../../datahub-graphql-core/src/main/resources/entity.graphql >> combined.graphql
-cat ../../datahub-graphql-core/src/main/resources/assertions.graphql >> combined.graphql
-cat ../../datahub-graphql-core/src/main/resources/ingestion.graphql >> combined.graphql
-cat ../../datahub-graphql-core/src/main/resources/recommendation.graphql >> combined.graphql
-cat ../../datahub-graphql-core/src/main/resources/search.graphql >> combined.graphql
-cat ../../datahub-graphql-core/src/main/resources/tests.graphql >> combined.graphql
-cat ../../datahub-graphql-core/src/main/resources/timeline.graphql >> combined.graphql
-cat ../../datahub-graphql-core/src/main/resources/step.graphql >> combined.graphql
-cat ../../datahub-graphql-core/src/main/resources/lineage.graphql >> combined.graphql
-cat ../../datahub-graphql-core/src/main/resources/properties.graphql >> combined.graphql
-cat ../../datahub-graphql-core/src/main/resources/forms.graphql >> combined.graphql
-cat ../../datahub-graphql-core/src/main/resources/connection.graphql >> combined.graphql
+cat ../../datahub-graphql-core/src/main/resources/*.graphql >> combined.graphql

docs/how/updating-datahub.md

@@ -22,6 +22,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 
 - Protobuf CLI will no longer create binary encoded protoc custom properties. Flag added `-protocProp` in case this 
   behavior is required.
+- #10814 Data flow info and data job info aspect will produce an additional field that will require a corresponding upgrade of server. Otherwise server can reject the aspects.
 - #10868 - OpenAPI V3 - Creation of aspects will need to be wrapped within a `value` key and the API is now symmetric with respect to input and outputs.
 
 Example Global Tags Aspect:

metadata-ingestion-modules/airflow-plugin/setup.py

@@ -73,7 +73,7 @@ def get_long_description():
     "flake8>=3.8.3",
     "flake8-tidy-imports>=4.3.0",
     "isort>=5.7.0",
-    "mypy>=1.4.0",
+    "mypy==1.10.1",
     # pydantic 1.8.2 is incompatible with mypy 0.910.
     # See https://github.com/samuelcolvin/pydantic/pull/3175#issuecomment-995382910.
     "pydantic>=1.10",

metadata-ingestion/setup.py

@@ -330,7 +330,14 @@
     # sqlalchemy-bigquery is included here since it provides an implementation of
     # a SQLalchemy-conform STRUCT type definition
     "athena": sql_common
-    | {"PyAthena[SQLAlchemy]>=2.6.0,<3.0.0", "sqlalchemy-bigquery>=1.4.1"},
+    # We need to set tenacity lower than 8.4.0 as
+    # this version has missing dependency asyncio
+    # https://github.com/jd/tenacity/issues/471
+    | {
+        "PyAthena[SQLAlchemy]>=2.6.0,<3.0.0",
+        "sqlalchemy-bigquery>=1.4.1",
+        "tenacity!=8.4.0",
+    },
     "azure-ad": set(),
     "bigquery": sql_common
     | bigquery_common

metadata-ingestion/src/datahub/api/entities/datajob/dataflow.py

@@ -3,6 +3,7 @@
 from typing import Callable, Dict, Iterable, List, Optional, Set, cast
 
 import datahub.emitter.mce_builder as builder
+from datahub.configuration.source_common import ALL_ENV_TYPES
 from datahub.emitter.generic_emitter import Emitter
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.metadata.schema_classes import (
@@ -110,7 +111,20 @@ def generate_tags_aspect(self) -> List[GlobalTagsClass]:
         )
         return [tags]
 
+    def _get_env(self) -> Optional[str]:
+        env: Optional[str] = None
+        if self.cluster in ALL_ENV_TYPES:
+            env = self.cluster
+        elif self.env in ALL_ENV_TYPES:
+            env = self.env
+        else:
+            logger.warning(
+                f"cluster {self.cluster} and {self.env} is not a valid environment type so Environment filter won't work."
+            )
+        return env
+
     def generate_mce(self) -> MetadataChangeEventClass:
+        env = self._get_env()
         flow_mce = MetadataChangeEventClass(
             proposedSnapshot=DataFlowSnapshotClass(
                 urn=str(self.urn),
@@ -120,6 +134,7 @@ def generate_mce(self) -> MetadataChangeEventClass:
                         description=self.description,
                         customProperties=self.properties,
                         externalUrl=self.url,
+                        env=env,
                     ),
                     *self.generate_ownership_aspect(),
                     *self.generate_tags_aspect(),
@@ -130,13 +145,15 @@ def generate_mce(self) -> MetadataChangeEventClass:
         return flow_mce
 
     def generate_mcp(self) -> Iterable[MetadataChangeProposalWrapper]:
+        env = self._get_env()
         mcp = MetadataChangeProposalWrapper(
             entityUrn=str(self.urn),
             aspect=DataFlowInfoClass(
                 name=self.name if self.name is not None else self.id,
                 description=self.description,
                 customProperties=self.properties,
                 externalUrl=self.url,
+                env=env,
             ),
         )
         yield mcp

metadata-ingestion/src/datahub/api/entities/datajob/datajob.py

@@ -1,7 +1,9 @@
+import logging
 from dataclasses import dataclass, field
 from typing import Callable, Dict, Iterable, List, Optional, Set
 
 import datahub.emitter.mce_builder as builder
+from datahub.configuration.source_common import ALL_ENV_TYPES
 from datahub.emitter.generic_emitter import Emitter
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.metadata.schema_classes import (
@@ -22,6 +24,8 @@
 from datahub.utilities.urns.data_job_urn import DataJobUrn
 from datahub.utilities.urns.dataset_urn import DatasetUrn
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass
 class DataJob:
@@ -103,6 +107,13 @@ def generate_tags_aspect(self) -> Iterable[GlobalTagsClass]:
     def generate_mcp(
         self, materialize_iolets: bool = True
     ) -> Iterable[MetadataChangeProposalWrapper]:
+        env: Optional[str] = None
+        if self.flow_urn.cluster in ALL_ENV_TYPES:
+            env = self.flow_urn.cluster
+        else:
+            logger.warning(
+                f"cluster {self.flow_urn.cluster} is not a valid environment type so Environment filter won't work."
+            )
         mcp = MetadataChangeProposalWrapper(
             entityUrn=str(self.urn),
             aspect=DataJobInfoClass(
@@ -111,6 +122,7 @@ def generate_mcp(
                 description=self.description,
                 customProperties=self.properties,
                 externalUrl=self.url,
+                env=env,
             ),
         )
         yield mcp