Merge branch 'datahub-project:master' into master

acryldata · Apr 19, 2024 · 0da6cee · 0da6cee
2 parents 733c403 + bf8e3a9
commit 0da6cee
Show file tree

Hide file tree

Showing 76 changed files with 4,220 additions and 2,454 deletions.
diff --git a/build.gradle b/build.gradle
@@ -172,6 +172,7 @@ project.ext.externalDependency = [
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
     'jsonSmart': 'net.minidev:json-smart:2.4.9',
     'json': 'org.json:json:20231013',
+    'jsonSchemaValidator': 'com.github.java-json-tools:json-schema-validator:2.2.14',
     'junit': 'junit:junit:4.13.2',
     'junitJupiterApi': "org.junit.jupiter:junit-jupiter-api:$junitJupiterVersion",
     'junitJupiterParams': "org.junit.jupiter:junit-jupiter-params:$junitJupiterVersion",

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
@@ -14,10 +14,11 @@ dependencies {
    * Other companies are also separately maintaining forks (like: https://github.com/java-json-tools/json-schema-avro).
    * We have built several customizations on top of it for various bug fixes, especially around union scheams
    */
-  implementation('io.acryl:json-schema-avro:0.2.2') {
+  implementation('io.acryl:json-schema-avro:0.2.3') {
     exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind'
     exclude group: 'com.google.guava', module: 'guava'
   }
+  implementation 'com.github.java-json-tools:json-schema-validator:2.2.14'
   implementation 'com.google.guava:guava:32.1.2-jre'
   implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.5'
   implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.5'

diff --git a/datahub-web-react/src/app/entity/dataFlow/DataFlowEntity.tsx b/datahub-web-react/src/app/entity/dataFlow/DataFlowEntity.tsx
@@ -81,6 +81,9 @@ export class DataFlowEntity implements Entity<DataFlow> {
                 {
                     name: 'Tasks',
                     component: DataFlowJobsTab,
+                    properties: {
+                        urn,
+                    },
                 },
                 {
                     name: 'Incidents',

diff --git a/datahub-web-react/src/app/entity/shared/tabs/Entity/DataFlowJobsTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Entity/DataFlowJobsTab.tsx
@@ -1,19 +1,60 @@
-import React from 'react';
-import { useBaseEntity } from '../../EntityContext';
+import React, { useState } from 'react';
 import { EntityType } from '../../../../../types.generated';
 import { EntityList } from './components/EntityList';
 import { useEntityRegistry } from '../../../../useEntityRegistry';
+import { useGetDataFlowChildJobsQuery } from '../../../../../graphql/dataFlow.generated';
+import { SearchCfg } from '../../../../../conf';
 
-export const DataFlowJobsTab = () => {
-    const entity = useBaseEntity() as any;
-    const dataFlow = entity && entity.dataFlow;
+interface Props {
+    properties?: {
+        urn: string;
+    };
+}
+
+export const DataFlowJobsTab = ({ properties = { urn: '' } }: Props) => {
+    const [page, setPage] = useState(1);
+    const [numResultsPerPage, setNumResultsPerPage] = useState(SearchCfg.RESULTS_PER_PAGE);
+
+    const start: number = (page - 1) * numResultsPerPage;
+
+    const { data, loading, error } = useGetDataFlowChildJobsQuery({
+        variables: {
+            urn: properties.urn,
+            start,
+            count: numResultsPerPage,
+        },
+    });
+
+    const onChangePage = (newPage: number) => {
+        setPage(newPage);
+    };
+
+    const dataFlow = data && data?.dataFlow;
     const dataJobs = dataFlow?.childJobs?.relationships.map((relationship) => relationship.entity);
     const entityRegistry = useEntityRegistry();
     const totalJobs = dataFlow?.childJobs?.total || 0;
+    const pageSize = data?.dataFlow?.childJobs?.count || 0;
+    const pageStart = data?.dataFlow?.childJobs?.start || 0;
+    const lastResultIndex = pageStart + pageSize > totalJobs ? totalJobs : pageStart + pageSize;
     const title = `Contains ${totalJobs} ${
         totalJobs === 1
             ? entityRegistry.getEntityName(EntityType.DataJob)
             : entityRegistry.getCollectionName(EntityType.DataJob)
     }`;
-    return <EntityList title={title} type={EntityType.DataJob} entities={dataJobs || []} />;
+    return (
+        <EntityList
+            title={title}
+            type={EntityType.DataJob}
+            entities={dataJobs || []}
+            showPagination
+            loading={loading}
+            error={error}
+            totalAssets={totalJobs}
+            page={page}
+            pageSize={numResultsPerPage}
+            lastResultIndex={lastResultIndex}
+            onChangePage={onChangePage}
+            setNumResultsPerPage={setNumResultsPerPage}
+        />
+    );
 };
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Entity/components/EntityList.tsx b/datahub-web-react/src/app/entity/shared/tabs/Entity/components/EntityList.tsx
@@ -1,9 +1,27 @@
 import React from 'react';
-import { List } from 'antd';
+import { List, Pagination, Typography } from 'antd';
 import styled from 'styled-components';
 import { useEntityRegistry } from '../../../../../useEntityRegistry';
 import { PreviewType } from '../../../../Entity';
 import { EntityType } from '../../../../../../types.generated';
+import { SearchCfg } from '../../../../../../conf';
+import { Message } from '../../../../../shared/Message';
+
+const ScrollWrapper = styled.div`
+    overflow: auto;
+    height: 100%;
+
+    &::-webkit-scrollbar {
+        height: 12px;
+        width: 5px;
+        background: #f2f2f2;
+    }
+    &::-webkit-scrollbar-thumb {
+        background: #cccccc;
+        -webkit-border-radius: 1ex;
+        -webkit-box-shadow: 0px 1px 2px rgba(0, 0, 0, 0.75);
+    }
+`;
 
 const StyledList = styled(List)`
     padding-left: 40px;
@@ -28,22 +46,94 @@ const StyledListItem = styled(List.Item)`
     padding-top: 20px;
 `;
 
+const PaginationInfoContainer = styled.span`
+    padding: 8px;
+    padding-left: 16px;
+    border-top: 1px solid;
+    border-color: ${(props) => props.theme.styles['border-color-base']};
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+`;
+
+const StyledPagination = styled(Pagination)`
+    padding: 12px 12px 12px 12px;
+    width: 100%;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+`;
+
+const PaginationInfo = styled(Typography.Text)`
+    padding: 0px;
+    width: 20%;
+`;
+
 type EntityListProps = {
     type: EntityType;
     entities: Array<any>;
     title?: string;
+    totalAssets?: number;
+    pageSize?: any;
+    page?: number;
+    lastResultIndex?: any;
+    showPagination?: boolean;
+    loading?: boolean;
+    error?: any;
+    onChangePage?: (number: any) => void;
+    setNumResultsPerPage?: (number: any) => void;
 };
 
-export const EntityList = ({ type, entities, title }: EntityListProps) => {
+export const EntityList = ({
+    type,
+    entities,
+    title,
+    totalAssets,
+    pageSize,
+    page,
+    lastResultIndex,
+    showPagination = false,
+    loading = false,
+    error = undefined,
+    onChangePage,
+    setNumResultsPerPage,
+}: EntityListProps) => {
     const entityRegistry = useEntityRegistry();
+
     return (
-        <StyledList
-            bordered
-            dataSource={entities}
-            header={title || `${entities.length || 0} ${entityRegistry.getCollectionName(type)}`}
-            renderItem={(item) => (
-                <StyledListItem>{entityRegistry.renderPreview(type, PreviewType.PREVIEW, item)}</StyledListItem>
+        <>
+            <ScrollWrapper>
+                <StyledList
+                    bordered
+                    dataSource={entities}
+                    header={title || `${entities.length || 0} ${entityRegistry.getCollectionName(type)}`}
+                    renderItem={(item) => (
+                        <StyledListItem>{entityRegistry.renderPreview(type, PreviewType.PREVIEW, item)}</StyledListItem>
+                    )}
+                />
+            </ScrollWrapper>
+            {loading && <Message type="loading" content="Loading..." style={{ marginTop: '10%' }} />}
+            {error && <Message type="error" content="Failed to load results! An unexpected error occurred." />}
+            {showPagination && (
+                <PaginationInfoContainer>
+                    <PaginationInfo>
+                        <b>
+                            {lastResultIndex > 0 ? ((page as number) - 1) * pageSize + 1 : 0} - {lastResultIndex}
+                        </b>{' '}
+                        of <b>{totalAssets}</b>
+                    </PaginationInfo>
+                    <StyledPagination
+                        current={page}
+                        pageSize={pageSize}
+                        total={totalAssets}
+                        showLessItems
+                        onChange={onChangePage}
+                        showSizeChanger={(totalAssets as any) > SearchCfg.RESULTS_PER_PAGE}
+                        onShowSizeChange={(_currNum, newNum) => setNumResultsPerPage?.(newNum)}
+                        pageSizeOptions={['10', '20', '50', '100']}
+                    />
+                </PaginationInfoContainer>
             )}
-        />
+        </>
     );
 };
diff --git a/datahub-web-react/src/graphql/dataFlow.graphql b/datahub-web-react/src/graphql/dataFlow.graphql
@@ -70,7 +70,24 @@ query getDataFlow($urn: String!) {
         downstream: lineage(input: { direction: DOWNSTREAM, start: 0, count: 100 }) {
             ...partialLineageResults
         }
-        childJobs: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 100 }) {
+        autoRenderAspects: aspects(input: { autoRenderOnly: true }) {
+            ...autoRenderAspectFields
+        }
+        structuredProperties {
+            properties {
+                ...structuredPropertiesFields
+            }
+        }
+        forms {
+            ...formsFields
+        }
+    }
+}
+
+query getDataFlowChildJobs($urn: String!, $start: Int, $count: Int) {
+    dataFlow(urn: $urn) {
+        ...dataFlowFields
+        childJobs: relationships(input: { types: ["IsPartOf"], direction: INCOMING, start: $start, count: $count }) {
             start
             count
             total
@@ -111,17 +128,6 @@ query getDataFlow($urn: String!) {
                 }
             }
         }
-        autoRenderAspects: aspects(input: { autoRenderOnly: true }) {
-            ...autoRenderAspectFields
-        }
-        structuredProperties {
-            properties {
-                ...structuredPropertiesFields
-            }
-        }
-        forms {
-            ...formsFields
-        }
     }
 }
 

diff --git a/docker/kafka-setup/Dockerfile b/docker/kafka-setup/Dockerfile
@@ -22,7 +22,7 @@ ARG ALPINE_REPO_URL
 ARG APACHE_DOWNLOAD_URL
 ARG GITHUB_REPO_URL
 
-ENV KAFKA_VERSION 3.5.2
+ENV KAFKA_VERSION 3.7.0
 ENV SCALA_VERSION 2.13
 
 LABEL name="kafka" version=${KAFKA_VERSION}

diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js
@@ -273,6 +273,13 @@ module.exports = {
       "@docusaurus/preset-classic",
       {
         docs: {
+            lastVersion: "current",
+              versions: {
+                current: {
+                  label: "Next",
+                  banner: 'none',
+                },
+              },
           path: "genDocs",
           sidebarPath: require.resolve("./sidebars.js"),
           ...(!isSaas && {
@@ -306,6 +313,19 @@ module.exports = {
     ],
   ],
   plugins: [
+    [
+      '@docusaurus/plugin-client-redirects',
+      {
+        createRedirects(existingPath) {
+          if (existingPath.includes('/docs')) {
+            return [
+              existingPath.replace('/docs', '/docs/next'),
+            ];
+          }
+          return undefined; // Return a falsy value: no redirect created
+        },
+      },
+    ],
     ["@docusaurus/plugin-ideal-image", { quality: 100, sizes: [320, 640, 1280, 1440, 1600] }],
     "docusaurus-plugin-sass",
     [

diff --git a/docs-website/filterTagIndexes.json b/docs-website/filterTagIndexes.json
@@ -430,9 +430,9 @@
       }
     },
     {
-      "Path": "docs/generated/ingestion/sources/presto-on-hive",
+      "Path": "docs/generated/ingestion/sources/hive-metastore",
       "imgPath": "img/logos/platforms/presto.svg",
-      "Title": "Presto on Hive",
+      "Title": "Hive Metastore",
       "Description": "Presto on Hive is a data tool that allows users to query and analyze large datasets stored in Hive using SQL-like syntax.",
       "tags": {
         "Platform Type": "Datastore",

diff --git a/docs-website/package.json b/docs-website/package.json
@@ -25,6 +25,7 @@
   "dependencies": {
     "@ant-design/icons": "^4.7.0",
     "@docusaurus/core": "^2.4.1",
+    "@docusaurus/plugin-client-redirects": "2.4.3",
     "@docusaurus/plugin-content-docs": "2.4.1",
     "@docusaurus/plugin-ideal-image": "^2.4.1",
     "@docusaurus/preset-classic": "^2.4.1",

diff --git a/docs-website/src/pages/docs/index.js b/docs-website/src/pages/docs/index.js
@@ -2,7 +2,7 @@ import React from 'react';
 import { Redirect } from '@docusaurus/router';
 
 const Home = () => {
-  return <Redirect to="/docs/next" />;
+  return <Redirect to="/docs/features" />;
 };
 
 export default Home;