Merge branch 'datahub-project:master' into master

.github/workflows/metadata-model.yml

@@ -37,6 +37,19 @@ jobs:
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Run model generation
         run: ./gradlew :metadata-models:build
+      - name: Generate metadata files
+        if: ${{ needs.setup.outputs.publish == 'true' }}
+        run: ./gradlew :metadata-ingestion:modelDocGen
+      - name: Configure AWS Credentials
+        if: ${{ needs.setup.outputs.publish == 'true' }}
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          aws-access-key-id: ${{ secrets.ACRYL_CI_ARTIFACTS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.ACRYL_CI_ARTIFACTS_ACCESS_KEY }}
+          aws-region: us-west-2
+      - name: Upload metadata to S3
+        if: ${{ needs.setup.outputs.publish == 'true' }}
+        run: aws s3 cp ./metadata-ingestion/generated/docs/metadata_model_mces.json s3://${{ secrets.ACRYL_CI_ARTIFACTS_BUCKET }}/datahub/demo/metadata/
       - name: Upload metadata to DataHub
         if: ${{ needs.setup.outputs.publish == 'true' }}
         env:

build.gradle

@@ -167,6 +167,7 @@ project.ext.externalDependency = [
     'parquetHadoop': 'org.apache.parquet:parquet-hadoop:1.13.1',
     'picocli': 'info.picocli:picocli:4.5.0',
     'playCache': "com.typesafe.play:play-cache_2.12:$playVersion",
+    'playCaffeineCache': "com.typesafe.play:play-caffeine-cache_2.12:$playVersion",
     'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.12:2.1.10',
     'playDocs': "com.typesafe.play:play-docs_2.12:$playVersion",
     'playGuice': "com.typesafe.play:play-guice_2.12:$playVersion",

datahub-frontend/build.gradle

@@ -89,6 +89,22 @@ docker {
   buildx(true)
   load(true)
   push(false)
+
+  // Add build args if they are defined (needed for some CI or enterprise environments)
+  def dockerBuildArgs = [:]
+  if (project.hasProperty('alpineApkRepositoryUrl')) {
+    dockerBuildArgs.ALPINE_REPO_URL = project.getProperty('alpineApkRepositoryUrl')
+  }
+  if (project.hasProperty('githubMirrorUrl')) {
+    dockerBuildArgs.GITHUB_REPO_URL = project.getProperty('githubMirrorUrl')
+  }
+  if (project.hasProperty('mavenCentralRepositoryUrl')) {
+    dockerBuildArgs.MAVEN_CENTRAL_REPO_URL = project.getProperty('mavenCentralRepositoryUrl')
+  }
+
+  if (dockerBuildArgs.size() > 0) {
+    buildArgs(dockerBuildArgs)
+  }
 }
 
 task unversionZip(type: Copy, dependsOn: [':datahub-web-react:build', dist]) {
@@ -104,4 +120,4 @@ task cleanLocalDockerImages {
     rootProject.ext.cleanLocalDockerImages(docker_registry, docker_repo, "${version}")
   }
 }
-dockerClean.finalizedBy(cleanLocalDockerImages)
+dockerClean.finalizedBy(cleanLocalDockerImages)

datahub-frontend/play.gradle

@@ -58,6 +58,7 @@ dependencies {
   implementation externalDependency.shiroCore
 
   implementation externalDependency.playCache
+  implementation externalDependency.playCaffeineCache
   implementation externalDependency.playWs
   implementation externalDependency.playServer
   implementation externalDependency.playAkkaHttpServer

datahub-upgrade/build.gradle

@@ -108,6 +108,22 @@ docker {
   buildx(true)
   load(true)
   push(false)
+
+  // Add build args if they are defined (needed for some CI or enterprise environments)
+  def dockerBuildArgs = [:]
+  if (project.hasProperty('alpineApkRepositoryUrl')) {
+    dockerBuildArgs.ALPINE_REPO_URL = project.getProperty('alpineApkRepositoryUrl')
+  }
+  if (project.hasProperty('githubMirrorUrl')) {
+    dockerBuildArgs.GITHUB_REPO_URL = project.getProperty('githubMirrorUrl')
+  }
+  if (project.hasProperty('mavenCentralRepositoryUrl')) {
+    dockerBuildArgs.MAVEN_CENTRAL_REPO_URL = project.getProperty('mavenCentralRepositoryUrl')
+  }
+
+  if (dockerBuildArgs.size() > 0) {
+    buildArgs(dockerBuildArgs)
+  }
 }
 tasks.getByPath(":datahub-upgrade:docker").dependsOn([bootJar])
 

datahub-web-react/.eslintrc.js

@@ -1,15 +1,14 @@
 module.exports = {
     parser: '@typescript-eslint/parser', // Specifies the ESLint parser
     extends: [
-        'react-app',
-        'plugin:react/recommended', // Uses the recommended rules from @eslint-plugin-react
-        'plugin:@typescript-eslint/recommended', // Uses the recommended rules from @typescript-eslint/eslint-plugin
-        'plugin:jest/recommended',
+        'airbnb',
         'airbnb-typescript',
         'airbnb/hooks',
+        'plugin:@typescript-eslint/recommended',
+        'plugin:jest/recommended',
         'prettier',
-        'plugin:prettier/recommended',
     ],
+    plugins: ['@typescript-eslint'],
     parserOptions: {
         ecmaVersion: 2020, // Allows for the parsing of modern ECMAScript features
         sourceType: 'module', // Allows for the use of imports
@@ -19,29 +18,34 @@ module.exports = {
         project: './tsconfig.json',
     },
     rules: {
-        eqeqeq: ['error', 'always'],
-        'react/destructuring-assignment': 'off',
-        'no-console': 'off',
-        'no-debugger': 'warn',
-        'require-await': 'warn',
+        '@typescript-eslint/no-explicit-any': 'off',
+        'arrow-body-style': 'off',
+        'class-methods-use-this': 'off',
+        'import/no-extraneous-dependencies': 'off',
         'import/prefer-default-export': 'off', // TODO: remove this lint rule
-        'import/extensions': 'off',
-        'react/jsx-props-no-spreading': 'off',
+        'no-console': 'off',
         'no-plusplus': 'off',
         'no-prototype-builtins': 'off',
-        'react/require-default-props': 'off',
+        'no-restricted-exports': ['off', { restrictedNamedExports: ['default', 'then'] }],
         'no-underscore-dangle': 'off',
+        'no-unsafe-optional-chaining': 'off',
+        'prefer-exponentiation-operator': 'off',
+        'prefer-regex-literals': 'off',
+        'react/destructuring-assignment': 'off',
+        'react/function-component-definition': 'off',
+        'react/jsx-no-bind': 'off',
+        'react/jsx-no-constructed-context-values': 'off',
+        'react/jsx-no-useless-fragment': 'off',
+        'react/jsx-props-no-spreading': 'off',
+        'react/no-unstable-nested-components': 'off',
+        'react/require-default-props': 'off',
         '@typescript-eslint/no-unused-vars': [
             'error',
             {
                 varsIgnorePattern: '^_',
                 argsIgnorePattern: '^_',
             },
         ],
-        '@typescript-eslint/no-empty-interface': 'off',
-        "@typescript-eslint/explicit-module-boundary-types": "off",
-        "@typescript-eslint/no-explicit-any": 'off',
-        "import/no-extraneous-dependencies": 'off'
     },
     settings: {
         react: {

datahub-web-react/craco.config.js

@@ -1,8 +1,11 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
 require('dotenv').config();
+const { whenProd } = require('@craco/craco');
 const CracoAntDesignPlugin = require('craco-antd');
 const path = require('path');
 const CopyWebpackPlugin = require('copy-webpack-plugin');
 
+// eslint-disable-next-line import/no-dynamic-require
 const themeConfig = require(`./src/conf/theme/${process.env.REACT_APP_THEME_CONFIG}`);
 
 function addLessPrefixToKeys(styles) {
@@ -15,6 +18,23 @@ function addLessPrefixToKeys(styles) {
 
 module.exports = {
     webpack: {
+        configure: {
+            optimization: whenProd(() => ({
+                splitChunks: {
+                    cacheGroups: {
+                        vendor: {
+                            test: /[\\/]node_modules[\\/]/,
+                            name: 'vendors',
+                            chunks: 'all',
+                        },
+                    },
+                },
+            })),
+            // Webpack 5 no longer automatically pollyfill core Node.js modules
+            resolve: { fallback: { fs: false } },
+            // Ignore Webpack 5's missing source map warnings from node_modules
+            ignoreWarnings: [{ module: /node_modules/, message: /source-map-loader/ }],
+        },
         plugins: {
             add: [
                 // Self host images by copying them to the build directory
@@ -24,8 +44,8 @@ module.exports = {
                 // Copy monaco-editor files to the build directory
                 new CopyWebpackPlugin({
                     patterns: [
-                        { from: "node_modules/monaco-editor/min/vs/", to: "monaco-editor/vs" },
-                        { from: "node_modules/monaco-editor/min-maps/vs/", to: "monaco-editor/min-maps/vs" },
+                        { from: 'node_modules/monaco-editor/min/vs/', to: 'monaco-editor/vs' },
+                        { from: 'node_modules/monaco-editor/min-maps/vs/', to: 'monaco-editor/min-maps/vs' },
                     ],
                 }),
             ],
@@ -41,13 +61,15 @@ module.exports = {
         },
     ],
     jest: {
-        configure: (jestConfig) => {
-            jestConfig.transformIgnorePatterns = [
-                // Ensures that lib0 and y-protocol libraries are transformed through babel as well
-                'node_modules/(?!(lib0|y-protocols)).+\\.(js|jsx|mjs|cjs|ts|tsx)$',
-                '^.+\\.module\\.(css|sass|scss)$',
-            ];
-            return jestConfig;
+        configure: {
+            // Use dist files instead of source files
+            moduleNameMapper: {
+                '^d3-interpolate-path': `d3-interpolate-path/build/d3-interpolate-path`,
+                '^d3-(.*)$': `d3-$1/dist/d3-$1`,
+                '^lib0/((?!dist).*)$': 'lib0/dist/$1.cjs',
+                '^y-protocols/(.*)$': 'y-protocols/dist/$1.cjs',
+                '\\.(css|less)$': '<rootDir>/src/__mocks__/styleMock.js',
+            },
         },
     },
 };

datahub-web-react/package.json

@@ -9,12 +9,12 @@
         "@ant-design/colors": "^5.0.0",
         "@ant-design/icons": "^4.3.0",
         "@apollo/client": "^3.3.19",
-        "@craco/craco": "^6.1.1",
-        "@data-ui/xy-chart": "^0.0.84",
+        "@craco/craco": "^7.1.0",
         "@graphql-codegen/fragment-matcher": "^5.0.0",
         "@miragejs/graphql": "^0.1.11",
         "@monaco-editor/react": "^4.3.1",
         "@react-hook/window-size": "^3.0.7",
+        "@react-spring/web": "^9.7.3",
         "@remirror/pm": "^2.0.3",
         "@remirror/react": "^2.0.24",
         "@remirror/styles": "^2.0.3",
@@ -30,21 +30,22 @@
         "@types/react-router": "^5.1.8",
         "@types/react-router-dom": "^5.1.6",
         "@uiw/react-md-editor": "^3.3.4",
-        "@vx/axis": "^0.0.199",
-        "@vx/glyph": "^0.0.199",
-        "@vx/group": "^0.0.199",
-        "@vx/hierarchy": "^0.0.199",
-        "@vx/legend": "^0.0.199",
-        "@vx/scale": "^0.0.199",
-        "@vx/shape": "^0.0.199",
-        "@vx/tooltip": "^0.0.199",
-        "@vx/zoom": "^0.0.199",
+        "@visx/axis": "^3.1.0",
+        "@visx/curve": "^3.0.0",
+        "@visx/group": "^3.0.0",
+        "@visx/hierarchy": "^3.0.0",
+        "@visx/legend": "^3.2.0",
+        "@visx/scale": "^3.2.0",
+        "@visx/shape": "^3.2.0",
+        "@visx/xychart": "^3.2.0",
+        "@visx/zoom": "^3.1.1",
         "analytics": "^0.8.9",
         "antd": "4.24.7",
         "color-hash": "^2.0.1",
-        "craco-antd": "^1.19.0",
+        "craco-antd": "^2.0.0",
+        "craco-babel-loader": "^1.0.4",
         "cronstrue": "^1.122.0",
-        "d3-scale": "^3.3.0",
+        "d3-scale": "^4.0.2",
         "dayjs": "^1.11.7",
         "deepmerge": "^4.2.2",
         "diff": "^5.0.0",
@@ -67,14 +68,13 @@
         "react": "^17.0.0",
         "react-color": "^2.19.3",
         "react-dom": "^17.0.0",
-        "react-email-share-link": "^1.0.3",
         "react-helmet-async": "^1.3.0",
         "react-highlighter": "^0.4.3",
         "react-icons": "4.3.1",
         "react-js-cron": "^2.1.0",
         "react-router": "^5.2.0",
         "react-router-dom": "^5.1.6",
-        "react-scripts": "4.0.3",
+        "react-scripts": "^5.0.1",
         "react-syntax-highlighter": "^15.4.4",
         "react-visibility-sensor": "^5.1.1",
         "reactour": "1.18.7",
@@ -83,7 +83,7 @@
         "start-server-and-test": "1.12.2",
         "styled-components": "^5.2.1",
         "turndown-plugin-gfm": "^1.0.2",
-        "typescript": "^4.1.3",
+        "typescript": "^4.8.4",
         "uuid": "^8.3.2",
         "virtualizedtableforantd4": "^1.2.1",
         "web-vitals": "^0.2.4",
@@ -123,28 +123,37 @@
         ]
     },
     "devDependencies": {
+        "@babel/plugin-proposal-private-property-in-object": "^7.21.11",
         "@graphql-codegen/cli": "1.20.0",
         "@graphql-codegen/near-operation-file-preset": "^1.17.13",
         "@graphql-codegen/typescript-operations": "1.17.13",
         "@graphql-codegen/typescript-react-apollo": "2.2.1",
         "@types/graphql": "^14.5.0",
         "@types/query-string": "^6.3.0",
         "@types/styled-components": "^5.1.7",
-        "@typescript-eslint/eslint-plugin": "^4.25.0",
-        "@typescript-eslint/parser": "^4.25.0",
-        "babel-loader": "8.2.2",
-        "copy-webpack-plugin": "6.4.1",
-        "eslint": "^7.27.0",
-        "eslint-config-airbnb-typescript": "^12.3.1",
-        "eslint-config-prettier": "^8.3.0",
-        "eslint-plugin-prettier": "^3.4.0",
-        "eslint-plugin-react": "^7.23.2",
+        "@typescript-eslint/eslint-plugin": "^5.38.1",
+        "@typescript-eslint/parser": "^5.38.1",
+        "copy-webpack-plugin": "^11.0.0",
+        "eslint": "^8.2.0",
+        "eslint-config-airbnb": "19.0.4",
+        "eslint-config-airbnb-typescript": "^17.0.0",
+        "eslint-config-prettier": "^8.8.0",
+        "eslint-plugin-import": "^2.25.3",
+        "eslint-plugin-jsx-a11y": "^6.5.1",
+        "eslint-plugin-react": "^7.28.0",
+        "eslint-plugin-react-hooks": "^4.3.0",
         "http-proxy-middleware": "2.0.0",
-        "prettier": "^2.3.0",
+        "prettier": "^2.8.8",
         "source-map-explorer": "^2.5.2"
     },
     "resolutions": {
         "@ant-design/colors": "6.0.0",
-        "refractor": "3.3.1"
+        "refractor": "3.3.1",
+        "json-schema": "0.4.0",
+        "@babel/traverse": ">=7.23.2",
+        "ansi-regex": "3.0.1",
+        "minimatch": "3.0.5",
+        "prismjs": "^1.27.0",
+        "nth-check": "^2.0.1"
     }
 }

datahub-web-react/src/__mocks__/styleMock.js

@@ -0,0 +1 @@
+module.exports = {};

datahub-web-react/src/app/analyticsDashboard/components/BarChart.tsx

@@ -1,8 +1,8 @@
 import React, { useMemo } from 'react';
-import { BarStack } from '@vx/shape';
-import { scaleOrdinal, scaleLinear, scaleBand } from '@vx/scale';
-import { Group } from '@vx/group';
-import { AxisBottom, AxisRight } from '@vx/axis';
+import { BarStack } from '@visx/shape';
+import { scaleOrdinal, scaleLinear, scaleBand } from '@visx/scale';
+import { Group } from '@visx/group';
+import { AxisBottom, AxisRight } from '@visx/axis';
 
 import { BarChart as BarChartType } from '../../../types.generated';
 import { lineColors } from './lineColors';
@@ -85,7 +85,7 @@ export const BarChart = ({ chartData, width, height }: Props) => {
             <svg width={width + WIDTH_MARGIN_SIZE} height={height}>
                 <rect x={0} y={0} width={width} height={height} fill="white" rx={14} />
                 <Group top={HEIGHT_MARGIN_SIZE} left={WIDTH_MARGIN_SIZE}>
-                    <BarStack<typeof transformedChartData[0], typeof keys[number]>
+                    <BarStack<(typeof transformedChartData)[0], (typeof keys)[number]>
                         data={transformedChartData}
                         keys={keys}
                         x={(data) => data.displayName}

datahub-web-react/src/app/analyticsDashboard/components/Legend.tsx

@@ -1,6 +1,6 @@
 import React from 'react';
 import { Col, Row } from 'antd';
-import { LegendOrdinal, LegendItem, LegendLabel } from '@vx/legend';
+import { LegendOrdinal, LegendItem, LegendLabel } from '@visx/legend';
 import { ScaleOrdinal } from 'd3-scale/src/ordinal';
 import styled from 'styled-components';