Merge branch 'master' into fix/PRD-804

Salman-Apptware · Nov 30, 2023 · 9eb398a · 9eb398a
2 parents d6cf135 + dd09f5e
commit 9eb398a
Show file tree

Hide file tree

Showing 86 changed files with 1,738 additions and 1,204 deletions.
diff --git a/.github/workflows/metadata-model.yml b/.github/workflows/metadata-model.yml
@@ -37,6 +37,19 @@ jobs:
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Run model generation
         run: ./gradlew :metadata-models:build
+      - name: Generate metadata files
+        if: ${{ needs.setup.outputs.publish == 'true' }}
+        run: ./gradlew :metadata-ingestion:modelDocGen
+      - name: Configure AWS Credentials
+        if: ${{ needs.setup.outputs.publish == 'true' }}
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          aws-access-key-id: ${{ secrets.ACRYL_CI_ARTIFACTS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.ACRYL_CI_ARTIFACTS_ACCESS_KEY }}
+          aws-region: us-west-2
+      - name: Upload metadata to S3
+        if: ${{ needs.setup.outputs.publish == 'true' }}
+        run: aws s3 cp ./metadata-ingestion/generated/docs/metadata_model_mces.json s3://${{ secrets.ACRYL_CI_ARTIFACTS_BUCKET }}/datahub/demo/metadata/
       - name: Upload metadata to DataHub
         if: ${{ needs.setup.outputs.publish == 'true' }}
         env:

diff --git a/build.gradle b/build.gradle
@@ -167,6 +167,7 @@ project.ext.externalDependency = [
     'parquetHadoop': 'org.apache.parquet:parquet-hadoop:1.13.1',
     'picocli': 'info.picocli:picocli:4.5.0',
     'playCache': "com.typesafe.play:play-cache_2.12:$playVersion",
+    'playCaffeineCache': "com.typesafe.play:play-caffeine-cache_2.12:$playVersion",
     'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.12:2.1.10',
     'playDocs': "com.typesafe.play:play-docs_2.12:$playVersion",
     'playGuice': "com.typesafe.play:play-guice_2.12:$playVersion",

diff --git a/datahub-frontend/build.gradle b/datahub-frontend/build.gradle
@@ -89,6 +89,22 @@ docker {
   buildx(true)
   load(true)
   push(false)
+
+  // Add build args if they are defined (needed for some CI or enterprise environments)
+  def dockerBuildArgs = [:]
+  if (project.hasProperty('alpineApkRepositoryUrl')) {
+    dockerBuildArgs.ALPINE_REPO_URL = project.getProperty('alpineApkRepositoryUrl')
+  }
+  if (project.hasProperty('githubMirrorUrl')) {
+    dockerBuildArgs.GITHUB_REPO_URL = project.getProperty('githubMirrorUrl')
+  }
+  if (project.hasProperty('mavenCentralRepositoryUrl')) {
+    dockerBuildArgs.MAVEN_CENTRAL_REPO_URL = project.getProperty('mavenCentralRepositoryUrl')
+  }
+
+  if (dockerBuildArgs.size() > 0) {
+    buildArgs(dockerBuildArgs)
+  }
 }
 
 task unversionZip(type: Copy, dependsOn: [':datahub-web-react:build', dist]) {
@@ -104,4 +120,4 @@ task cleanLocalDockerImages {
     rootProject.ext.cleanLocalDockerImages(docker_registry, docker_repo, "${version}")
   }
 }
-dockerClean.finalizedBy(cleanLocalDockerImages)
+dockerClean.finalizedBy(cleanLocalDockerImages)
diff --git a/datahub-frontend/play.gradle b/datahub-frontend/play.gradle
@@ -58,6 +58,7 @@ dependencies {
   implementation externalDependency.shiroCore
 
   implementation externalDependency.playCache
+  implementation externalDependency.playCaffeineCache
   implementation externalDependency.playWs
   implementation externalDependency.playServer
   implementation externalDependency.playAkkaHttpServer

diff --git a/datahub-upgrade/build.gradle b/datahub-upgrade/build.gradle
@@ -108,6 +108,22 @@ docker {
   buildx(true)
   load(true)
   push(false)
+
+  // Add build args if they are defined (needed for some CI or enterprise environments)
+  def dockerBuildArgs = [:]
+  if (project.hasProperty('alpineApkRepositoryUrl')) {
+    dockerBuildArgs.ALPINE_REPO_URL = project.getProperty('alpineApkRepositoryUrl')
+  }
+  if (project.hasProperty('githubMirrorUrl')) {
+    dockerBuildArgs.GITHUB_REPO_URL = project.getProperty('githubMirrorUrl')
+  }
+  if (project.hasProperty('mavenCentralRepositoryUrl')) {
+    dockerBuildArgs.MAVEN_CENTRAL_REPO_URL = project.getProperty('mavenCentralRepositoryUrl')
+  }
+
+  if (dockerBuildArgs.size() > 0) {
+    buildArgs(dockerBuildArgs)
+  }
 }
 tasks.getByPath(":datahub-upgrade:docker").dependsOn([bootJar])
 

diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json
@@ -150,6 +150,8 @@
         "@ant-design/colors": "6.0.0",
         "refractor": "3.3.1",
         "json-schema": "0.4.0",
-        "@babel/traverse": ">=7.23.2"
+        "@babel/traverse": ">=7.23.2",
+        "prismjs": "^1.27.0",
+        "nth-check": "^2.0.1"
     }
 }
diff --git a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx
@@ -319,7 +319,7 @@ export default function PolicyPrivilegeForm({
                             .filter((privs) => privs.resourceType !== 'all')
                             .map((resPrivs) => {
                                 return (
-                                    <Select.Option value={resPrivs.resourceType}>
+                                    <Select.Option key={resPrivs.resourceType} value={resPrivs.resourceType}>
                                         {resPrivs.resourceTypeDisplayName}
                                     </Select.Option>
                                 );
@@ -355,7 +355,9 @@ export default function PolicyPrivilegeForm({
                         )}
                     >
                         {resourceSearchResults?.map((result) => (
-                            <Select.Option value={result.entity.urn}>{renderSearchResult(result)}</Select.Option>
+                            <Select.Option key={result.entity.urn} value={result.entity.urn}>
+                                {renderSearchResult(result)}
+                            </Select.Option>
                         ))}
                     </Select>
                 </Form.Item>
@@ -389,7 +391,9 @@ export default function PolicyPrivilegeForm({
                             dropdownStyle={isShowingDomainNavigator ? { display: 'none' } : {}}
                         >
                             {domainSearchResults?.map((result) => (
-                                <Select.Option value={result.entity.urn}>{renderSearchResult(result)}</Select.Option>
+                                <Select.Option key={result.entity.urn} value={result.entity.urn}>
+                                    {renderSearchResult(result)}
+                                </Select.Option>
                             ))}
                         </Select>
                         <BrowserWrapper isHidden={!isShowingDomainNavigator} width="100%" maxHeight={300}>
@@ -412,9 +416,14 @@ export default function PolicyPrivilegeForm({
                         </Tag>
                     )}
                 >
-                    {privilegeOptions.map((priv) => (
-                        <Select.Option value={priv.type}>{priv.displayName}</Select.Option>
-                    ))}
+                    {privilegeOptions.map((priv, index) => {
+                        const key = `${priv.type}-${index}`;
+                        return (
+                            <Select.Option key={key} value={priv.type}>
+                                {priv.displayName}
+                            </Select.Option>
+                        );
+                    })}
                     <Select.Option value="All">All Privileges</Select.Option>
                 </Select>
             </Form.Item>

diff --git a/datahub-web-react/src/app/search/sidebar/BrowseSidebar.tsx b/datahub-web-react/src/app/search/sidebar/BrowseSidebar.tsx
@@ -26,12 +26,12 @@ const SidebarHeader = styled.div`
     white-space: nowrap;
 `;
 
-const SidebarBody = styled.div`
+const SidebarBody = styled.div<{ visible: boolean }>`
     height: calc(100% - 47px);
     padding-left: 16px;
     padding-right: 12px;
     padding-bottom: 200px;
-    overflow: auto;
+    overflow: ${(props) => (props.visible ? 'auto' : 'hidden')};
     white-space: nowrap;
 `;
 
@@ -50,7 +50,7 @@ const BrowseSidebar = ({ visible, width }: Props) => {
             <SidebarHeader>
                 <Typography.Text strong>Navigate</Typography.Text>
             </SidebarHeader>
-            <SidebarBody>
+            <SidebarBody visible={visible}>
                 {entityAggregations && !entityAggregations.length && <div>No results found</div>}
                 {entityAggregations?.map((entityAggregation) => (
                     <BrowseProvider key={entityAggregation.value} entityAggregation={entityAggregation}>

diff --git a/datahub-web-react/src/app/shared/LogoCountCard.tsx b/datahub-web-react/src/app/shared/LogoCountCard.tsx
@@ -7,6 +7,7 @@ import { HomePageButton } from './components';
 
 const PlatformLogo = styled(Image)`
     max-height: 32px;
+    height: 32px;
     width: auto;
     object-fit: contain;
     background-color: transparent;

diff --git a/datahub-web-react/yarn.lock b/datahub-web-react/yarn.lock
@@ -6063,7 +6063,7 @@ bonjour-service@^1.0.11:
     fast-deep-equal "^3.1.3"
     multicast-dns "^7.2.5"
 
-boolbase@^1.0.0, boolbase@~1.0.0:
+boolbase@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e"
   integrity sha1-aN/1++YMUes3cl6p4+0xDcwed24=
@@ -6502,15 +6502,6 @@ cli-width@^3.0.0:
   resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-3.0.0.tgz#a2f48437a2caa9a22436e794bf071ec9e61cedf6"
   integrity sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==
 
-clipboard@^2.0.0:
-  version "2.0.8"
-  resolved "https://registry.yarnpkg.com/clipboard/-/clipboard-2.0.8.tgz#ffc6c103dd2967a83005f3f61976aa4655a4cdba"
-  integrity sha512-Y6WO0unAIQp5bLmk1zdThRhgJt/x3ks6f30s3oE3H1mgIEU33XyQjEf8gsf6DxC7NPX8Y1SsNWjUjL/ywLnnbQ==
-  dependencies:
-    good-listener "^1.2.2"
-    select "^1.1.2"
-    tiny-emitter "^2.0.0"
-
 cliui@^6.0.0:
   version "6.0.0"
   resolved "https://registry.yarnpkg.com/cliui/-/cliui-6.0.0.tgz#511d702c0c4e41ca156d7d0e96021f23e13225b1"
@@ -7454,11 +7445,6 @@ delayed-stream@~1.0.0:
   resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
   integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk=
 
-delegate@^3.1.2:
-  version "3.2.0"
-  resolved "https://registry.yarnpkg.com/delegate/-/delegate-3.2.0.tgz#b66b71c3158522e8ab5744f720d8ca0c2af59166"
-  integrity sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw==
-
 [email protected]:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df"
@@ -9032,13 +9018,6 @@ globby@^13.1.1:
     merge2 "^1.4.1"
     slash "^4.0.0"
 
-good-listener@^1.2.2:
-  version "1.2.2"
-  resolved "https://registry.yarnpkg.com/good-listener/-/good-listener-1.2.2.tgz#d53b30cdf9313dffb7dc9a0d477096aa6d145c50"
-  integrity sha1-1TswzfkxPf+33JoNR3CWqm0UXFA=
-  dependencies:
-    delegate "^3.1.2"
-
 got@^9.6.0:
   version "9.6.0"
   resolved "https://registry.yarnpkg.com/got/-/got-9.6.0.tgz#edf45e7d67f99545705de1f7bbeeeb121765ed85"
@@ -12182,14 +12161,7 @@ npm-run-path@^4.0.1:
   dependencies:
     path-key "^3.0.0"
 
-nth-check@^1.0.2:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-1.0.2.tgz#b2bd295c37e3dd58a3bf0700376663ba4d9cf05c"
-  integrity sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==
-  dependencies:
-    boolbase "~1.0.0"
-
-nth-check@^2.0.1:
+nth-check@^1.0.2, nth-check@^2.0.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.1.1.tgz#c9eab428effce36cd6b92c924bdb000ef1f1ed1d"
   integrity sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==
@@ -13322,17 +13294,10 @@ pretty-format@^28.1.3:
     ansi-styles "^5.0.0"
     react-is "^18.0.0"
 
-prismjs@^1.22.0:
-  version "1.24.1"
-  resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.24.1.tgz#c4d7895c4d6500289482fa8936d9cdd192684036"
-  integrity sha512-mNPsedLuk90RVJioIky8ANZEwYm5w9LcvCXrxHlwf4fNVSn8jEipMybMkWUyyF0JhnC+C4VcOVSBuHRKs1L5Ow==
-
-prismjs@~1.23.0:
-  version "1.23.0"
-  resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.23.0.tgz#d3b3967f7d72440690497652a9d40ff046067f33"
-  integrity sha512-c29LVsqOaLbBHuIbsTxaKENh1N2EQBOHaWv7gkHN4dgRbxSREqDnDbtFJYdpPauS4YCplMSNCABQ6Eeor69bAA==
-  optionalDependencies:
-    clipboard "^2.0.0"
+prismjs@^1.22.0, prismjs@^1.27.0, prismjs@~1.23.0:
+  version "1.29.0"
+  resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.29.0.tgz#f113555a8fa9b57c35e637bba27509dcf802dd12"
+  integrity sha512-Kx/1w86q/epKcmte75LNrEoT+lX8pBpavuAbvJWRXar7Hz8jrtF+e3vY751p0R8H9HdArwaCTNDDzHg/ScJK1Q==
 
 process-nextick-args@~2.0.0:
   version "2.0.1"
@@ -15099,11 +15064,6 @@ select-hose@^2.0.0:
   resolved "https://registry.yarnpkg.com/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca"
   integrity sha1-Yl2GWPhlr0Psliv8N2o3NZpJlMo=
 
-select@^1.1.2:
-  version "1.1.2"
-  resolved "https://registry.yarnpkg.com/select/-/select-1.1.2.tgz#0e7350acdec80b1108528786ec1d4418d11b396d"
-  integrity sha1-DnNQrN7ICxEIUoeG7B1EGNEbOW0=
-
 selfsigned@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/selfsigned/-/selfsigned-2.1.1.tgz#18a7613d714c0cd3385c48af0075abf3f266af61"
@@ -16127,11 +16087,6 @@ thunky@^1.0.2:
   resolved "https://registry.yarnpkg.com/thunky/-/thunky-1.1.0.tgz#5abaf714a9405db0504732bbccd2cedd9ef9537d"
   integrity sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA==
 
-tiny-emitter@^2.0.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/tiny-emitter/-/tiny-emitter-2.1.0.tgz#1d1a56edfc51c43e863cbb5382a72330e3555423"
-  integrity sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==
-
 tiny-invariant@^1.0.2:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/tiny-invariant/-/tiny-invariant-1.1.0.tgz#634c5f8efdc27714b7f386c35e6760991d230875"

diff --git a/docker/build.gradle b/docker/build.gradle
@@ -15,6 +15,7 @@ ext {
             ':metadata-service:war',
             ':datahub-frontend',
     ]
+
     debug_modules = quickstart_modules - [':metadata-jobs:mce-consumer-job',
                                           ':metadata-jobs:mae-consumer-job']
     debug_compose_args = [
@@ -27,6 +28,13 @@ ext {
             'datahub-gms',
             'datahub-frontend-react'
     ]
+
+    // Postgres
+    pg_quickstart_modules = quickstart_modules - [':docker:mysql-setup'] + [':docker:postgres-setup']
+    pg_compose_args = [
+            '-f', 'docker-compose-without-neo4j.yml',
+            '-f', 'docker-compose-without-neo4j.postgres.override.yml'
+    ]
 }
 
 task quickstart(type: Exec, dependsOn: ':metadata-ingestion:install') {
@@ -125,3 +133,29 @@ task debugReload(type: Exec) {
     def cmd = ['docker compose -p datahub'] + debug_compose_args + ['restart'] + debug_reloadable
     commandLine 'bash', '-c', cmd.join(" ")
 }
+
+task quickstartPg(type: Exec, dependsOn: ':metadata-ingestion:install') {
+    dependsOn(pg_quickstart_modules.collect { it + ':dockerTag' })
+    shouldRunAfter ':metadata-ingestion:clean', 'quickstartNuke'
+
+    environment "DATAHUB_TELEMETRY_ENABLED", "false"
+    environment "DOCKER_COMPOSE_BASE", "file://${rootProject.projectDir}"
+    environment "DATAHUB_POSTGRES_VERSION", "15.5"
+
+    // OpenSearch
+    environment "DATAHUB_SEARCH_IMAGE", 'opensearchproject/opensearch'
+    environment "DATAHUB_SEARCH_TAG", '2.9.0'
+    environment "XPACK_SECURITY_ENABLED", 'plugins.security.disabled=true'
+    environment "USE_AWS_ELASTICSEARCH", 'true'
+
+    def cmd = [
+            'source ../metadata-ingestion/venv/bin/activate && ',
+            'datahub docker quickstart',
+            '--no-pull-images',
+            '--standalone_consumers',
+            '--version', "v${version}",
+            '--dump-logs-on-failure'
+    ] + pg_compose_args
+
+    commandLine 'bash', '-c', cmd.join(" ")
+}
diff --git a/docker/datahub-frontend/Dockerfile b/docker/datahub-frontend/Dockerfile
@@ -3,14 +3,22 @@ ARG APP_ENV=prod
 
 FROM alpine:3 AS base
 
+# Configurable repositories
+ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine
+ARG GITHUB_REPO_URL=https://github.com
+ARG MAVEN_CENTRAL_REPO_URL=https://repo1.maven.org/maven2
+
 RUN addgroup -S datahub && adduser -S datahub -G datahub
 
+# Optionally set corporate mirror for apk
+RUN if [ "${ALPINE_REPO_URL}" != "http://dl-cdn.alpinelinux.org/alpine" ] ; then sed -i "s#http.*://dl-cdn.alpinelinux.org/alpine#${ALPINE_REPO_URL}#g" /etc/apk/repositories ; fi
+
 # Upgrade Alpine and base packages
 # PFP-260: Upgrade Sqlite to >=3.28.0-r0 to fix https://security.snyk.io/vuln/SNYK-ALPINE39-SQLITE-449762
 RUN apk --no-cache --update-cache --available upgrade \
     && apk --no-cache add curl sqlite libc6-compat java-snappy \
-    && apk --no-cache add openjdk11-jre-headless --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
-    && apk --no-cache add jattach --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/
+    && apk --no-cache add openjdk11-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \
+    && apk --no-cache add jattach --repository ${ALPINE_REPO_URL}/edge/community/
 
 ENV LD_LIBRARY_PATH="/lib:/lib64"
 
@@ -22,8 +30,8 @@ COPY ./docker/monitoring/client-prometheus-config.yaml /datahub-frontend/
 RUN chown -R datahub:datahub /datahub-frontend && chmod 755 /datahub-frontend
 
 ENV JMX_VERSION=0.18.0
-RUN wget https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar -O opentelemetry-javaagent.jar \
-    && wget https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar
+RUN wget ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar -O opentelemetry-javaagent.jar \
+    && wget ${MAVEN_CENTRAL_REPO_URL}/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar
 
 FROM base as dev-install
 # Dummy stage for development. Assumes code is built on your machine and mounted to this image.